From d93debc4a9907dab7de692f3bc0f4a2e254136d4 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Tue, 26 Mar 2019 19:06:49 +0100
Subject: [PATCH 2/4] matomo: manage matomo's webservices (#31778)

---
 hobo/matomo/utils.py       | 292 ++++++++++++++++
 hobo/settings.py           |  17 +
 requirements.txt           |   1 +
 tests/test_matomo_utils.py | 662 +++++++++++++++++++++++++++++++++++++
 tox.ini                    |   1 +
 5 files changed, 973 insertions(+)
 create mode 100644 hobo/matomo/utils.py
 create mode 100644 tests/test_matomo_utils.py

diff --git a/hobo/matomo/utils.py b/hobo/matomo/utils.py
new file mode 100644
index 0000000..7603487
--- /dev/null
+++ b/hobo/matomo/utils.py
@@ -0,0 +1,292 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2015  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import requests
+import string
+
+from lxml import etree
+from random import *
+
+from django.db import connection
+from django.conf import settings
+from django.core import exceptions
+from django.utils.six.moves.urllib import parse as urlparse
+
+from hobo.environment.models import Variable, Wcs, Combo, Fargo
+
+
+def get_variable(name, default=''):
+    """get hobo variables from DB
+    set it to '' into DB if not already created
+    """
+    variable, created = Variable.objects.get_or_create(
+        name=name,
+        defaults={'auto': True, 'value': default})
+    return variable
+
+def get_variable_value(name, default=''):
+    """get hobo variables's value from DB"""
+    try:
+        value = Variable.objects.get(name=name).value
+    except exceptions.ObjectDoesNotExist:
+        value = default
+    return value
+
+def get_tracking_js():
+    """merge JS code from the 2 above variables"""
+    tracking_js = get_variable_value('cnil_compliant_visits_tracking_js')
+    tracking_js += get_variable_value('visits_tracking_js')
+    return tracking_js
+
+def put_tracking_js(tracking_js):
+    """store JS code into only one of the 2 above variables"""
+    variable1 = get_variable('cnil_compliant_visits_tracking_js')
+    variable2 = get_variable('visits_tracking_js')
+    if compute_cnil_acknowledgment_level(tracking_js) != 'error':
+        variable1.value = tracking_js
+        variable1.save()
+        variable2.delete()
+    else:
+        variable1.delete()
+        variable2.value = tracking_js
+        variable2.save()
+
+def get_tenant_name_and_public_urls():
+    """get an alias for our matomo's id and urls to monitor"""
+    tenant_name = None
+    services = [x for x in Combo.objects.all() if x.template_name == 'portal-user']
+    if services != [] and services[0] != '':
+        tenant_name = urlparse.urlparse(services[0].base_url).netloc
+    services += [x for x in Wcs.objects.all()]
+    services += [x for x in Fargo.objects.all()]
+    site_urls = [x.base_url for x in services if x.base_url != '']
+    return tenant_name, site_urls
+
+class MatomoException(Exception):
+    """unexpected Matomo internal error"""
+
+class MatomoError(MatomoException):
+    """expected Matomo error responses"""
+
+class MatomoWS(object):
+    """api for matomo webservices"""
+
+    def __init__(self):
+        config = settings.MATOMO_SERVER
+        try:
+            self.url_ws_base = config['URL']
+            self.token_auth = config['TOKEN_AUTH']
+            self.email_template = config['EMAIL_TEMPLATE']
+        except KeyError as exc:
+            raise MatomoError('no settings for matomo: %s' % str(exc))
+
+    @staticmethod
+    def parse_response(content):
+        try:
+            tree = etree.fromstring(content)
+        except etree.XMLSyntaxError as exc:
+            raise MatomoException('etree.XMLSyntaxError: %s' % str(exc))
+        return tree
+
+    @staticmethod
+    def raise_on_error(tree):
+        """handle matomo XML error messages"""
+        tags = tree.xpath('/result/error')
+        if tags != []:
+            try:
+                attr = tags[0].items()[0]
+                if attr[0] == 'message':
+                    raise MatomoError(attr[1])
+            except IndexError:
+                pass
+            raise MatomoException('internal error')
+
+    def call(self, data):
+        data['module'] = 'API'
+        data['token_auth'] = self.token_auth
+        resp = requests.post(self.url_ws_base, data=data)
+        tree = self.parse_response(resp.content)
+        self.raise_on_error(tree)
+        return tree
+
+    def get_site_from_id(self, id_site):
+        data = {'method': 'SitesManager.getSiteFromId',
+                'idSite': id_site}
+        tree = self.call(data)
+        try:
+            tag = tree.xpath('/result/row/idsite')[0]
+        except IndexError:
+            raise MatomoException('get_site_from_id fails')
+        return tag.text
+
+    def add_site(self, site_name, site_urls):
+        data = {'method': 'SitesManager.addSite',
+                'siteName': site_name}
+        cpt = 0
+        for url in site_urls:
+            key = 'urls[%i]' % cpt
+            data[key] = url
+            cpt += 1
+        tree = self.call(data)
+        try:
+            tag = tree.xpath('/result')[0]
+        except IndexError:
+            raise MatomoException('add_site fails')
+        return tag.text
+
+    def add_user(self, user_login, password, initial_id_site):
+        data = {'method': 'UsersManager.addUser',
+                'userLogin': user_login,
+                'password': password,
+                'email': self.email_template % user_login,
+                'initialIdSite': initial_id_site}
+        tree = self.call(data)
+        success = True
+        tags = tree.xpath('/result/success')
+        if tags != []:
+            try:
+                attr = tags[0].items()[0]
+                if attr[0] != 'message' or attr[1] != 'ok':
+                    success = False
+            except IndexError:
+                success = False
+        if not success:
+            raise MatomoException('add_user fails')
+
+    def get_javascript_tag(self, id_site):
+        data = {'method': 'SitesManager.getJavascriptTag',
+                'idSite': id_site}
+        tree = self.call(data)
+        try:
+            tag = tree.xpath('/result')[0]
+        except IndexError:
+            raise MatomoException('get_javascript_tag fails')
+        return tag.text
+
+def is_site_already_added(matomo, id_site):
+    # API does not allow to retrieve our site without using the matomo's id.
+    try:
+        id_site_resp = matomo.get_site_from_id(id_site)
+    except MatomoError:
+        return False
+    return id_site_resp == id_site
+
+def add_user_if_not_already_there(matomo, user_login, password, id_site):
+    try:
+        matomo.add_user(user_login, password, id_site)
+    except MatomoError as exc:
+        # not an error if user is already here
+        if str(exc) != "Username '%s' already exists." % user_login:
+            raise exc
+
+def get_matomo_javascript_tag(matomo, id_site):
+    """addapt JS return by Matomo and merge it whith previous JS code we have"""
+    CNIL_JS = """
+  // disallow cookie's time extension
+  _paq.push([function() {
+    var self = this;
+    function getOriginalVisitorCookieTimeout() {
+      var now = new Date(),
+      nowTs = Math.round(now.getTime() / 1000),
+      visitorInfo = self.getVisitorInfo();
+      var createTs = parseInt(visitorInfo[2]);
+      var cookieTimeout = 33696000; // 13 mois en secondes
+      var originalTimeout = createTs + cookieTimeout - nowTs;
+      return originalTimeout;
+    }
+    this.setVisitorCookieTimeout( getOriginalVisitorCookieTimeout() );
+  }]);
+"""
+    matomo_tag = matomo.get_javascript_tag(id_site)
+    lines = matomo_tag.split('\n')
+
+    # acording to publik-base-theme/templates/includes/tracking.html,
+    # we need to remove <script> tags from matomo's output javascript,
+    regex = re.compile('</?script.*>')
+    count = len(lines)
+    while count > 0:
+        count -= 1
+        if regex.match(lines[count]):
+            del lines[count]
+
+    # and we also need to addapt matomo HTML comments to JS
+    regex = re.compile('<!-- (.*) -->')
+    for count, line in enumerate(lines):
+        lines[count] = regex.sub('// \\1', line)
+
+    # disallow cookie's time extension
+    regex = re.compile(r'\s*var _paq = window._paq \|\| \[\];')
+    for count, line in enumerate(lines):
+        if regex.match(line):
+            lines.insert(count+1, CNIL_JS)
+            break
+
+    enhanced_tag = '\n'.join(lines)
+    return enhanced_tag
+
+def compute_cnil_acknowledgment_level(tracking_js):
+    if tracking_js.find('google') != -1:
+        # google reference found into javascript
+        return 'error'
+    if tracking_js.find('getOriginalVisitorCookieTimeout') == -1:
+        # can't find cookie's life time extension prevention
+        return 'warning'
+    return 'success'
+
+def auto_configure_matomo():
+    """main function"""
+
+    # retrieve matomo's variables
+    id_site_var = get_variable('matomo_id_site')
+    password_var = get_variable('matomo_password')
+    error_message_var = get_variable('matomo_error')
+
+    id_site = id_site_var.value
+    password = password_var.value
+    if password_var.value == '':
+        characters = string.ascii_letters + string.punctuation  + string.digits
+        password = "".join(choice(characters) for x in range(randint(8, 16)))
+
+    try:
+        tenant_name, site_urls = get_tenant_name_and_public_urls()
+        if tenant_name is None:
+            raise MatomoException("no portal-user url available for a matomo's id")
+
+        matomo = MatomoWS()
+
+        # create an account
+        if not is_site_already_added(matomo, id_site):
+            id_site = matomo.add_site(tenant_name, site_urls)
+        add_user_if_not_already_there(matomo, tenant_name, password, id_site)
+
+        # overwrite tracking_js (previous content is lost)
+        tracking_js = get_matomo_javascript_tag(matomo, id_site)
+
+    except MatomoException as exc:
+        error_message_var.value = str(exc)
+        error_message_var.save()
+        return False
+
+    # save matomo's variables
+    id_site_var.value = id_site
+    password_var.value = password
+    error_message_var.value = ''
+    id_site_var.save()
+    password_var.save()
+    error_message_var.save()
+    put_tracking_js(tracking_js)
+    return True
diff --git a/hobo/settings.py b/hobo/settings.py
index 0086f68..3678eef 100644
--- a/hobo/settings.py
+++ b/hobo/settings.py
@@ -198,6 +198,23 @@ VARIABLE_SETTINGS_DEFAULTS = {}
 
 MELLON_USERNAME_TEMPLATE = '{attributes[name_id_content]}'
 
+# MATOMO_SERVER: allow automatic configuration on a matomo server.
+# This variable excepts:
+# - the URL of the matomo server to connect
+# - an authentication token for a matomo admin user
+# - an email template for new emails associated with new matomo users
+# The token is available on the matomo GUI into the personal parameters
+# of the user.
+#
+# Example:
+# MATOMO_SERVER = {
+#     'URL': 'https://matomo.domain.org',
+#     'TOKEN_AUTH': '0123456789abcdef0123456789abcdef',
+#     'EMAIL_TEMPLATE': 'noreply+%s@domain.org'
+# }
+
+MATOMO_SERVER = {}
+
 local_settings_file = os.environ.get('HOBO_SETTINGS_FILE',
         os.path.join(os.path.dirname(__file__), 'local_settings.py'))
 if os.path.exists(local_settings_file):
diff --git a/requirements.txt b/requirements.txt
index aa381f3..0c273f8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,5 @@ django>=1.8,<1.9
 -e git+http://repos.entrouvert.org/gadjo.git/#egg=gadjo
 celery<4
 django-mellon
+lxml
 prometheus_client
diff --git a/tests/test_matomo_utils.py b/tests/test_matomo_utils.py
new file mode 100644
index 0000000..f028222
--- /dev/null
+++ b/tests/test_matomo_utils.py
@@ -0,0 +1,662 @@
+# -*- coding: utf-8 -*-
+
+import mock
+import pytest
+from requests import Response
+
+from django.test import override_settings
+
+from hobo.environment.models import Variable, Wcs, Combo, Fargo
+from hobo.matomo.utils import MatomoError, MatomoException, MatomoWS, \
+       get_variable, get_variable_value, get_tracking_js, put_tracking_js, \
+       get_tenant_name_and_public_urls, \
+       is_site_already_added, add_user_if_not_already_there, \
+       get_matomo_javascript_tag, compute_cnil_acknowledgment_level, \
+       auto_configure_matomo
+
+pytestmark = pytest.mark.django_db
+
+CONFIG = {'URL': 'https://matomo.test',
+          'TOKEN_AUTH': '1234',
+          'EMAIL_TEMPLATE': 'noreply+%s@entrouvert.test'}
+
+GET_SITE_42_ALREADY_THERE = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <row>
+        <idsite>42</idsite>
+        <moretags>...</moretags>
+    </row>
+</result>
+"""
+
+GET_SITE_34_ALREADY_THERE = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <row>
+        <idsite>34</idsite>
+        <moretags>...</moretags>
+    </row>
+</result>
+"""
+
+GET_SITE_NO_ID_PROVIDED = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+        <error message="Please specify a value for 'idSite'." />
+</result>
+"""
+
+GET_SITE_BAD_RESPONSE = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <row>
+        <not_idsite>there is no idsite tag</not_idsite>
+        <moretags>...</moretags>
+    </row>
+</result>
+"""
+
+GET_SITE_NOT_ALREADY_THERE = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <error message="An unexpected website was found in the request: website id was set to '42' ." />
+</result>
+"""
+
+ADD_SITE_SUCCESS = """<?xml version="1.0" encoding="utf-8" ?>
+<result>42</result>
+"""
+
+ADD_SITE_ERROR = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+        <error message="Please specify a value for 'siteName'." />
+</result>
+"""
+
+ADD_SITE_BAD_RESPONSE = """<?xml version="1.0" encoding="utf-8" ?>
+<not_result>no result tag</not_result>
+"""
+
+ADD_USER_SUCCESS = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <success message="ok" />
+</result>
+"""
+
+USER_ALREADY_THERE = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <error message="Username 'hobo_dev_publik_love' already exists." />
+</result>"""
+
+MAIL_ALREADY_THERE = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <error message="User with email 'nestor@testor.org' already exists." />
+</result>"""
+
+BAD_CREDENTIAL = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <error message="You can\'t access this resource as it requires a \'superuser\' access." />
+</result>"""
+
+ADD_USER_BAD_RESPONSE_1 = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <success message="KO" />
+</result>
+"""
+
+ADD_USER_BAD_RESPONSE_2 = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <success>no message attribute</success>
+    <not_success>no success tag</not_success>
+</result>
+"""
+
+JAVASCRIPT_TAG = """<?xml version="1.0" encoding="utf-8" ?>
+<result>&lt;!-- Matomo --&gt;
+&lt;script type=&quot;text/javascript&quot;&gt;
+  var _paq = window._paq || [];
+  /* tracker methods like &quot;setCustomDimension&quot; should be called before &quot;trackPageView&quot; */
+  _paq.push(['trackPageView']);
+  _paq.push(['enableLinkTracking']);
+  (function() {
+    var u=&quot;//matomo-test.entrouvert.org/&quot;;
+    _paq.push(['setTrackerUrl', u+'piwik.php']);
+    _paq.push(['setSiteId', '7']);
+    var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
+    g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s);
+  })();
+&lt;/script&gt;
+&lt;!-- End Matomo Code --&gt;
+</result>
+"""
+
+ENHANCED_JAVASCRIPT_TAG = """// Matomo
+  var _paq = window._paq || [];
+
+  // disallow cookie's time extension
+  _paq.push([function() {
+    var self = this;
+    function getOriginalVisitorCookieTimeout() {
+      var now = new Date(),
+      nowTs = Math.round(now.getTime() / 1000),
+      visitorInfo = self.getVisitorInfo();
+      var createTs = parseInt(visitorInfo[2]);
+      var cookieTimeout = 33696000; // 13 mois en secondes
+      var originalTimeout = createTs + cookieTimeout - nowTs;
+      return originalTimeout;
+    }
+    this.setVisitorCookieTimeout( getOriginalVisitorCookieTimeout() );
+  }]);
+
+  /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
+  _paq.push(['trackPageView']);
+  _paq.push(['enableLinkTracking']);
+  (function() {
+    var u="//matomo-test.entrouvert.org/";
+    _paq.push(['setTrackerUrl', u+'piwik.php']);
+    _paq.push(['setSiteId', '7']);
+    var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
+    g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s);
+  })();
+// End Matomo Code
+"""
+
+BAS_CREDENTIALS = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+        <error message="You can't access this resource as it requires 'view' access for the website id = 42." />
+</result>
+"""
+
+JAVASCRIPT_TAG_BAD_RESPONSE = """<?xml version="1.0" encoding="utf-8" ?>
+<no_result_tag/>
+"""
+
+def test_get_variable():
+    """hobo variables from"""
+
+    # create the variable with '' value if not there
+    id_site_var = get_variable('name1')
+    assert id_site_var.value == ''
+
+    # retrieve the variable if already there
+    Variable.objects.create(name='name2', value='42')
+    id_site_var = get_variable('name2')
+    assert id_site_var.value == '42'
+
+def test_get_variable_value():
+    """hobo variables from DB"""
+
+    # variable not there: return default value
+    assert get_variable_value('name1') == ''
+    assert get_variable_value('name2', default='42') == '42'
+
+    # variable already there
+    get_variable('name3', '42')
+    assert get_variable_value('name3') == '42'
+
+def test_get_tenant_name_and_public_urls():
+    Combo.objects.create(base_url='https://combo.dev.publik.love',
+                         template_name='portal-user')
+    Wcs.objects.create(base_url='https://wcs.dev.publik.love')
+    Fargo.objects.create(base_url='https://fargo.dev.publik.love')
+    tenant_name, site_urls = get_tenant_name_and_public_urls()
+    assert tenant_name == 'combo.dev.publik.love'
+    assert site_urls[2] == 'https://fargo.dev.publik.love/'
+
+def test_matomo_constructor():
+    """build the matomo webservice object"""
+    with override_settings(MATOMO_SERVER=CONFIG):
+        matomo = MatomoWS()
+        assert matomo.url_ws_base == 'https://matomo.test'
+        assert matomo.token_auth == '1234'
+
+    with override_settings(MATOMO_SERVER={}):
+        try:
+            matomo = MatomoWS()
+        except MatomoError as exc:
+            assert str(exc) == "no settings for matomo: 'URL'"
+        else:
+            assert False
+
+def test_parse_response():
+    """parser used by all matomo webservice calls"""
+    with override_settings(MATOMO_SERVER=CONFIG):
+        matomo = MatomoWS()
+
+        # no error (expected format)
+        content = """<?xml version="1.0" encoding="utf-8" ?><ok/>"""
+        tree = matomo.parse_response(content)
+        assert tree.tag == 'ok'
+
+        # error (not XML format)
+        content = """this is not XML"""
+        try:
+            tree = matomo.parse_response(content)
+        except MatomoException as exc:
+            assert str(exc).find("XMLSyntaxError: Start tag expected") != -1
+        else:
+            assert False
+
+def test_get_error_message():
+    """error handler used by all matomo webservice calls"""
+    with override_settings(MATOMO_SERVER=CONFIG):
+        matomo = MatomoWS()
+
+        # no error (expected format)
+        content = """<?xml version="1.0" encoding="utf-8" ?><ok/>"""
+        tree = matomo.parse_response(content)
+        matomo.raise_on_error(tree)
+        assert tree.tag == 'ok'
+
+        # error (expected format)
+        content = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <error message="here is the error message" />
+</result>
+"""
+        tree = matomo.parse_response(content)
+        try:
+            matomo.raise_on_error(tree)
+        except MatomoError as exc:
+            assert str(exc) == 'here is the error message'
+        else:
+            assert False
+
+        # error (unexpected format)
+        content = """<?xml version="1.0" encoding="utf-8" ?>
+<result>
+    <error>no 'message' attribute here</error>
+</result>
+"""
+        tree = matomo.parse_response(content)
+        try:
+            matomo.raise_on_error(tree)
+        except MatomoException as exc:
+            assert str(exc) == 'internal error'
+        else:
+            assert False
+
+@mock.patch('requests.post')
+def test_get_site_from_id(mocked_post):
+    """webservice to test if the site is already registered"""
+    with override_settings(MATOMO_SERVER=CONFIG):
+        matomo = MatomoWS()
+
+        # site already here
+        content = GET_SITE_42_ALREADY_THERE
+        mocked_post.return_value.content = content
+        assert matomo.get_site_from_id('42') == '42'
+
+        # site not already here
+        content = GET_SITE_NOT_ALREADY_THERE
+        mocked_post.return_value.content = content
+        try:
+            matomo.get_site_from_id(42)
+        except MatomoError as exc:
+            assert str(exc).find('An unexpected website was found in ') != -1
+        else:
+            assert False
+
+        # error on empty id
+        content = GET_SITE_NO_ID_PROVIDED
+        mocked_post.return_value.content = content
+        try:
+            matomo.get_site_from_id(42)
+        except MatomoError as exc:
+            assert str(exc) == "Please specify a value for 'idSite'."
+        else:
+            assert False
+
+        # bad response (error on success response)
+        content = GET_SITE_BAD_RESPONSE
+        mocked_post.return_value.content = content
+        try:
+            matomo.get_site_from_id(42)
+        except MatomoException as exc:
+            assert str(exc) == 'get_site_from_id fails'
+        else:
+            assert False
+
+@mock.patch('requests.post')
+def test_is_site_already_added(mocked_post):
+    """function to test if the site is already regisered"""
+    with override_settings(MATOMO_SERVER=CONFIG):
+        matomo = MatomoWS()
+
+        # site already here
+        content = GET_SITE_42_ALREADY_THERE
+        mocked_post.return_value.content = content
+        assert is_site_already_added(matomo, '42') is True
+
+        # site not already here
+        content = GET_SITE_NOT_ALREADY_THERE
+        mocked_post.return_value.content = content
+        assert is_site_already_added(matomo, '42') is False
+
+        # empty id provided
+        content = GET_SITE_NO_ID_PROVIDED
+        mocked_post.return_value.content = content
+        assert is_site_already_added(matomo, '42') is False
+
+        # strange case
+        content = GET_SITE_34_ALREADY_THERE
+        mocked_post.return_value.content = content
+        assert is_site_already_added(matomo, '42') is False
+
+        # error
+        content = GET_SITE_BAD_RESPONSE
+        mocked_post.return_value.content = content
+        try:
+            is_site_already_added(matomo, '42')
+        except MatomoException as exc:
+            assert str(exc) == 'get_site_from_id fails'
+        else:
+            assert False
+
+@mock.patch('requests.post')
+def test_add_site(mocked_post):
+    """webservice to add a new site"""
+    urls = ['https://combo.dev.publik.love',
+            'https://wcs.dev.publik.love']
+    with override_settings(MATOMO_SERVER=CONFIG):
+        matomo = MatomoWS()
+
+        # success
+        content = ADD_SITE_SUCCESS
+        mocked_post.return_value.content = content
+        site_id = matomo.add_site("hobo_dev_publik_love", urls)
+        assert site_id == '42'
+
+        # error
+        content = ADD_SITE_ERROR
+        mocked_post.return_value.content = content
+        try:
+            site_id = matomo.add_site("hobo_dev_publik_love", urls)
+        except MatomoError as exc:
+            assert str(exc) == "Please specify a value for 'siteName'."
+        else:
+            assert False
+
+        # strange message
+        content = ADD_SITE_BAD_RESPONSE
+        mocked_post.return_value.content = content
+        try:
+            site_id = matomo.add_site("hobo_dev_publik_love", urls)
+        except MatomoException as exc:
+            assert str(exc) == 'add_site fails'
+        else:
+            assert False
+
+@mock.patch('requests.post')
+def test_add_user(mocked_post):
+    """webservice to add new user"""
+    with override_settings(MATOMO_SERVER=CONFIG):
+        matomo = MatomoWS()
+
+        # success
+        content = ADD_USER_SUCCESS
+        mocked_post.return_value.content = content
+        matomo.add_user('nestor', 'xxx', '42')
+        assert True
+
+        # error (user already here)
+        content = USER_ALREADY_THERE
+        mocked_post.return_value.content = content
+        try:
+            matomo.add_user('hobo_dev_publik_love', 'xxx', '42')
+        except MatomoError as exc:
+            assert str(exc).find("Username 'hobo_dev_publik_love' already") != -1
+        else:
+            assert False
+
+        # error (mail already registered)
+        content = MAIL_ALREADY_THERE
+        mocked_post.return_value.content = content
+        try:
+            matomo.add_user('nestor', 'xxx', '42')
+        except MatomoError as exc:
+            assert str(exc).find("User with email 'nestor@testor.org'") != -1
+        else:
+            assert False
+
+        # error (bad credentials)
+        content = BAD_CREDENTIAL
+        mocked_post.return_value.content = content
+        try:
+            matomo.add_user('nestor', 'xxx', '42')
+        except MatomoError as exc:
+            assert str(exc).find("You can\'t access this resource") != -1
+        else:
+            assert False
+
+        # bad success message (wrong attribute value)
+        content = ADD_USER_BAD_RESPONSE_1
+        mocked_post.return_value.content = content
+        try:
+            matomo.add_user('nestor', 'xxx', '42')
+        except MatomoException as exc:
+            assert str(exc) == 'add_user fails'
+        else:
+            assert False
+
+       # bad success message (no message attribute)
+        content = ADD_USER_BAD_RESPONSE_2
+        mocked_post.return_value.content = content
+        try:
+            matomo.add_user('nestor', 'xxx', '42')
+        except MatomoException as exc:
+            assert str(exc) == 'add_user fails'
+        else:
+            assert False
+
+@mock.patch('requests.post')
+def test_add_user_if_not_already_there(mocked_post):
+    """function to assert we have a user"""
+    with override_settings(MATOMO_SERVER=CONFIG):
+        matomo = MatomoWS()
+
+        # success (add a new user)
+        content = ADD_USER_SUCCESS
+        mocked_post.return_value.content = content
+        add_user_if_not_already_there(matomo, 'hobo_dev_publik_love', 'xxx', '42')
+        assert True
+
+        # success (user already here)
+        content = USER_ALREADY_THERE
+        mocked_post.return_value.content = content
+        add_user_if_not_already_there(matomo, 'hobo_dev_publik_love', 'xxx', '42')
+        assert True
+
+        # error (bad credentials)
+        content = BAD_CREDENTIAL
+        mocked_post.return_value.content = content
+        try:
+            add_user_if_not_already_there(matomo, 'tenant_name', 'xxx', '42')
+        except MatomoError:
+            assert True
+        else:
+            assert False
+
+@mock.patch('requests.post')
+def test_get_javascript_tag(mocked_post):
+    """webservice to get matomo JS tag"""
+    with override_settings(MATOMO_SERVER=CONFIG):
+        matomo = MatomoWS()
+
+        # success
+        content = JAVASCRIPT_TAG
+        mocked_post.return_value.content = content
+        javascript_tag = matomo.get_javascript_tag('42')
+        assert javascript_tag.find('(function() {') != -1
+
+        # error (bad credentials)
+        content = BAD_CREDENTIAL
+        mocked_post.return_value.content = content
+        try:
+            javascript_tag = matomo.get_javascript_tag('42')
+        except MatomoError as exc:
+            assert str(exc).find("You can't access this resource ") != -1
+        else:
+            assert False
+
+       # bad response (no result tag)
+        content = JAVASCRIPT_TAG_BAD_RESPONSE
+        mocked_post.return_value.content = content
+        try:
+            javascript_tag = matomo.get_javascript_tag('42')
+        except MatomoException as exc:
+            assert str(exc) == 'get_javascript_tag fails'
+        else:
+            assert False
+
+def test_compute_cnil_acknowledgment_level():
+    """function use to inspect javascript content"""
+    warning_content = JAVASCRIPT_TAG
+
+    # can't find cookie's life time extension prevention
+    assert compute_cnil_acknowledgment_level(warning_content) == 'warning'
+
+    # ok
+    success_content = warning_content + '\n...getOriginalVisitorCookieTimeout...'
+    assert compute_cnil_acknowledgment_level(success_content) == 'success'
+
+    # google reference found into javascript
+    error_content = success_content + '\n...google...'
+    assert compute_cnil_acknowledgment_level(error_content) == 'error'
+
+def test_get_tracking_js():
+    """read previous tracking JS from hobo variables"""
+    var1 = get_variable('cnil_compliant_visits_tracking_js', 'content1')
+    assert get_tracking_js() == 'content1'
+
+    var1.delete()
+    var1 = get_variable('cnil_compliant_visits_tracking_js', '')
+    var2 = get_variable('visits_tracking_js', 'content2')
+    assert get_tracking_js() == 'content2'
+
+    var1.delete()
+    var2.delete()
+    get_variable('cnil_compliant_visits_tracking_js', 'content1')
+    get_variable('visits_tracking_js', 'content2')
+    assert get_tracking_js() == "content1content2"
+
+def test_put_tracking_js():
+    """write tracking js into hobo variables
+    a banner will be displayed depending on the variable used
+    """
+    put_tracking_js('no gafa => no banner')
+    value1 = get_variable_value('cnil_compliant_visits_tracking_js')
+    value2 = get_variable_value('visits_tracking_js')
+    assert value1 == 'no gafa => no banner'
+    assert value2 == ''
+
+    put_tracking_js('google => banner')
+    value1 = get_variable_value('cnil_compliant_visits_tracking_js')
+    value2 = get_variable_value('visits_tracking_js')
+    assert value1 == ''
+    assert value2 == 'google => banner'
+
+@mock.patch('requests.post')
+def test_get_matomo_javascript_tag(mocked_post):
+    """function to get matomo JS tag"""
+    with override_settings(MATOMO_SERVER=CONFIG):
+        matomo = MatomoWS()
+
+        # success
+        content = JAVASCRIPT_TAG
+        mocked_post.return_value.content = content
+        javascript_tag = get_matomo_javascript_tag(matomo, '42')
+        assert javascript_tag.find('(function() {') != -1
+        assert javascript_tag.find('&lt;script') == -1
+        assert javascript_tag.find('script&gt;') == -1
+        assert javascript_tag == ENHANCED_JAVASCRIPT_TAG
+        assert compute_cnil_acknowledgment_level(javascript_tag) == 'success'
+
+def auto_conf_mocked_post(url, **kwargs):
+    contents = [GET_SITE_NO_ID_PROVIDED,
+                ADD_SITE_SUCCESS,
+                ADD_USER_SUCCESS,
+                JAVASCRIPT_TAG]
+    response = Response()
+    response._content = contents[auto_conf_mocked_post.cpt]
+    response.status_code = 200
+
+    auto_conf_mocked_post.cpt += 1
+    return response
+
+@mock.patch('requests.post', side_effect=auto_conf_mocked_post)
+def test_auto_configure_matomo(mocked_post):
+    get_variable('matomo_id_site', '42')
+    get_variable('matomo_password', '567')
+    get_variable('tracking_js_var', 'js_code')
+    get_variable('matomo_error', '')
+
+    Combo.objects.create(base_url='https://combo.dev.publik.love',
+                         template_name='portal-user')
+    Wcs.objects.create(base_url='https://wcs.dev.publik.love')
+    Fargo.objects.create(base_url='https://fargo.dev.publik.love')
+
+    auto_conf_mocked_post.cpt = 0
+    with override_settings(MATOMO_SERVER=CONFIG):
+        assert auto_configure_matomo() is True
+
+@mock.patch('requests.post', side_effect=auto_conf_mocked_post)
+def test_auto_configure_matomo_no_url(mocked_post):
+    get_variable('matomo_id_site', '42')
+    get_variable('matomo_password', '567')
+    get_variable('tracking_js_var', 'js_code')
+    get_variable('matomo_error', '')
+
+    # no Wc url so as to raise
+    Wcs.objects.create(base_url='https://wcs.dev.publik.love')
+    Fargo.objects.create(base_url='https://fargo.dev.publik.love')
+
+    auto_conf_mocked_post.cpt = 0
+    with override_settings(MATOMO_SERVER=CONFIG):
+        assert auto_configure_matomo() is False
+    message = Variable.objects.get(name='matomo_error').value
+    assert message == "no portal-user url available for a matomo's id"
+
+@mock.patch('requests.post', side_effect=auto_conf_mocked_post)
+def test_auto_configure_matomo_no_password(mocked_post):
+    get_variable('matomo_id_site', '42')
+
+    # force to generate new password
+    get_variable('matomo_password', '')
+    get_variable('tracking_js_var', 'js_code')
+    get_variable('matomo_error', '')
+
+    Combo.objects.create(base_url='https://combo.dev.publik.love',
+                         template_name='portal-user')
+    Wcs.objects.create(base_url='https://wcs.dev.publik.love')
+    Fargo.objects.create(base_url='https://fargo.dev.publik.love')
+
+    auto_conf_mocked_post.cpt = 0
+    with override_settings(MATOMO_SERVER=CONFIG):
+        assert auto_configure_matomo() is True
+
+def auto_conf_mocked_post_error(url, **kwargs):
+    contents = [GET_SITE_NO_ID_PROVIDED,
+                ADD_SITE_SUCCESS,
+                ADD_USER_SUCCESS,
+                JAVASCRIPT_TAG_BAD_RESPONSE]
+    response = Response()
+    response._content = contents[auto_conf_mocked_post.cpt]
+    response.status_code = 200
+
+    auto_conf_mocked_post.cpt += 1
+    return response
+
+@mock.patch('requests.post', side_effect=auto_conf_mocked_post_error)
+def test_auto_configure_matomo_error(mocked_post):
+    get_variable('matomo_id_site', '42')
+    get_variable('matomo_password', '567')
+    get_variable('tracking_js_var', 'js_code')
+    get_variable('matomo_error', '')
+
+    Combo.objects.create(base_url='https://combo.dev.publik.love',
+                         template_name='portal-user')
+    Wcs.objects.create(base_url='https://wcs.dev.publik.love')
+    Fargo.objects.create(base_url='https://fargo.dev.publik.love')
+
+    auto_conf_mocked_post.cpt = 0
+    with override_settings(MATOMO_SERVER=CONFIG):
+        assert auto_configure_matomo() is False
+    message = Variable.objects.get(name='matomo_error').value
+    assert message == 'get_javascript_tag fails'
diff --git a/tox.ini b/tox.ini
index a51472d..dd07003 100644
--- a/tox.ini
+++ b/tox.ini
@@ -52,6 +52,7 @@ deps:
 	multitenant: systemd-python
 	http://git.entrouvert.org/debian/django-tenant-schemas.git/snapshot/django-tenant-schemas-master.tar.gz
 	httmock
+	lxml
 	requests
 commands =
 	./getlasso.sh
-- 
2.20.1

