From 0dfe8d6b3acd76d78fd6a953b568565a2703514a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Mon, 31 Jan 2022 19:34:52 +0100
Subject: [PATCH 1/8] general: reduce logging infrastructure, do not expose
 anything in UI (#61292)

---
 tests/admin_pages/test_settings.py |  22 ---
 tests/form_pages/test_formdata.py  |   6 +-
 tests/test_datasource_chrono.py    |  10 --
 tests/test_fc_auth.py              |   1 -
 tests/test_saml_auth.py            |   2 -
 wcs/admin/settings.py              |   8 --
 wcs/qommon/admin/logger.py         | 224 -----------------------------
 wcs/qommon/admin/settings.py       |   3 +-
 wcs/qommon/logger.py               |  49 -------
 wcs/qommon/publisher.py            |   2 +-
 10 files changed, 3 insertions(+), 324 deletions(-)
 delete mode 100644 wcs/qommon/admin/logger.py

diff --git a/tests/admin_pages/test_settings.py b/tests/admin_pages/test_settings.py
index cb965f209..0cb3675f3 100644
--- a/tests/admin_pages/test_settings.py
+++ b/tests/admin_pages/test_settings.py
@@ -1,5 +1,4 @@
 import io
-import logging
 import os
 import urllib.parse
 import zipfile
@@ -726,27 +725,6 @@ def test_settings_filetypes_update(pub):
     }
 
 
-def test_settings_logs(pub):
-    # reset logging state
-    logging.shutdown()
-    if os.path.exists(os.path.join(pub.app_dir, 'wcs.log')):
-        os.unlink(os.path.join(pub.app_dir, 'wcs.log'))
-
-    create_superuser(pub)
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/settings/')
-    assert 'Logs' in resp.text
-    resp = resp.click('Logs')
-    assert '<td class="message">login</td>' in resp.text
-
-    resp = app.get('/backoffice/settings/debug_options')
-    assert resp.form['logger'].checked is True
-    resp.form['logger'].checked = False
-    resp = resp.form.submit()
-    resp = resp.follow()
-    assert 'Logs' not in resp.text
-
-
 def test_settings_geolocation(pub):
     create_superuser(pub)
     app = login(get_app(pub))
diff --git a/tests/form_pages/test_formdata.py b/tests/form_pages/test_formdata.py
index e553b0ae1..2b66997c8 100644
--- a/tests/form_pages/test_formdata.py
+++ b/tests/form_pages/test_formdata.py
@@ -770,7 +770,7 @@ def test_formdata_generated_document_odt_to_pdf_download_push_to_portfolio(
     user.name = 'Foo Baré'
     user.store()
 
-    pub.cfg['debug'] = {'logger': True}
+    pub.cfg['debug'] = {}
     pub.write_cfg()
     wf = Workflow(name='status')
     st1 = wf.add_status('Status1', 'st1')
@@ -814,10 +814,6 @@ def test_formdata_generated_document_odt_to_pdf_download_push_to_portfolio(
         http_post_request.return_value = None, 200, 'null', None
         resp = resp.form.submit('button_export_to')
         assert http_post_request.call_count == 1
-        if locale.getpreferredencoding() == 'UTF-8':
-            assert caplog.records[-1].message == "file 'template.pdf' pushed to portfolio of 'Foo Baré'"
-        else:  # Python < 3.7
-            assert caplog.records[-1].message == "file 'template.pdf' pushed to portfolio of 'Foo Bar\xe9'"
 
     resp = resp.follow()  # $form/$id/create_doc
     resp = resp.follow()  # $form/$id/create_doc/
diff --git a/tests/test_datasource_chrono.py b/tests/test_datasource_chrono.py
index b29551b9b..f6203d35e 100644
--- a/tests/test_datasource_chrono.py
+++ b/tests/test_datasource_chrono.py
@@ -1,6 +1,5 @@
 import io
 import json
-import os
 from unittest import mock
 
 import pytest
@@ -24,15 +23,6 @@ def pub(request):
     req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
     pub.set_app_dir(req)
     pub._set_request(req)
-
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        fd.write(
-            '''
-[debug]
-logger=true
-'''
-        )
-
     pub.load_site_options()
 
     return pub
diff --git a/tests/test_fc_auth.py b/tests/test_fc_auth.py
index 24b8c9667..645d32548 100644
--- a/tests/test_fc_auth.py
+++ b/tests/test_fc_auth.py
@@ -66,7 +66,6 @@ def setup_user_profile(pub):
     # setup an hobo profile
     CmdCheckHobos().update_profile(PROFILE, pub)
     pub.cfg['users']['field_name'] = ['_prenoms', '_nom']
-    pub.cfg['debug'] = {'logger': True}
     pub.user_class.wipe()
     pub.write_cfg()
 
diff --git a/tests/test_saml_auth.py b/tests/test_saml_auth.py
index b33e027ab..713eb0776 100644
--- a/tests/test_saml_auth.py
+++ b/tests/test_saml_auth.py
@@ -267,8 +267,6 @@ def test_assertion_consumer_existing_federation(pub, caplog):
 
     CmdCheckHobos().update_profile(PROFILE, pub)
 
-    pub.cfg['debug'] = {'logger': True}
-    pub.write_cfg()
     pub.set_config()
 
     pub.role_class.wipe()
diff --git a/wcs/admin/settings.py b/wcs/admin/settings.py
index 443f91406..04fe2d48c 100644
--- a/wcs/admin/settings.py
+++ b/wcs/admin/settings.py
@@ -41,7 +41,6 @@ from wcs.formdef import FormDef
 from wcs.qommon import _, errors, get_cfg, ident, misc, template
 from wcs.qommon.admin.cfg import cfg_submit
 from wcs.qommon.admin.emails import EmailsDirectory
-from wcs.qommon.admin.logger import LoggerDirectory
 from wcs.qommon.admin.menu import error_page
 from wcs.qommon.admin.settings import SettingsDirectory as QommonSettingsDirectory
 from wcs.qommon.admin.texts import TextsDirectory
@@ -518,7 +517,6 @@ class SettingsDirectory(QommonSettingsDirectory):
         ('user-templates', 'user_templates'),
         ('data-sources', 'data_sources'),
         'wscalls',
-        'logs',
         ('api-access', 'api_access'),
     ]
 
@@ -530,7 +528,6 @@ class SettingsDirectory(QommonSettingsDirectory):
     filetypes = FileTypesDirectory()
     data_sources = NamedDataSourcesDirectory()
     wscalls = NamedWsCallsDirectory()
-    logs = LoggerDirectory()
     api_access = ApiAccessDirectory()
 
     def _q_index(self):
@@ -619,11 +616,6 @@ class SettingsDirectory(QommonSettingsDirectory):
                 _('Debug Options'),
                 _('Configure options useful for debugging'),
             )
-            if get_cfg('debug', {}).get('logger', True):
-                r += htmltext('<dt><a href="logs/">%s</a></dt> <dd>%s</dd>') % (
-                    _('Logs'),
-                    _('Access application log files'),
-                )
             r += htmltext('</dl>')
             r += htmltext('</div>')
 
diff --git a/wcs/qommon/admin/logger.py b/wcs/qommon/admin/logger.py
deleted file mode 100644
index 510b4c073..000000000
--- a/wcs/qommon/admin/logger.py
+++ /dev/null
@@ -1,224 +0,0 @@
-# w.c.s. - web application for online forms
-# Copyright (C) 2005-2010  Entr'ouvert
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
-
-import os
-import random
-
-from quixote import get_publisher, get_request, get_response
-from quixote.directory import Directory
-from quixote.html import TemplateIO, htmltext
-
-from .. import _, errors, logger
-from ..admin.menu import error_page
-from ..backoffice.menu import html_top
-
-
-class ByUserDirectory(Directory):
-    def _q_lookup(self, component):
-        return ByUserPages(component)
-
-
-class LoggerDirectory(Directory):
-    _q_exports = ['', 'download', 'by_user']
-
-    by_user = ByUserDirectory()
-
-    def _q_index(self):
-        get_response().breadcrumb.append(('logs/', _('Logs')))
-        html_top('logs', title=_('Logs'))
-        r = TemplateIO(html=False)
-        request = get_request()
-        logfile = str(request.get_field('logfile', get_publisher().APP_NAME + '.log'))
-        if not logfile.startswith(str(get_publisher().APP_NAME + '.log')) or '/' in str(logfile):
-            return error_page('logs/', _('Bad log file: %s') % logfile)
-        logfilename = str(os.path.join(get_publisher().app_dir, logfile))
-
-        if not os.path.exists(logfilename):
-            r += _('Nothing to show')
-        else:
-            get_response().filter['sidebar'] = self.get_sidebar(logfile)
-
-            user_color_keys = {}
-            last_date = None
-            r += htmltext('<table id="logs">\n')
-            r += htmltext('<thead> <tr>')
-            r += htmltext(' <th>%s</th>') % _('Time')
-            r += htmltext(' <th>%s</th>') % _('User')
-            r += htmltext(' <th>%s</th>') % _('Message')
-            r += htmltext('<tr></thead>\n')
-            r += htmltext('<tbody>\n')
-            userlabels = {}
-            with open(logfilename) as fd:
-                for d in logger.parse_logstream(fd):
-                    if not d:
-                        continue
-
-                    if d.get('user_id'):
-                        user_color_key = d['user_id']
-                        if user_color_key == 'anonymous':
-                            user_color_key += d['ip']
-                        if user_color_key not in user_color_keys:
-                            user_color_keys[user_color_key] = ''.join(
-                                ['%x' % random.randint(0xC, 0xF) for x in range(3)]
-                            )
-                        r += htmltext('<tr class="level-%s" style="background: #%s;">') % (
-                            d['level'].lower(),
-                            user_color_keys[user_color_key],
-                        )
-                    else:
-                        r += htmltext('<tr class="level-%s">') % d['level'].lower()
-
-                    if last_date != d['date']:
-                        r += htmltext(' <td class="time">%s&nbsp;%s</td>') % (d['date'], d['hour'][:-4])
-                        last_date = d['date']
-                    else:
-                        r += htmltext(' <td class="time">%s</td>') % (d['hour'][:-4])
-
-                    user_id = d.get('user_id')
-                    if not user_id:
-                        userlabel = None
-                    elif user_id == 'anonymous':
-                        userlabel = _('Anonymous')
-                        ip = d['ip']
-                        r += htmltext(' <td class="userlabel"><span title="%s">%s</span></td>') % (
-                            ip,
-                            userlabel,
-                        )
-                    elif user_id == 'unlogged':
-                        userlabel = _('Unlogged')
-                        ip = d['ip']
-                        r += htmltext(' <td class="userlabel"><span title="%s">%s</span></td>') % (
-                            ip,
-                            userlabel,
-                        )
-                    elif user_id == 'bot':
-                        userlabel = _('Bot')
-                        r += htmltext(' <td class="userlabel">%s</td>') % userlabel
-                    else:
-                        userlabel = userlabels.get(user_id)
-                        if not userlabel:
-                            try:
-                                user = get_publisher().user_class.get(user_id)
-                                userlabel = htmltext(user.display_name.replace(' ', '&nbsp;'))
-                            except KeyError:
-                                userlabel = _('Unknown')
-                            userlabels[user_id] = userlabel
-                        r += htmltext(' <td class="userlabel">%s</td>') % userlabel
-                    if userlabel:
-                        r += htmltext(' <td class="message">%s</td>') % d['message']
-                    else:
-                        r += htmltext('<td class="message" colspan="2">%s</td>') % d['message']
-                    r += htmltext('</tr>\n')
-            r += htmltext('</tbody>\n')
-            r += htmltext('</table>\n')
-        return r.getvalue()
-
-    def get_sidebar(self, logfile):
-        r = TemplateIO(html=True)
-        r += htmltext('<ul>')
-        if logfile:
-            r += htmltext('<li><a href="download?logfile=%s">%s</a></li>') % (
-                logfile,
-                _('Download Raw Log File'),
-            )
-        else:
-            r += htmltext('<li><a href="download">%s</a></li>') % _('Download Raw Log File')
-        r += htmltext('</ul>')
-
-        logfiles = [
-            x for x in os.listdir(get_publisher().app_dir) if x.startswith(get_publisher().APP_NAME + '.log')
-        ]
-        if len(logfiles) > 1:
-            options = []
-            for lfile in logfiles:
-                with open(os.path.join(get_publisher().app_dir, lfile)) as fd:
-                    firstline = fd.readline()
-                d = logger.readline(firstline)
-                if not d:
-                    continue
-                if logfile == lfile:
-                    selected = 'selected="selected" '
-                else:
-                    selected = ''
-                options.append(
-                    {'selected': selected, 'lfile': lfile, 'date': '%s %s' % (d['date'], d['hour'])}
-                )
-
-            r += htmltext('<form id="other-log-select">')
-            r += _('Select another logfile:')
-            r += htmltext('<select name="logfile">')
-            options.sort(key=lambda x: x['date'])
-            options.reverse()
-            for option in options:
-                option['since'] = str(_('Since: %s') % option['date'])[:-4]
-                r += htmltext('<option value="%(lfile)s"%(selected)s>%(since)s</option>') % option
-            r += htmltext('</select>')
-            r += htmltext('<input type="submit" value="%s" />') % _('Submit')
-
-        return r.getvalue()
-
-    def download(self):
-        request = get_request()
-        logfile = request.get_field('logfile', get_publisher().APP_NAME + '.log')
-        if not logfile.startswith(get_publisher().APP_NAME + '.log') or '/' in logfile:
-            return error_page('logs/', _('Bad log file: %s') % logfile)
-        logfilename = os.path.join(get_publisher().app_dir, logfile)
-        response = get_response()
-        response.set_content_type('text/x-log', 'iso-8859-1')
-        response.set_header('content-disposition', 'attachment; filename=%s' % logfile)
-        with open(logfilename) as fd:
-            return fd.read()
-
-
-class ByUserPages(Directory):
-    _q_exports = ['']
-
-    def __init__(self, component):
-        try:
-            self.user = get_publisher().user_class.get(component)
-        except KeyError:
-            raise errors.TraversalError()
-
-    def _q_index(self):
-        html_top('logs', title=_('Logs'))
-        r = TemplateIO(html=True)
-        r += htmltext('<h2>%s - %s</h2>') % (_('User'), self.user.name)
-
-        last_date = None
-        r += htmltext('<table id="logs">')
-        r += htmltext('<thead> <tr>')
-        r += htmltext(' <th>%s</th>') % _('Time')
-        r += htmltext(' <th>%s</th>') % _('Message')
-        r += htmltext('<tr></thead>')
-        r += htmltext('<tbody>')
-        logfilename = str(os.path.join(get_publisher().app_dir, get_publisher().APP_NAME + '.log'))
-        if os.path.exists(logfilename):
-            with open(logfilename) as fd:
-                for line in fd:
-                    d = logger.readline(line)
-                    if not d or d['user_id'] != str(self.user.id):
-                        continue
-                    r += htmltext('<tr>')
-                    if last_date != d['date']:
-                        r += htmltext(' <td class="time">%s&nbsp;%s</td>') % (d['date'], d['hour'][:-4])
-                        last_date = d['date']
-                    else:
-                        r += htmltext(' <td class="time">%s</td>') % (d['hour'][:-4])
-                    r += htmltext(' <td><a href="%s">%s</a></td>') % (d['url'], d['message'])
-                    r += htmltext('</tr>')
-        r += htmltext('</tbody>')
-        r += htmltext('</table>')
-        return r.getvalue()
diff --git a/wcs/qommon/admin/settings.py b/wcs/qommon/admin/settings.py
index 172b47586..f13a82d7c 100644
--- a/wcs/qommon/admin/settings.py
+++ b/wcs/qommon/admin/settings.py
@@ -73,7 +73,6 @@ class SettingsDirectory(AccessControlled, Directory):
             title=_('Email for Tracebacks'),
             value=debug_cfg.get('error_email', ''),
         )
-        form.add(CheckboxWidget, 'logger', title=_('Logger'), value=debug_cfg.get('logger', True))
         form.add(
             CheckboxWidget,
             'debug_mode',
@@ -104,6 +103,6 @@ class SettingsDirectory(AccessControlled, Directory):
             cfg_submit(
                 form,
                 'debug',
-                ('error_email', 'logger', 'debug_mode', 'mail_redirection'),
+                ('error_email', 'debug_mode', 'mail_redirection'),
             )
             return redirect('.')
diff --git a/wcs/qommon/logger.py b/wcs/qommon/logger.py
index 3085d9f0e..d00356214 100644
--- a/wcs/qommon/logger.py
+++ b/wcs/qommon/logger.py
@@ -97,52 +97,3 @@ class Formatter(logging.Formatter):
         ) or '[nosession]'
 
         return logging.Formatter.format(self, record).replace('\n', '\n ')
-
-
-def parse_logstream(stream):
-    """
-    Parse a stream of lines making a log file, each log line start with a
-    non-blank character continue until the lines does not start with a blank
-    character
-    """
-    line = next(stream)
-    while True:
-        r = readline(line)
-        try:
-            # Skip badly formatted lines
-            if r is None:
-                line = next(stream)
-                continue
-            while True:
-                line = next(stream)
-                if line.startswith(' '):
-                    # Append the line without the first blank
-                    r['message'] = r['message'] + line[1:]
-                    continue
-                break
-        except StopIteration:
-            # Ont last line return the current one
-            if r is not None:
-                r['message'] = r['message'].strip()
-                yield r
-            break
-        r['message'] = r['message'].strip()
-        yield r
-
-
-def readline(line):
-    if not line:
-        return None
-    try:
-        date, hour, level, ip, session_id, url, user_id, dash, message = line.split(' ', 8)
-    except ValueError:
-        try:
-            date, hour, level, message = line.split(' ', 3)
-        except ValueError:
-            return None  # misformatted line
-        del line
-        return locals()
-    if dash != '-':
-        return None
-    del line
-    return locals()
diff --git a/wcs/qommon/publisher.py b/wcs/qommon/publisher.py
index 37827e300..23e5efa55 100644
--- a/wcs/qommon/publisher.py
+++ b/wcs/qommon/publisher.py
@@ -694,7 +694,7 @@ class QommonPublisher(Publisher):
             cls.root_directory_class(),
             session_cookie_name=cls.APP_NAME,
             session_cookie_path='/',
-            logger=logger.ApplicationLogger(error_log=cls.ERROR_LOG),
+            logger=logger.ApplicationLogger(),
         )
         publisher.substitutions = Substitutions()
         publisher.app_dir = cls.APP_DIR
-- 
2.34.1

