From 9d9c44dd0dadd525dfa6ae7154a8c578b3922599 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Thu, 22 Oct 2020 11:57:51 +0200
Subject: [PATCH] logging: manage log_retention_days log parameters (#47426)

---
 ...23_loggingparameters_log_retention_days.py | 20 ++++++++++
 passerelle/base/models.py                     |  9 ++++-
 passerelle/base/views.py                      |  6 ++-
 tests/test_misc.py                            | 37 ++++++++++++++++++-
 4 files changed, 69 insertions(+), 3 deletions(-)
 create mode 100644 passerelle/base/migrations/0023_loggingparameters_log_retention_days.py

diff --git a/passerelle/base/migrations/0023_loggingparameters_log_retention_days.py b/passerelle/base/migrations/0023_loggingparameters_log_retention_days.py
new file mode 100644
index 00000000..f2b0d4d8
--- /dev/null
+++ b/passerelle/base/migrations/0023_loggingparameters_log_retention_days.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2020-11-03 11:16
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0022_auto_20200715_1033'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='loggingparameters',
+            name='log_retention_days',
+            field=models.PositiveIntegerField(blank=True, help_text='Number of days to keep logs', null=True, verbose_name='Log retention days'),
+        ),
+    ]
diff --git a/passerelle/base/models.py b/passerelle/base/models.py
index e447aa75..1ea5fb4c 100644
--- a/passerelle/base/models.py
+++ b/passerelle/base/models.py
@@ -444,17 +444,18 @@ class BaseResource(models.Model):
                                 '%s is unsupported' % (field, cls))
         instance.save()
         if 'log_level' in d:
             instance.set_log_level(d['log_level'])
         return instance
 
     def clean_logs(self):
         # clean logs
-        timestamp = timezone.now() - datetime.timedelta(days=settings.LOG_RETENTION_DAYS)
+        timestamp = timezone.now() - datetime.timedelta(
+            days=self.logging_parameters.log_retention_days or settings.LOG_RETENTION_DAYS)
         ResourceLog.objects.filter(
                 appname=self.get_connector_slug(),
                 slug=self.slug,
                 timestamp__lt=timestamp).delete()
 
     def check_status(self):
         # should raise an exception if status is not ok
         raise NotImplementedError
@@ -665,16 +666,22 @@ class LoggingParameters(models.Model):
         help_text=_('Maximum HTTP request size to log'),
         default=settings.LOGGED_REQUESTS_MAX_SIZE
     )
     responses_max_size = models.PositiveIntegerField(
         verbose_name=_('Responses maximum size'),
         help_text=_('Maximum HTTP reponse size to log'),
         default=settings.LOGGED_RESPONSES_MAX_SIZE
     )
+    log_retention_days = models.PositiveIntegerField(
+        verbose_name=_('Log retention days'),
+        help_text=_('Number of days to keep logs'),
+        blank=True,
+        null=True
+    )
 
     class Meta:
         unique_together = (('resource_type', 'resource_pk'))
 
 
 def parse_notification_delays(value):
     delays = [int(v.strip()) for v in value.split(',')]
     if not all(delay >= 0 for delay in delays):
diff --git a/passerelle/base/views.py b/passerelle/base/views.py
index 66896a82..a8dc03d6 100644
--- a/passerelle/base/views.py
+++ b/passerelle/base/views.py
@@ -14,16 +14,17 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import datetime
 import json
 
 from dateutil import parser as date_parser
 
+from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.core.urlresolvers import reverse
 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
 from django.db.models import Q
 from django.forms import models as model_forms
 from django.views.generic import (
     View, DetailView, ListView, CreateView, UpdateView, DeleteView, FormView)
 from django.http import Http404, HttpResponse, HttpResponseRedirect
@@ -141,44 +142,47 @@ class LoggingParametersUpdateView(FormView):
     def get_context_data(self, **kwargs):
         context = super(LoggingParametersUpdateView, self).get_context_data(**kwargs)
         context['connector'] = self.get_resource()
         return context
 
     def get_form_class(self):
         form_class = model_forms.modelform_factory(
             LoggingParameters,
-            fields=['log_level', 'trace_emails', 'requests_max_size', 'responses_max_size'])
+            fields=['log_level', 'trace_emails', 'requests_max_size', 'responses_max_size',
+                    'log_retention_days'])
         form_class.base_fields['trace_emails'].widget.attrs['rows'] = '3'
         return form_class
 
     def get_initial(self):
         d = self.initial.copy()
         d['resource_type'] = self.kwargs['resource_type']
         d['resource_pk'] = self.kwargs['resource_pk']
         parameters = self.get_resource().logging_parameters
         d['log_level'] = parameters.log_level
         d['trace_emails'] = parameters.trace_emails
         d['requests_max_size'] = parameters.requests_max_size
         d['responses_max_size'] = parameters.responses_max_size
+        d['log_retention_days'] = parameters.log_retention_days
         return d
 
     def get_resource(self):
         content_type = ContentType.objects.get_for_id(self.kwargs['resource_type'])
         return content_type.model_class().objects.get(pk=self.kwargs['resource_pk'])
 
     def get_success_url(self):
         return self.get_resource().get_absolute_url()
 
     def form_valid(self, form):
         parameters = self.get_resource().logging_parameters
         parameters.log_level = form.cleaned_data['log_level']
         parameters.trace_emails = form.cleaned_data['trace_emails']
         parameters.requests_max_size = form.cleaned_data['requests_max_size']
         parameters.responses_max_size = form.cleaned_data['responses_max_size']
+        parameters.log_retention_days = form.cleaned_data['log_retention_days']
         parameters.save()
         return super(LoggingParametersUpdateView, self).form_valid(form)
 
 
 class ManageAvailabilityView(UpdateView):
     template_name = 'passerelle/manage/manage_availability_form.html'
     form_class = AvailabilityParametersForm
 
diff --git a/tests/test_misc.py b/tests/test_misc.py
index ec14ff0d..66a4e82c 100644
--- a/tests/test_misc.py
+++ b/tests/test_misc.py
@@ -1,22 +1,26 @@
 import datetime
 import pytest
 from mock import patch
 
+from django.contrib.contenttypes.models import ContentType
 from django.core.files import File
 from django.db import connection
 from django.db.migrations.executor import MigrationExecutor
+from django.core.urlresolvers import reverse
 from django.utils import timezone
 from django.utils.six import StringIO
 
 from passerelle.base.models import ResourceLog
 from passerelle.apps.opengis.models import OpenGIS
 from passerelle.apps.clicrdv.models import ClicRdv
 
+from test_manager import login, admin_user
+
 
 def test_get_description_url_fields(db):
     connector = OpenGIS(slug='plop', wms_service_url='http://www.example.net')
     assert 'http://www.example.net' in [x[1] for x in connector.get_description_fields()]
 
     connector = OpenGIS(slug='plop', wms_service_url='http://username:secret@www.example.net')
     assert 'http://***:***@www.example.net' in [x[1] for x in connector.get_description_fields()]
 
@@ -25,28 +29,59 @@ def test_get_description_url_fields(db):
 
 
 def test_get_description_secret_fields(db):
     connector = ClicRdv(slug='plop', apikey='secret1', username='plop', password='secret2')
     assert not 'secret1' in [x[1] for x in connector.get_description_fields()]
     assert not 'secret2' in [x[1] for x in connector.get_description_fields()]
 
 
-def test_log_cleaning(db):
+def test_log_cleaning(app, db, admin_user, settings):
     ResourceLog.objects.all().delete()
     connector = OpenGIS(slug='plop', wms_service_url='http://www.example.net')
+    connector.save()
     connector.logger.error('hello1')
     connector.logger.error('hello2')
 
     assert ResourceLog.objects.all().count() == 2
 
     ResourceLog.objects.update(timestamp=timezone.now() - datetime.timedelta(days=10))
     connector.logger.error('hello3')
     assert ResourceLog.objects.all().count() == 3
 
+    settings.LOG_RETENTION_DAYS = 11
+    connector.daily()
+    assert ResourceLog.objects.all().count() == 3
+    settings.LOG_RETENTION_DAYS = 10
+    connector.daily()
+    assert ResourceLog.objects.all().count() == 1
+
+    ResourceLog.objects.all().delete()
+    connector.logger.error('hello1')
+    connector.logger.error('hello2')
+    assert ResourceLog.objects.all().count() == 2
+    ResourceLog.objects.update(timestamp=timezone.now() - datetime.timedelta(days=10))
+    connector.logger.error('hello3')
+    assert ResourceLog.objects.all().count() == 3
+
+    url = reverse('logging-parameters', kwargs={
+        'resource_type': ContentType.objects.get_for_model(connector).id,
+        'resource_pk': connector.id})
+    app = login(app)
+    resp = app.get(url)
+    assert not resp.html.find('input', {'name': 'log_retention_days'}).has_attr('value')
+    resp.form['log_retention_days'] = '11'
+    resp.form.submit()
+    connector.daily()
+    assert ResourceLog.objects.all().count() == 3
+
+    resp = app.get(url)
+    assert int(resp.html.find('input', {'name': 'log_retention_days'})['value']) == 11
+    resp.form['log_retention_days'] = '10'
+    resp.form.submit()
     connector.daily()
     assert ResourceLog.objects.all().count() == 1
 
 @pytest.fixture
 def email_handler():
     import logging
     from django.utils.log import AdminEmailHandler
 
-- 
2.28.0

