From 9d0e13e56c9a19f7a9c3dbef5df1bc5d3f279b97 Mon Sep 17 00:00:00 2001 From: Serghei Mihai Date: Tue, 3 Oct 2017 23:15:00 +0200 Subject: [PATCH] agenda: add support for remote calendar file with exceptions (#19070) Remote calendars can be specified by desk and updated hourly, or used once. --- .../0020_desk_timeperiod_exceptions_remote_url.py | 19 +++ chrono/agendas/models.py | 18 +++ chrono/manager/forms.py | 30 ++++- .../commands/sync_desks_timeperiod_exceptions.py | 33 +++++ .../chrono/manager_import_exceptions.html | 4 +- chrono/manager/views.py | 5 +- debian/chrono.cron.hourly | 4 + debian/control | 3 +- requirements.txt | 1 + setup.py | 3 +- tests/test_agendas.py | 101 +++++++++++++++ tests/test_manager.py | 143 +++++++++++++++++++++ 12 files changed, 352 insertions(+), 12 deletions(-) create mode 100644 chrono/agendas/migrations/0020_desk_timeperiod_exceptions_remote_url.py create mode 100644 chrono/manager/management/commands/sync_desks_timeperiod_exceptions.py create mode 100644 debian/chrono.cron.hourly diff --git a/chrono/agendas/migrations/0020_desk_timeperiod_exceptions_remote_url.py b/chrono/agendas/migrations/0020_desk_timeperiod_exceptions_remote_url.py new file mode 100644 index 0000000..a19524c --- /dev/null +++ b/chrono/agendas/migrations/0020_desk_timeperiod_exceptions_remote_url.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('agendas', '0019_timeperiodexception'), + ] + + operations = [ + migrations.AddField( + model_name='desk', + name='timeperiod_exceptions_remote_url', + field=models.URLField(null=True, verbose_name='URL to fetch time period exceptions from', blank=True), + ), + ] diff --git a/chrono/agendas/models.py b/chrono/agendas/models.py index 3a1c1a0..55cb26d 100644 --- a/chrono/agendas/models.py +++ b/chrono/agendas/models.py @@ -17,6 +17,7 @@ import datetime import vobject +import requests from django.contrib.auth.models import Group from django.core.exceptions import ValidationError @@ -43,6 +44,18 @@ def is_midnight(dtime): dtime = localtime(dtime) return dtime.hour == 0 and dtime.minute == 0 +def get_remote_calendar(url): + try: + response = requests.get(url) + response.raise_for_status() + except requests.exceptions.HTTPError as e: + raise ICSError(_('Failed to retrieve remote calendar (HTTP error %s).') % e.response.status_code) + except requests.exceptions.ConnectionError: + raise ICSError(_('Failed to retrieve remote calendar (connection error).')) + except requests.exceptions.Timeout: + raise ICSError(_('Failed to retrieve remote calendar (HTTP timeout).')) + return response.text + class ICSError(Exception): pass @@ -358,6 +371,8 @@ class Desk(models.Model): agenda = models.ForeignKey(Agenda) label = models.CharField(_('Label'), max_length=150) slug = models.SlugField(_('Identifier'), max_length=150) + timeperiod_exceptions_remote_url = models.URLField(_('URL to fetch time period exceptions from'), + null=True, blank=True) def __unicode__(self): return self.label @@ -417,6 +432,9 @@ class Desk(models.Model): in_two_weeks = self.get_exceptions_within_two_weeks() return self.timeperiodexception_set.count() == len(in_two_weeks) + def create_timeperiod_exceptions_from_remote_ics(self, url): + return self.create_timeperiod_exceptions_from_ics(get_remote_calendar(url)) + def create_timeperiod_exceptions_from_ics(self, data): try: parsed = vobject.readOne(data) diff --git a/chrono/manager/forms.py b/chrono/manager/forms.py index cc6e37c..0c8dfb5 100644 --- a/chrono/manager/forms.py +++ b/chrono/manager/forms.py @@ -16,13 +16,14 @@ import csv import datetime +import requests from django import forms from django.forms import ValidationError from django.utils.translation import ugettext_lazy as _ from chrono.agendas.models import (Event, MeetingType, TimePeriod, Desk, - TimePeriodException) + TimePeriodException, ICSError, get_remote_calendar) from . import widgets @@ -77,22 +78,33 @@ class TimePeriodForm(forms.ModelForm): exclude = [] -class NewDeskForm(forms.ModelForm): +class DeskForm(forms.ModelForm): class Meta: model = Desk widgets = { 'agenda': forms.HiddenInput(), } - exclude = ['slug'] + exclude = [] + def is_valid(self): + if not self.data['timeperiod_exceptions_remote_url']: + return super(DeskForm, self).is_valid() -class DeskForm(forms.ModelForm): + try: + get_remote_calendar(self.data['timeperiod_exceptions_remote_url']) + except ICSError as e: + self.add_error('timeperiod_exceptions_remote_url', e) + return False + return super(DeskForm, self).is_valid() + + +class NewDeskForm(DeskForm): class Meta: model = Desk widgets = { 'agenda': forms.HiddenInput(), } - exclude = [] + exclude = ['slug'] class TimePeriodExceptionForm(forms.ModelForm): @@ -170,5 +182,11 @@ class ExceptionsImportForm(forms.ModelForm): model = Desk fields = [] - ics_file = forms.FileField(label=_('ICS File'), + ics_file = forms.FileField(label=_('ICS File'), required=False, help_text=_('ICS file containing events which will be considered as exceptions')) + ics_url = forms.URLField(label=_('URL'), required=False) + + def clean(self): + cleaned_data = super(ExceptionsImportForm, self).clean() + if not cleaned_data['ics_file'] and not cleaned_data['ics_url']: + raise ValidationError(_('A file or an url should be filled.')) diff --git a/chrono/manager/management/commands/sync_desks_timeperiod_exceptions.py b/chrono/manager/management/commands/sync_desks_timeperiod_exceptions.py new file mode 100644 index 0000000..bd8328f --- /dev/null +++ b/chrono/manager/management/commands/sync_desks_timeperiod_exceptions.py @@ -0,0 +1,33 @@ +# chrono - agendas system +# Copyright (C) 2016-2017 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys +import logging + +from django.core.management.base import BaseCommand +from chrono.agendas.models import Desk, ICSError + + +class Command(BaseCommand): + help = 'Synchronize time period exceptions from desks remote ics' + + def handle(self, **options): + logger = logging.getLogger(__name__) + for desk in Desk.objects.filter(timeperiod_exceptions_remote_url__isnull=False): + try: + desk.create_timeperiod_exceptions_from_remote_ics(desk.timeperiod_exceptions_remote_url) + except ICSError as e: + logger.warning(u'unable to create timeperiod exceptions for "%s": %s' % (desk, e)) diff --git a/chrono/manager/templates/chrono/manager_import_exceptions.html b/chrono/manager/templates/chrono/manager_import_exceptions.html index d3eacc5..91dc9ef 100644 --- a/chrono/manager/templates/chrono/manager_import_exceptions.html +++ b/chrono/manager/templates/chrono/manager_import_exceptions.html @@ -11,12 +11,10 @@ {% endblock %} {% block content %} -
+

{% trans "You can upload a file or specify an address to remote calendar." %}

{% csrf_token %} {{ form.as_p }} -

-

{% trans 'Cancel' %} diff --git a/chrono/manager/views.py b/chrono/manager/views.py index c5d4eae..0595628 100644 --- a/chrono/manager/views.py +++ b/chrono/manager/views.py @@ -396,7 +396,10 @@ class DeskImportTimePeriodExceptionsView(ManagedAgendaSubobjectMixin, UpdateView def form_valid(self, form): try: - exceptions = form.instance.create_timeperiod_exceptions_from_ics(form.cleaned_data['ics_file']) + if form.cleaned_data['ics_file']: + exceptions = form.instance.create_timeperiod_exceptions_from_ics(form.cleaned_data['ics_file']) + else: + exceptions = form.instance.create_timeperiod_exceptions_from_remote_ics(form.cleaned_data['ics_url']) except ICSError as e: form.add_error(None, unicode(e)) return self.form_invalid(form) diff --git a/debian/chrono.cron.hourly b/debian/chrono.cron.hourly new file mode 100644 index 0000000..ec5c168 --- /dev/null +++ b/debian/chrono.cron.hourly @@ -0,0 +1,4 @@ +#!/bin/sh + +/sbin/runuser -u chrono /usr/bin/chrono-manage -- tenant_command sync_desks_timeperiod_exceptions --all-tenants + diff --git a/debian/control b/debian/control index f80b130..0ce7c45 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,8 @@ Architecture: all Depends: ${misc:Depends}, ${python:Depends}, python-django (>= 1.8), python-gadjo, - python-intervaltree + python-intervaltree, + python-requests Recommends: python-django-mellon Description: Agendas System (Python module) diff --git a/requirements.txt b/requirements.txt index 536c4b8..9a35d86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ gadjo djangorestframework>=3.1, <3.7 django-jsonfield >= 0.9.3 intervaltree +requests diff --git a/setup.py b/setup.py index 8fefafb..bfcaa56 100644 --- a/setup.py +++ b/setup.py @@ -107,7 +107,8 @@ setup( 'djangorestframework>=3.1, <3.7', 'django-jsonfield >= 0.9.3', 'intervaltree', - 'vobject' + 'vobject', + 'requests' ], zip_safe=False, cmdclass={ diff --git a/tests/test_agendas.py b/tests/test_agendas.py index 7560a66..a7121af 100644 --- a/tests/test_agendas.py +++ b/tests/test_agendas.py @@ -1,7 +1,11 @@ import pytest import datetime +import mock +import requests +import logging from django.utils.timezone import now, make_aware, localtime +from django.core.management import call_command from chrono.agendas.models import (Agenda, Event, Booking, MeetingType, Desk, TimePeriodException, ICSError) @@ -220,3 +224,100 @@ def test_timeexception_create_from_ics_with_no_events(): with pytest.raises(ICSError) as e: exceptions_count = desk.create_timeperiod_exceptions_from_ics(ICS_SAMPLE_WITH_NO_EVENTS) assert str(e.value) == "The file doesn't contain any events." + +@mock.patch('chrono.agendas.models.requests.get') +def test_timeperiodexception_creation_from_remote_ics(mocked_get): + agenda = Agenda(label=u'Test 8 agenda') + agenda.save() + desk = Desk(label='Test 8 desk', agenda=agenda) + desk.save() + mocked_response = mock.Mock() + mocked_response.text = ICS_SAMPLE + mocked_get.return_value = mocked_response + exceptions_count = desk.create_timeperiod_exceptions_from_remote_ics('http://example.com/sample.ics') + assert exceptions_count == 2 + +@mock.patch('chrono.agendas.models.requests.get') +def test_timeperiodexception_creation_from_unreachable_remote_ics(mocked_get): + agenda = Agenda(label=u'Test 9 agenda') + agenda.save() + desk = Desk(label='Test 9 desk', agenda=agenda) + desk.save() + mocked_response = mock.Mock() + mocked_response.content.return_value = ICS_SAMPLE + mocked_get.return_value = mocked_response + def mocked_requests_connection_error(*args, **kwargs): + raise requests.exceptions.ConnectionError('unreachable') + mocked_get.side_effect = mocked_requests_connection_error + with pytest.raises(ICSError) as e: + exceptions_count = desk.create_timeperiod_exceptions_from_remote_ics('http://example.com/sample.ics') + assert str(e.value) == "Failed to retrieve remote calendar (connection error)." + +@mock.patch('chrono.agendas.models.requests.get') +def test_timeperiodexception_creation_from_forbidden_remote_ics(mocked_get): + agenda = Agenda(label=u'Test 10 agenda') + agenda.save() + desk = Desk(label='Test 10 desk', agenda=agenda) + desk.save() + mocked_response = mock.Mock() + mocked_response.status_code = 403 + mocked_get.return_value = mocked_response + def mocked_requests_http_forbidden_error(*args, **kwargs): + raise requests.exceptions.HTTPError(response=mocked_response) + mocked_get.side_effect = mocked_requests_http_forbidden_error + + with pytest.raises(ICSError) as e: + exceptions_count = desk.create_timeperiod_exceptions_from_remote_ics('http://example.com/sample.ics') + assert str(e.value) == "Failed to retrieve remote calendar (HTTP error 403)." + +@mock.patch('chrono.agendas.models.requests.get') +def test_timeperiodexception_creation_from_remote_ics_with_timeout_error(mocked_get): + agenda = Agenda(label=u'Test 11 agenda') + agenda.save() + desk = Desk(label='Test 11 desk', agenda=agenda) + desk.save() + mocked_response = mock.Mock() + mocked_get.return_value = mocked_response + def mocked_requests_http_timeout_error(*args, **kwargs): + raise requests.exceptions.Timeout(response=mocked_response) + mocked_get.side_effect = mocked_requests_http_timeout_error + + with pytest.raises(ICSError) as e: + exceptions_count = desk.create_timeperiod_exceptions_from_remote_ics('http://example.com/sample.ics') + assert str(e.value) == "Failed to retrieve remote calendar (HTTP timeout)." + +@mock.patch('chrono.agendas.models.requests.get') +def test_timeperiodexception_creation_from_remote_ics_with_ssl_error(mocked_get): + agenda = Agenda(label=u'Test 11 agenda') + agenda.save() + desk = Desk(label='Test 11 desk', agenda=agenda) + desk.save() + mocked_response = mock.Mock() + mocked_get.return_value = mocked_response + def mocked_requests_http_ssl_error(*args, **kwargs): + raise requests.exceptions.SSLError(response=mocked_response) + mocked_get.side_effect = mocked_requests_http_ssl_error + + with pytest.raises(ICSError) as e: + exceptions_count = desk.create_timeperiod_exceptions_from_remote_ics('http://example.com/sample.ics') + assert str(e.value) == "Failed to retrieve remote calendar (connection error)." + +@mock.patch('chrono.agendas.models.requests.get') +def test_sync_desks_timeperiod_exceptions_from_ics(mocked_get, caplog): + agenda = Agenda(label=u'Test 11 agenda') + agenda.save() + desk = Desk(label='Test 11 desk', agenda=agenda, timeperiod_exceptions_remote_url='http:example.com/sample.ics') + desk.save() + mocked_response = mock.Mock() + mocked_response.status_code = 403 + mocked_get.return_value = mocked_response + def mocked_requests_http_forbidden_error(*args, **kwargs): + raise requests.exceptions.HTTPError(response=mocked_response) + mocked_get.side_effect = mocked_requests_http_forbidden_error + call_command('sync_desks_timeperiod_exceptions') + records = caplog.records() + assert len(records) == 1 + for record in records: + assert record.name == 'chrono.manager.management.commands.sync_desks_timeperiod_exceptions' + assert record.levelno == logging.WARNING + assert record.getMessage() == 'unable to create timeperiod exceptions for "Test 11 desk": Failed to retrieve remote calendar (HTTP error 403).' diff --git a/tests/test_manager.py b/tests/test_manager.py index a39190e..f66ebb1 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -5,6 +5,8 @@ from django.utils.timezone import make_aware, now, localtime import datetime import pytest from webtest import TestApp, Upload +import mock +import requests from chrono.wsgi import application @@ -699,6 +701,49 @@ def test_meetings_agenda_add_desk(app, admin_user): assert 'Desk A' in resp.text assert 'Desk B' in resp.text +@mock.patch('chrono.manager.forms.requests.get') +def test_meetings_agenda_add_desk_with_non_existing_exceptions_url(mocked_get, app, admin_user): + app = login(app) + resp = app.get('/manage/', status=200) + resp = resp.click('New') + resp.form['label'] = 'Foo bar' + resp.form['kind'] = 'meetings' + resp = resp.form.submit() + agenda = Agenda.objects.get(slug='foo-bar') + resp = app.get('/manage/agendas/%s/' % agenda.id, status=200) + mocked_response = mock.Mock() + mocked_response.status_code = 403 + mocked_get.return_value = mocked_response + def mocked_requests_http_forbidden_error(*args, **kwargs): + raise requests.exceptions.HTTPError(response=mocked_response) + mocked_get.side_effect = mocked_requests_http_forbidden_error + resp = resp.click('New Desk') + resp.form['label'] = 'Desk A' + resp.form['timeperiod_exceptions_remote_url'] = 'http://nowhere.com/unknown.ics' + resp = resp.form.submit(status=200) + assert 'Failed to retrieve remote calendar (HTTP error 403).' in resp.text + +@mock.patch('chrono.manager.forms.requests.get') +def test_meetings_agenda_add_desk_with_unreachable_exceptions_url(mocked_get, app, admin_user): + app = login(app) + resp = app.get('/manage/', status=200) + resp = resp.click('New') + resp.form['label'] = 'Foo bar' + resp.form['kind'] = 'meetings' + resp = resp.form.submit() + agenda = Agenda.objects.get(slug='foo-bar') + resp = app.get('/manage/agendas/%s/' % agenda.id, status=200) + mocked_response = mock.Mock() + mocked_get.return_value = mocked_response + def mocked_requests_connection_error(*args, **kwargs): + raise requests.exceptions.ConnectionError('unreachable') + mocked_get.side_effect = mocked_requests_connection_error + resp = resp.click('New Desk') + resp.form['label'] = 'Desk A' + resp.form['timeperiod_exceptions_remote_url'] = 'http://nowhere.com/unknown.ics' + resp = resp.form.submit(status=200) + assert 'Failed to retrieve remote calendar (connection error).' in resp.text + def test_meetings_agenda_delete_desk(app, admin_user): app = login(app) @@ -840,6 +885,9 @@ def test_agenda_import_time_period_exception_from_ics(app, admin_user): resp = app.get('/manage/agendas/%d/' % agenda.pk) assert 'Import exceptions from .ics' in resp.content resp = resp.click('upload') + assert "You can upload a file or specify an address to remote calendar." in resp + resp = resp.form.submit(status=200) + assert "A file or an url should be filled." in resp.body resp.form['ics_file'] = Upload('exceptions.ics', 'invalid content', 'text/calendar') resp = resp.form.submit(status=200) assert 'File format is invalid' in resp.content @@ -889,3 +937,98 @@ END:VCALENDAR""" assert TimePeriodException.objects.count() == 1 resp = resp.follow() assert 'An exception has been imported.' in resp.content + +@mock.patch('chrono.agendas.models.requests.get') +def test_agenda_import_time_period_exception_from_remote_ics(mocked_get, app, admin_user): + agenda = Agenda.objects.create(label='New Example', kind='meetings') + desk = Desk.objects.create(agenda=agenda, label='New Desk') + MeetingType(agenda=agenda, label='Bar').save() + login(app) + resp = app.get('/manage/agendas/%d/' % agenda.pk) + assert 'Import exceptions from .ics' not in resp.content + + TimePeriod.objects.create(weekday=1, desk=desk, + start_time=datetime.time(10, 0), end_time=datetime.time(12, 0)) + + resp = app.get('/manage/agendas/%d/' % agenda.pk) + resp = resp.click('upload') + + assert 'ics_file' in resp.form.fields + assert 'ics_url' in resp.form.fields + resp.form['ics_url'] = 'http://example.com/foo.ics' + mocked_response = mock.Mock() + mocked_get.return_value = mocked_response + def mocked_requests_connection_error(*args, **kwargs): + raise requests.exceptions.ConnectionError('unreachable') + mocked_get.side_effect = mocked_requests_connection_error + resp = resp.form.submit(status=200) + assert 'Failed to retrieve remote calendar (connection error).' in resp.content + +@mock.patch('chrono.agendas.models.requests.get') +def test_agenda_import_time_period_exception_from_forbidden_remote_ics(mocked_get, app, admin_user): + agenda = Agenda.objects.create(label='New Example', kind='meetings') + desk = Desk.objects.create(agenda=agenda, label='New Desk') + MeetingType(agenda=agenda, label='Bar').save() + login(app) + resp = app.get('/manage/agendas/%d/' % agenda.pk) + assert 'Import exceptions from .ics' not in resp.content + + TimePeriod.objects.create(weekday=1, desk=desk, + start_time=datetime.time(10, 0), end_time=datetime.time(12, 0)) + + resp = app.get('/manage/agendas/%d/' % agenda.pk) + resp = resp.click('upload') + resp.form['ics_url'] = 'http://example.com/foo.ics' + mocked_response = mock.Mock() + mocked_response.status_code = 403 + mocked_get.return_value = mocked_response + def mocked_requests_http_forbidden_error(*args, **kwargs): + raise requests.exceptions.HTTPError(response=mocked_response) + mocked_get.side_effect = mocked_requests_http_forbidden_error + resp = resp.form.submit(status=200) + assert 'Failed to retrieve remote calendar (HTTP error 403).' in resp.content + +@mock.patch('chrono.agendas.models.requests.get') +def test_agenda_import_time_period_exception_from_remote_ics_with_timeout_error(mocked_get, app, admin_user): + agenda = Agenda.objects.create(label='New Example', kind='meetings') + desk = Desk.objects.create(agenda=agenda, label='New Desk') + MeetingType(agenda=agenda, label='Bar').save() + login(app) + resp = app.get('/manage/agendas/%d/' % agenda.pk) + assert 'Import exceptions from .ics' not in resp.content + + TimePeriod.objects.create(weekday=1, desk=desk, + start_time=datetime.time(10, 0), end_time=datetime.time(12, 0)) + + resp = app.get('/manage/agendas/%d/' % agenda.pk) + resp = resp.click('upload') + resp.form['ics_url'] = 'http://example.com/foo.ics' + mocked_response = mock.Mock() + mocked_get.return_value = mocked_response + def mocked_requests_http_timeout_error(*args, **kwargs): + raise requests.exceptions.Timeout(response=mocked_response) + mocked_get.side_effect = mocked_requests_http_timeout_error + resp = resp.form.submit(status=200) + assert 'Failed to retrieve remote calendar (HTTP timeout).' in resp.content + +@mock.patch('chrono.agendas.models.requests.get') +def test_agenda_import_time_period_exception_from_remote_ics_with_ssl_error(mocked_get, app, admin_user): + agenda = Agenda.objects.create(label='New Example', kind='meetings') + desk = Desk.objects.create(agenda=agenda, label='New Desk') + MeetingType(agenda=agenda, label='Bar').save() + login(app) + resp = app.get('/manage/agendas/%d/' % agenda.pk) + assert 'Import exceptions from .ics' not in resp.content + TimePeriod.objects.create(weekday=1, desk=desk, + start_time=datetime.time(10, 0), end_time=datetime.time(12, 0)) + + resp = app.get('/manage/agendas/%d/' % agenda.pk) + resp = resp.click('upload') + resp.form['ics_url'] = 'http://example.com/foo.ics' + mocked_response = mock.Mock() + mocked_get.return_value = mocked_response + def mocked_requests_http_ssl_error(*args, **kwargs): + raise requests.exceptions.SSLError + mocked_get.side_effect = mocked_requests_http_ssl_error + resp = resp.form.submit(status=200) + assert 'Failed to retrieve remote calendar (connection error).' in resp.content -- 2.14.2