From f9701ba0f813a10fccbc8e81495d9c018aebd04e Mon Sep 17 00:00:00 2001 From: Serghei Mihai Date: Wed, 28 Sep 2016 14:53:10 +0200 Subject: [PATCH] create announces from external RSS feed (#12919) --- corbo/forms.py | 4 ++-- corbo/migrations/0007_auto_20160928_1454.py | 24 ++++++++++++++++++++++++ corbo/models.py | 22 ++++++++++++++++++++++ corbo/templates/corbo/category_detail.html | 2 +- corbo/templates/corbo/category_form.html | 2 +- debian/control | 4 +++- debian/corbo.cron.d | 1 + requirements.txt | 2 ++ setup.py | 2 ++ 9 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 corbo/migrations/0007_auto_20160928_1454.py diff --git a/corbo/forms.py b/corbo/forms.py index 3ec9b00..68ae4ea 100644 --- a/corbo/forms.py +++ b/corbo/forms.py @@ -8,7 +8,7 @@ class AnnounceForm(forms.ModelForm): class Meta: model = Announce - fields = '__all__' + exclude = ('identifier',) widgets = { 'publication_time': forms.TextInput(attrs={'class': 'datetimepicker', 'readonly': True}), @@ -26,5 +26,5 @@ class AnnounceForm(forms.ModelForm): class CategoryForm(forms.ModelForm): class Meta: - fields = ('name', ) + fields = ('name', 'rss_feed_url') model = Category diff --git a/corbo/migrations/0007_auto_20160928_1454.py b/corbo/migrations/0007_auto_20160928_1454.py new file mode 100644 index 0000000..1e7c73a --- /dev/null +++ b/corbo/migrations/0007_auto_20160928_1454.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('corbo', '0006_auto_20160928_0833'), + ] + + operations = [ + migrations.AddField( + model_name='announce', + name='identifier', + field=models.CharField(max_length=256, null=True, blank=True), + ), + migrations.AddField( + model_name='category', + name='rss_feed_url', + field=models.URLField(help_text='if defined, announces will be automatically created from rss items', null=True, verbose_name='Feed URL', blank=True), + ), + ] diff --git a/corbo/models.py b/corbo/models.py index 6b1f7cb..36291c8 100644 --- a/corbo/models.py +++ b/corbo/models.py @@ -1,11 +1,14 @@ import os import hashlib +from time import mktime from datetime import datetime import logging import urlparse from html2text import HTML2Text from emails.django import Message from lxml.etree import HTML as HTMLTree +import requests +import feedparser from django.utils import timezone from django.conf import settings @@ -37,6 +40,8 @@ def transform_image_src(src, **kwargs): class Category(models.Model): name = models.CharField(_('Name'), max_length=64, blank=False, null=False) + rss_feed_url = models.URLField(_('Feed URL'), blank=True, null=True, + help_text=_('if defined, announces will be automatically created from rss items')) ctime = models.DateTimeField(auto_now_add=True) def __unicode__(self): @@ -48,11 +53,28 @@ class Category(models.Model): def get_subscriptions_count(self): return self.subscription_set.all().count() + def save(self, *args, **kwargs): + super(Category, self).save(*args, **kwargs) + if not self.rss_feed_url: + return + feed_response = requests.get(self.rss_feed_url) + if feed_response.ok: + content = feedparser.parse(feed_response.content) + for entry in content.get('entries', []): + published = datetime.fromtimestamp(mktime(entry.published_parsed)) + announce, created = Announce.objects.get_or_create(identifier=entry['id'], + category=self, defaults={'title': entry['title'], + 'text': entry['summary'], + 'publication_time': published}) + if created: + Broadcast.objects.get_or_create(announce=announce) + class Announce(models.Model): category = models.ForeignKey('Category', verbose_name=_('category')) title = models.CharField(_('title'), max_length=256, help_text=_('maximum 256 characters')) + identifier = models.CharField(max_length=256, null=True, blank=True) text = RichTextField(_('Content')) publication_time = models.DateTimeField(_('publication time'), blank=True, null=True) diff --git a/corbo/templates/corbo/category_detail.html b/corbo/templates/corbo/category_detail.html index 5b7b9b2..8fc7dce 100644 --- a/corbo/templates/corbo/category_detail.html +++ b/corbo/templates/corbo/category_detail.html @@ -11,7 +11,7 @@ {% block appbar %}

{{ object.name }}

{% trans 'Delete' %} -{% trans 'Rename' %} +{% trans 'Edit' %} {% trans 'New announce' %} {% endblock %} diff --git a/corbo/templates/corbo/category_form.html b/corbo/templates/corbo/category_form.html index 560ab33..23a77c9 100644 --- a/corbo/templates/corbo/category_form.html +++ b/corbo/templates/corbo/category_form.html @@ -3,7 +3,7 @@ {% block appbar %} {% if object %} -

{% trans "Modify Category" %}

+

{% trans "Edit Category" %}

{% else %}

{% trans "New Category" %}

{% endif %} diff --git a/debian/control b/debian/control index f2e2477..dbd88db 100644 --- a/debian/control +++ b/debian/control @@ -12,7 +12,9 @@ Depends: ${misc:Depends}, ${python:Depends}, python-django (>= 1.7), python-django-ckeditor, python-gadjo, - python-emails + python-emails, + python-requests, + python-feedparser Description: Announces Manager Package: corbo diff --git a/debian/corbo.cron.d b/debian/corbo.cron.d index 46d2506..19112e1 100644 --- a/debian/corbo.cron.d +++ b/debian/corbo.cron.d @@ -1,3 +1,4 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin */5 * * * * corbo corbo-manage tenant_command send_announces --all-tenants +0 * * * * corbo corbo-manage tenant_command sync_external_feeds --all-tenants diff --git a/requirements.txt b/requirements.txt index 3b27fff..a08d428 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,6 @@ django-ckeditor<4.5.3 djangorestframework>=3.3,<3.4 html2text emails +feedparser +requests -e git+http://repos.entrouvert.org/gadjo.git/#egg=gadjo diff --git a/setup.py b/setup.py index 4804c7d..2ea6ba7 100644 --- a/setup.py +++ b/setup.py @@ -100,6 +100,8 @@ setup( 'gadjo', 'emails', 'lxml', + 'feedparser', + 'requests' ], zip_safe=False, cmdclass={ -- 2.9.3