1
|
import os
|
2
|
import io
|
3
|
import hashlib
|
4
|
from time import mktime
|
5
|
from datetime import datetime
|
6
|
import logging
|
7
|
import urlparse
|
8
|
from html2text import HTML2Text
|
9
|
from emails.django import Message
|
10
|
from lxml import etree
|
11
|
import requests
|
12
|
import feedparser
|
13
|
|
14
|
from django.utils import timezone
|
15
|
from django.conf import settings
|
16
|
from django.db import models
|
17
|
from django.core.files.storage import DefaultStorage
|
18
|
from django.utils.translation import ugettext_lazy as _
|
19
|
from django.core import signing
|
20
|
from django.template import loader, Context
|
21
|
from django.core.urlresolvers import reverse
|
22
|
from django.utils.translation import activate
|
23
|
|
24
|
from ckeditor.fields import RichTextField
|
25
|
|
26
|
channel_choices = (
|
27
|
('mailto', _('Email')),
|
28
|
)
|
29
|
|
30
|
UNSUBSCRIBE_LINK_PLACEHOLDER = '##UNSUBSCRIBE_LINK_PLACEHOLDER##'
|
31
|
|
32
|
logger = logging.getLogger(__name__)
|
33
|
|
34
|
|
35
|
def transform_image_src(src, **kwargs):
|
36
|
basename = os.path.basename(src)
|
37
|
if basename == src:
|
38
|
return src
|
39
|
name, ext = os.path.splitext(src)
|
40
|
hash = hashlib.sha256(name)
|
41
|
return '%s_%s%s' % (os.path.basename(name), hash.hexdigest()[:8], ext)
|
42
|
|
43
|
|
44
|
class Category(models.Model):
|
45
|
name = models.CharField(_('Name'), max_length=64, blank=False, null=False)
|
46
|
slug = models.SlugField(_('Slug'), unique=True)
|
47
|
rss_feed_url = models.URLField(_('Feed URL'), blank=True, null=True,
|
48
|
help_text=_('if defined, announces will be automatically created from rss items'))
|
49
|
ctime = models.DateTimeField(auto_now_add=True)
|
50
|
|
51
|
def __unicode__(self):
|
52
|
return self.name
|
53
|
|
54
|
def get_announces_count(self):
|
55
|
return self.announce_set.all().count()
|
56
|
|
57
|
def get_subscriptions_count(self):
|
58
|
return self.subscription_set.all().count()
|
59
|
|
60
|
def save(self, *args, **kwargs):
|
61
|
super(Category, self).save(*args, **kwargs)
|
62
|
if not self.rss_feed_url:
|
63
|
return
|
64
|
feed_response = requests.get(self.rss_feed_url)
|
65
|
if feed_response.ok:
|
66
|
content = feedparser.parse(feed_response.content)
|
67
|
for entry in content.get('entries', []):
|
68
|
published = datetime.fromtimestamp(mktime(entry.published_parsed))
|
69
|
html_tree = etree.HTML(entry['summary'])
|
70
|
storage = DefaultStorage()
|
71
|
for img in html_tree.xpath('//img'):
|
72
|
image_name = os.path.basename(img.attrib['src'])
|
73
|
r = requests.get(img.attrib['src'])
|
74
|
new_content = r.content
|
75
|
if storage.exists(image_name):
|
76
|
old_content = storage.open(image_name).read()
|
77
|
old_hash = hashlib.md5(old_content).hexdigest()
|
78
|
new_hash = hashlib.md5(new_content).hexdigest()
|
79
|
img.attrib['src'] = storage.url(image_name)
|
80
|
if new_hash == old_hash:
|
81
|
continue
|
82
|
new_image_name = storage.save(image_name, io.BytesIO(new_content))
|
83
|
img.attrib['src'] = storage.url(new_image_name)
|
84
|
|
85
|
announce, created = Announce.objects.get_or_create(identifier=entry['id'],
|
86
|
category=self)
|
87
|
announce.title = entry['title']
|
88
|
announce.text = etree.tostring(html_tree)
|
89
|
announce.publication_time = published
|
90
|
announce.save()
|
91
|
|
92
|
if created:
|
93
|
Broadcast.objects.get_or_create(announce=announce)
|
94
|
|
95
|
|
96
|
class Announce(models.Model):
|
97
|
category = models.ForeignKey('Category', verbose_name=_('category'))
|
98
|
title = models.CharField(_('title'), max_length=256,
|
99
|
help_text=_('maximum 256 characters'))
|
100
|
identifier = models.CharField(max_length=256, null=True, blank=True)
|
101
|
text = RichTextField(_('Content'))
|
102
|
publication_time = models.DateTimeField(_('Publication date'), blank=True,
|
103
|
null=True)
|
104
|
expiration_time = models.DateTimeField(_('Expiration date'), blank=True,
|
105
|
null=True)
|
106
|
ctime = models.DateTimeField(_('creation time'), auto_now_add=True)
|
107
|
mtime = models.DateTimeField(_('modification time'), auto_now=True)
|
108
|
|
109
|
def __unicode__(self):
|
110
|
return u'{title} ({id}) at {mtime}'.format(
|
111
|
title=self.title, id=self.id, mtime=self.mtime)
|
112
|
|
113
|
def is_expired(self):
|
114
|
if self.expiration_time:
|
115
|
return self.expiration_time < timezone.now()
|
116
|
return False
|
117
|
|
118
|
def is_published(self):
|
119
|
if self.publication_time:
|
120
|
return self.publication_time <= timezone.now()
|
121
|
return False
|
122
|
|
123
|
class Meta:
|
124
|
verbose_name = _('announce')
|
125
|
ordering = ('-mtime',)
|
126
|
|
127
|
|
128
|
class Broadcast(models.Model):
|
129
|
announce = models.ForeignKey(Announce, verbose_name=_('announce'))
|
130
|
deliver_time = models.DateTimeField(_('Deliver time'), null=True)
|
131
|
result = models.TextField(_('result'), blank=True)
|
132
|
|
133
|
def __unicode__(self):
|
134
|
if self.deliver_time:
|
135
|
return u'announce {id} delivered via at {time}'.format(
|
136
|
id=self.announce.id, time=self.deliver_time)
|
137
|
return u'announce {id} to deliver'.format(id=self.announce.id)
|
138
|
|
139
|
def send(self):
|
140
|
subscriptions = self.announce.category.subscription_set.all()
|
141
|
total_sent = 0
|
142
|
handler = HTML2Text()
|
143
|
activate(settings.LANGUAGE_CODE)
|
144
|
template = loader.get_template('corbo/announce.html')
|
145
|
message = Message(subject=self.announce.title, mail_from=settings.DEFAULT_FROM_EMAIL,
|
146
|
html=template.render(
|
147
|
Context({'content': self.announce.text,
|
148
|
'unsubscribe_link_placeholder': UNSUBSCRIBE_LINK_PLACEHOLDER})))
|
149
|
html_tree = etree.HTML(self.announce.text)
|
150
|
storage = DefaultStorage()
|
151
|
for img in html_tree.xpath('//img/@src'):
|
152
|
img_path = img.lstrip(storage.base_url)
|
153
|
message.attach(filename=transform_image_src(img), data=storage.open(img_path))
|
154
|
|
155
|
message.transformer.apply_to_images(func=transform_image_src)
|
156
|
# perform transformations in message html, like inline css parsing
|
157
|
message.transformer.load_and_transform()
|
158
|
# mark all attached images as inline
|
159
|
message.transformer.make_all_images_inline()
|
160
|
message.transformer.save()
|
161
|
for s in subscriptions:
|
162
|
if not s.identifier:
|
163
|
continue
|
164
|
unsubscribe_token = signing.dumps({'category': self.announce.category.pk,
|
165
|
'identifier': s.identifier})
|
166
|
unsubscribe_link = urlparse.urljoin(settings.SITE_BASE_URL, reverse(
|
167
|
'unsubscribe', kwargs={'unsubscription_token': unsubscribe_token}))
|
168
|
message.html = message.html.replace(UNSUBSCRIBE_LINK_PLACEHOLDER, unsubscribe_link)
|
169
|
handler.body_width = 0
|
170
|
message.text = handler.handle(message.html)
|
171
|
sent = message.send(to=s.identifier)
|
172
|
if sent:
|
173
|
total_sent += 1
|
174
|
logger.info('Announce "%s" sent to %s', self.announce.title, s.identifier)
|
175
|
else:
|
176
|
logger.warning('Error occured while sending announce "%s" to %s.',
|
177
|
self.announce.title, s.identifier)
|
178
|
self.result = total_sent
|
179
|
self.deliver_time = timezone.now()
|
180
|
self.save()
|
181
|
|
182
|
class Meta:
|
183
|
verbose_name = _('sent')
|
184
|
ordering = ('-deliver_time',)
|
185
|
|
186
|
|
187
|
class Subscription(models.Model):
|
188
|
category = models.ForeignKey('Category', verbose_name=_('Category'))
|
189
|
uuid = models.CharField(_('User identifier'), max_length=128, blank=True)
|
190
|
identifier = models.CharField(_('identifier'), max_length=128, blank=True,
|
191
|
help_text=_('ex.: mailto, ...'))
|
192
|
|
193
|
def get_identifier_display(self):
|
194
|
try:
|
195
|
scheme, identifier = self.identifier.split(':')
|
196
|
return identifier
|
197
|
except ValueError:
|
198
|
return self.identifier
|
199
|
|
200
|
class Meta:
|
201
|
unique_together = ('category', 'identifier', 'uuid')
|