Projet

Général

Profil

0001-add-indexing-of-pages-6793.patch

Frédéric Péters, 12 mars 2017 14:21

Télécharger (11,6 ko)

Voir les différences:

Subject: [PATCH 1/2] add indexing of pages (#6793)

 combo/apps/newsletters/models.py           |  6 ++-
 combo/apps/notifications/models.py         |  6 +--
 combo/data/models.py                       | 20 +++++++++-
 combo/data/search_indexes.py               | 37 ++++++++++++++++++
 combo/data/templates/combo/search/page.txt |  5 +++
 combo/settings.py                          | 10 +++++
 debian/control                             |  5 ++-
 debian/cron.hourly                         |  1 +
 requirements.txt                           |  2 +
 setup.py                                   |  2 +
 tests/test_notification.py                 |  4 +-
 tests/test_search.py                       | 60 +++++++++++++++++++++++++++++-
 12 files changed, 147 insertions(+), 11 deletions(-)
 create mode 100644 combo/data/search_indexes.py
 create mode 100644 combo/data/templates/combo/search/page.txt
combo/apps/newsletters/models.py
166 166
            context['form'] = form
167 167
        return super(NewslettersCell, self).render(context)
168 168

  
169
    def is_relevant(self, context):
170
        return bool(context.get('user').is_authenticated())
169
    def is_visible(self, user=None):
170
        if user is None or not user.is_authenticated():
171
            return False
172
        return super(NewslettersCell, self).is_visible(user)
combo/apps/notifications/models.py
110 110
    class Meta:
111 111
        verbose_name = _('User Notifications')
112 112

  
113
    def is_relevant(self, context):
114
        if not (getattr(context['request'], 'user', None) and context['request'].user.is_authenticated()):
113
    def is_visible(self, user=None):
114
        if user is None or not user.is_authenticated():
115 115
            return False
116
        return True
116
        return super(NotificationsCell, self).is_visible(user)
117 117

  
118 118
    def get_cell_extra_context(self, context):
119 119
        extra_context = super(NotificationsCell, self).get_cell_extra_context(context)
combo/data/models.py
34 34
from django.forms import models as model_forms
35 35
from django import forms
36 36
from django import template
37
from django.utils.html import strip_tags
37 38
from django.utils.safestring import mark_safe
38 39
from django.utils.text import slugify
39 40
from django.utils.translation import ugettext_lazy as _
......
206 207
    def is_visible(self, user=None):
207 208
        return element_is_visible(self, user=user)
208 209

  
210
    def get_cells(self):
211
        return CellBase.get_cells(page_id=self.id)
212

  
209 213
    def get_serialized_page(self):
210
        cells = CellBase.get_cells(page_id=self.id)
214
        cells = self.get_cells()
211 215
        serialized_page = json.loads(serializers.serialize('json', [self],
212 216
            use_natural_foreign_keys=True, use_natural_primary_keys=True))[0]
213 217
        del serialized_page['model']
......
464 468
        '''Apply changes to the template context that must visible to all cells in the page'''
465 469
        return context
466 470

  
471
    def render_for_search(self):
472
        context = Context({'synchronous': True})
473
        if not self.is_enabled():
474
            return
475
        if not self.is_visible(user=None):
476
            return
477
        if not self.is_relevant(context):
478
            return
479
        from HTMLParser import HTMLParser
480
        return HTMLParser().unescape(strip_tags(self.render(context)))
481

  
467 482

  
468 483
@register_cell_class
469 484
class TextCell(CellBase):
......
588 603
                root_page=self.root_page, depth=self.depth)
589 604
        return ctx
590 605

  
606
    def render_for_search(self):
607
        return ''
608

  
591 609

  
592 610
@register_cell_class
593 611
class LinkCell(CellBase):
combo/data/search_indexes.py
1
# combo - content management system
2
# Copyright (C) 2014-2017  Entr'ouvert
3
#
4
# This program is free software: you can redistribute it and/or modify it
5
# under the terms of the GNU Affero General Public License as published
6
# by the Free Software Foundation, either version 3 of the License, or
7
# (at your option) any later version.
8
#
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
# GNU Affero General Public License for more details.
13
#
14
# You should have received a copy of the GNU Affero General Public License
15
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16

  
17
from haystack import indexes
18
from haystack.exceptions import SkipDocument
19

  
20
from .models import Page, CellBase
21

  
22
class PageIndex(indexes.SearchIndex, indexes.Indexable):
23
    title = indexes.CharField(model_attr='title', boost=1.5)
24
    text = indexes.CharField(document=True, use_template=True,
25
            template_name='combo/search/page.txt')
26
    url = indexes.CharField(indexed=False)
27

  
28
    def get_model(self):
29
        return Page
30

  
31
    def prepare_url(self, obj):
32
        return obj.get_online_url()
33

  
34
    def prepare(self, obj):
35
        if not obj.is_visible(user=None):
36
            raise SkipDocument()
37
        return super(PageIndex, self).prepare(obj)
combo/data/templates/combo/search/page.txt
1
{{object.title}}
2

  
3
{% for cell in object.get_cells %}
4
 {{ cell.render_for_search }}
5
{% endfor %}
combo/settings.py
57 57
    'django.contrib.staticfiles',
58 58
    'rest_framework',
59 59
    'ckeditor',
60
    'haystack',
60 61
    'gadjo',
61 62
    'cmsplugin_blurp',
62 63
    'combo.data',
......
134 135

  
135 136
CKEDITOR_UPLOAD_PATH = 'uploads/'
136 137
CKEDITOR_IMAGE_BACKEND = 'pillow'
138

  
137 139
CKEDITOR_CONFIGS = {
138 140
    'default': {
139 141
        'allowedContent': True,
......
150 152
    },
151 153
}
152 154

  
155
HAYSTACK_CONNECTIONS = {
156
    'default': {
157
        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
158
        'PATH': os.path.join(BASE_DIR, 'whoosh_index'),
159
    },
160
}
161

  
162

  
153 163
COMBO_DEFAULT_PUBLIC_TEMPLATE = 'standard'
154 164
COMBO_PUBLIC_TEMPLATES = {
155 165
    'standard': {
debian/control
16 16
    python-feedparser,
17 17
    python-django-cmsplugin-blurp,
18 18
    python-xstatic-chartnew-js,
19
    python-eopayment (>= 1.9)
20
Recommends: python-django-mellon
19
    python-eopayment (>= 1.9),
20
    python-django-haystack (>= 2.4.0)
21
Recommends: python-django-mellon, python-whoosh
21 22
Conflicts: python-lingo
22 23
Description: Portal Management System (Python module)
23 24

  
debian/cron.hourly
1 1
#!/bin/sh
2 2
su combo -s /bin/sh -c "/usr/bin/combo-manage tenant_command update_transactions --all-tenants"
3 3
su combo -s /bin/sh -c "/usr/bin/combo-manage tenant_command update_momo_manifest --all-tenants -v0"
4
su combo -s /bin/sh -c "/usr/bin/combo-manage tenant_command update_index --remove --all-tenants -v0"
requirements.txt
9 9
eopayment>=1.13
10 10
python-dateutil
11 11
djangorestframework>=3.3, <3.4
12
django-haystack
13
whoosh
setup.py
114 114
        'eopayment>=1.13',
115 115
        'python-dateutil',
116 116
        'djangorestframework>=3.3, <3.4',
117
        'django-haystack',
118
        'whoosh',
117 119
        ],
118 120
    zip_safe=False,
119 121
    cmdclass={
tests/test_notification.py
85 85
    context['synchronous'] = True # to get fresh content
86 86

  
87 87
    context['request'].user = None
88
    assert cell.is_relevant(context) is False
88
    assert cell.is_visible(context['request'].user) is False
89 89
    context['request'].user = user
90
    assert cell.is_relevant(context) is True
90
    assert cell.is_visible(context['request'].user) is True
91 91
    assert cell.get_badge(context) is None
92 92

  
93 93
    id_noti1 = Notification.notify(user, 'notibar')
tests/test_search.py
8 8
from django.template import Context
9 9
from django.core.urlresolvers import reverse
10 10

  
11
from haystack.exceptions import SkipDocument
12

  
11 13
from combo.apps.search.models import SearchCell
12
from combo.data.models import Page, JsonCell
14
from combo.data.models import Page, JsonCell, TextCell, MenuCell
15
from combo.data.search_indexes import PageIndex
13 16

  
14 17
pytestmark = pytest.mark.django_db
15 18

  
......
111 114
            requests_get.return_value = mock.Mock(content=json.dumps(data), status_code=200)
112 115
            resp = app.get(url)
113 116
            assert requests_get.call_args[0][0] == 'http://www.example.net/search/foo/'
117

  
118

  
119
def test_search_cell_visibility(app):
120
    page = Page(title='example page', slug='example-page')
121
    page.save()
122

  
123
    with SearchServices(SEARCH_SERVICES):
124
        cell = SearchCell(page=page, order=0)
125
        assert not cell.is_visible()
126

  
127
        cell._search_service = '_text'
128
        assert cell.is_visible()
129

  
130
def test_search_contents():
131
    page = Page(title='example page', slug='example-page')
132
    page.save()
133

  
134
    # no indexation of private cells (is_visible check)
135
    cell = TextCell(page=page, text='foobar', public=False, order=0)
136
    assert cell.render_for_search() == ''
137

  
138
    # no indexation of empty cells (is_relevant check)
139
    cell = TextCell(page=page, text='', order=0)
140
    assert cell.render_for_search() == ''
141

  
142
    # indexation
143
    cell = TextCell(page=page, text='<p>foobar</p>', order=0)
144
    assert cell.render_for_search().strip() == 'foobar'
145

  
146
    # no indexation of menu cells
147
    cell = MenuCell(page=page, order=0)
148
    assert cell.render_for_search() == ''
149

  
150
def test_search_contents_index():
151
    page = Page(title='example page', slug='example-page')
152
    page.save()
153

  
154
    page_index = PageIndex()
155
    assert page_index.get_model() is Page
156

  
157
    assert page_index.prepare_url(page) == '/example-page/'
158

  
159
    page_index.prepare(page)
160

  
161
    page.public = False
162
    with pytest.raises(SkipDocument):
163
        page_index.prepare(page)
164

  
165
    page.public = True
166
    cell = TextCell(page=page, text='<p>foobar</p>', order=0)
167
    cell.save()
168

  
169
    prepared_data = page_index.prepare(page)
170
    assert page.title in prepared_data['text']
171
    assert 'foobar' in prepared_data['text']
114
-