From 20617c515a067574b5a68d15926b55d364e19dce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Sat, 6 Jun 2015 16:16:19 +0200
Subject: [PATCH 1/2] add indexing of pages (#6793)

---
 combo/apps/newsletters/models.py           |  6 ++-
 combo/apps/notifications/models.py         |  6 +--
 combo/data/models.py                       | 20 +++++++++-
 combo/data/search_indexes.py               | 37 ++++++++++++++++++
 combo/data/templates/combo/search/page.txt |  5 +++
 combo/settings.py                          | 10 +++++
 debian/control                             |  5 ++-
 debian/cron.hourly                         |  1 +
 requirements.txt                           |  2 +
 setup.py                                   |  2 +
 tests/test_notification.py                 |  4 +-
 tests/test_search.py                       | 60 +++++++++++++++++++++++++++++-
 12 files changed, 147 insertions(+), 11 deletions(-)
 create mode 100644 combo/data/search_indexes.py
 create mode 100644 combo/data/templates/combo/search/page.txt

diff --git a/combo/apps/newsletters/models.py b/combo/apps/newsletters/models.py
index 6e07119..ac35b89 100644
--- a/combo/apps/newsletters/models.py
+++ b/combo/apps/newsletters/models.py
@@ -166,5 +166,7 @@ class NewslettersCell(CellBase):
             context['form'] = form
         return super(NewslettersCell, self).render(context)
 
-    def is_relevant(self, context):
-        return bool(context.get('user').is_authenticated())
+    def is_visible(self, user=None):
+        if user is None or not user.is_authenticated():
+            return False
+        return super(NewslettersCell, self).is_visible(user)
diff --git a/combo/apps/notifications/models.py b/combo/apps/notifications/models.py
index f7f9435..080acc2 100644
--- a/combo/apps/notifications/models.py
+++ b/combo/apps/notifications/models.py
@@ -110,10 +110,10 @@ class NotificationsCell(CellBase):
     class Meta:
         verbose_name = _('User Notifications')
 
-    def is_relevant(self, context):
-        if not (getattr(context['request'], 'user', None) and context['request'].user.is_authenticated()):
+    def is_visible(self, user=None):
+        if user is None or not user.is_authenticated():
             return False
-        return True
+        return super(NotificationsCell, self).is_visible(user)
 
     def get_cell_extra_context(self, context):
         extra_context = super(NotificationsCell, self).get_cell_extra_context(context)
diff --git a/combo/data/models.py b/combo/data/models.py
index 65e850e..fac84b7 100644
--- a/combo/data/models.py
+++ b/combo/data/models.py
@@ -34,6 +34,7 @@ from django.db.models import Max
 from django.forms import models as model_forms
 from django import forms
 from django import template
+from django.utils.html import strip_tags
 from django.utils.safestring import mark_safe
 from django.utils.text import slugify
 from django.utils.translation import ugettext_lazy as _
@@ -206,8 +207,11 @@ class Page(models.Model):
     def is_visible(self, user=None):
         return element_is_visible(self, user=user)
 
+    def get_cells(self):
+        return CellBase.get_cells(page_id=self.id)
+
     def get_serialized_page(self):
-        cells = CellBase.get_cells(page_id=self.id)
+        cells = self.get_cells()
         serialized_page = json.loads(serializers.serialize('json', [self],
             use_natural_foreign_keys=True, use_natural_primary_keys=True))[0]
         del serialized_page['model']
@@ -464,6 +468,17 @@ class CellBase(models.Model):
         '''Apply changes to the template context that must visible to all cells in the page'''
         return context
 
+    def render_for_search(self):
+        context = Context({'synchronous': True})
+        if not self.is_enabled():
+            return ''
+        if not self.is_visible(user=None):
+            return ''
+        if not self.is_relevant(context):
+            return ''
+        from HTMLParser import HTMLParser
+        return HTMLParser().unescape(strip_tags(self.render(context)))
+
 
 @register_cell_class
 class TextCell(CellBase):
@@ -588,6 +603,9 @@ class MenuCell(CellBase):
                 root_page=self.root_page, depth=self.depth)
         return ctx
 
+    def render_for_search(self):
+        return ''
+
 
 @register_cell_class
 class LinkCell(CellBase):
diff --git a/combo/data/search_indexes.py b/combo/data/search_indexes.py
new file mode 100644
index 0000000..b48fb00
--- /dev/null
+++ b/combo/data/search_indexes.py
@@ -0,0 +1,37 @@
+# combo - content management system
+# Copyright (C) 2014-2017  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from haystack import indexes
+from haystack.exceptions import SkipDocument
+
+from .models import Page, CellBase
+
+class PageIndex(indexes.SearchIndex, indexes.Indexable):
+    title = indexes.CharField(model_attr='title', boost=1.5)
+    text = indexes.CharField(document=True, use_template=True,
+            template_name='combo/search/page.txt')
+    url = indexes.CharField(indexed=False)
+
+    def get_model(self):
+        return Page
+
+    def prepare_url(self, obj):
+        return obj.get_online_url()
+
+    def prepare(self, obj):
+        if not obj.is_visible(user=None):
+            raise SkipDocument()
+        return super(PageIndex, self).prepare(obj)
diff --git a/combo/data/templates/combo/search/page.txt b/combo/data/templates/combo/search/page.txt
new file mode 100644
index 0000000..9d03682
--- /dev/null
+++ b/combo/data/templates/combo/search/page.txt
@@ -0,0 +1,5 @@
+{{object.title}}
+
+{% for cell in object.get_cells %}
+ {{ cell.render_for_search }}
+{% endfor %}
diff --git a/combo/settings.py b/combo/settings.py
index 93cd60b..8c06f89 100644
--- a/combo/settings.py
+++ b/combo/settings.py
@@ -57,6 +57,7 @@ INSTALLED_APPS = (
     'django.contrib.staticfiles',
     'rest_framework',
     'ckeditor',
+    'haystack',
     'gadjo',
     'cmsplugin_blurp',
     'combo.data',
@@ -134,6 +135,7 @@ MEDIA_URL = '/media/'
 
 CKEDITOR_UPLOAD_PATH = 'uploads/'
 CKEDITOR_IMAGE_BACKEND = 'pillow'
+
 CKEDITOR_CONFIGS = {
     'default': {
         'allowedContent': True,
@@ -150,6 +152,14 @@ CKEDITOR_CONFIGS = {
     },
 }
 
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+        'PATH': os.path.join(BASE_DIR, 'whoosh_index'),
+    },
+}
+
+
 COMBO_DEFAULT_PUBLIC_TEMPLATE = 'standard'
 COMBO_PUBLIC_TEMPLATES = {
     'standard': {
diff --git a/debian/control b/debian/control
index c04d067..c55f368 100644
--- a/debian/control
+++ b/debian/control
@@ -16,8 +16,9 @@ Depends: ${misc:Depends}, ${python:Depends},
     python-feedparser,
     python-django-cmsplugin-blurp,
     python-xstatic-chartnew-js,
-    python-eopayment (>= 1.9)
-Recommends: python-django-mellon
+    python-eopayment (>= 1.9),
+    python-django-haystack (>= 2.4.0)
+Recommends: python-django-mellon, python-whoosh
 Conflicts: python-lingo
 Description: Portal Management System (Python module)
 
diff --git a/debian/cron.hourly b/debian/cron.hourly
index d04e9c3..9764037 100644
--- a/debian/cron.hourly
+++ b/debian/cron.hourly
@@ -1,3 +1,4 @@
 #!/bin/sh
 su combo -s /bin/sh -c "/usr/bin/combo-manage tenant_command update_transactions --all-tenants"
 su combo -s /bin/sh -c "/usr/bin/combo-manage tenant_command update_momo_manifest --all-tenants -v0"
+su combo -s /bin/sh -c "/usr/bin/combo-manage tenant_command update_index --remove --all-tenants -v0"
diff --git a/requirements.txt b/requirements.txt
index d0e520e..cf95de6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,3 +9,5 @@ XStatic-ChartNew.js
 eopayment>=1.13
 python-dateutil
 djangorestframework>=3.3, <3.4
+django-haystack
+whoosh
diff --git a/setup.py b/setup.py
index 6419426..1113f29 100644
--- a/setup.py
+++ b/setup.py
@@ -114,6 +114,8 @@ setup(
         'eopayment>=1.13',
         'python-dateutil',
         'djangorestframework>=3.3, <3.4',
+        'django-haystack',
+        'whoosh',
         ],
     zip_safe=False,
     cmdclass={
diff --git a/tests/test_notification.py b/tests/test_notification.py
index 8868404..816c8fc 100644
--- a/tests/test_notification.py
+++ b/tests/test_notification.py
@@ -85,9 +85,9 @@ def test_notification_cell(user, user2):
     context['synchronous'] = True # to get fresh content
 
     context['request'].user = None
-    assert cell.is_relevant(context) is False
+    assert cell.is_visible(context['request'].user) is False
     context['request'].user = user
-    assert cell.is_relevant(context) is True
+    assert cell.is_visible(context['request'].user) is True
     assert cell.get_badge(context) is None
 
     id_noti1 = Notification.notify(user, 'notibar')
diff --git a/tests/test_search.py b/tests/test_search.py
index 239b9e4..502d02e 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -8,8 +8,11 @@ from django.test.client import RequestFactory
 from django.template import Context
 from django.core.urlresolvers import reverse
 
+from haystack.exceptions import SkipDocument
+
 from combo.apps.search.models import SearchCell
-from combo.data.models import Page, JsonCell
+from combo.data.models import Page, JsonCell, TextCell, MenuCell
+from combo.data.search_indexes import PageIndex
 
 pytestmark = pytest.mark.django_db
 
@@ -111,3 +114,58 @@ def test_search_global_context(app):
             requests_get.return_value = mock.Mock(content=json.dumps(data), status_code=200)
             resp = app.get(url)
             assert requests_get.call_args[0][0] == 'http://www.example.net/search/foo/'
+
+
+def test_search_cell_visibility(app):
+    page = Page(title='example page', slug='example-page')
+    page.save()
+
+    with SearchServices(SEARCH_SERVICES):
+        cell = SearchCell(page=page, order=0)
+        assert not cell.is_visible()
+
+        cell._search_service = '_text'
+        assert cell.is_visible()
+
+def test_search_contents():
+    page = Page(title='example page', slug='example-page')
+    page.save()
+
+    # no indexation of private cells (is_visible check)
+    cell = TextCell(page=page, text='foobar', public=False, order=0)
+    assert cell.render_for_search() == ''
+
+    # no indexation of empty cells (is_relevant check)
+    cell = TextCell(page=page, text='', order=0)
+    assert cell.render_for_search() == ''
+
+    # indexation
+    cell = TextCell(page=page, text='<p>foobar</p>', order=0)
+    assert cell.render_for_search().strip() == 'foobar'
+
+    # no indexation of menu cells
+    cell = MenuCell(page=page, order=0)
+    assert cell.render_for_search() == ''
+
+def test_search_contents_index():
+    page = Page(title='example page', slug='example-page')
+    page.save()
+
+    page_index = PageIndex()
+    assert page_index.get_model() is Page
+
+    assert page_index.prepare_url(page) == '/example-page/'
+
+    page_index.prepare(page)
+
+    page.public = False
+    with pytest.raises(SkipDocument):
+        page_index.prepare(page)
+
+    page.public = True
+    cell = TextCell(page=page, text='<p>foobar</p>', order=0)
+    cell.save()
+
+    prepared_data = page_index.prepare(page)
+    assert page.title in prepared_data['text']
+    assert 'foobar' in prepared_data['text']
-- 
2.11.0