From 2568088249a0f7906111b4e4bc10d3d7e2239f46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Mon, 6 Apr 2020 16:37:33 +0200
Subject: [PATCH 2/2] search: search on page and subpages (#40224)

---
 combo/apps/search/forms.py                    | 30 +++++++
 combo/apps/search/manager_views.py            | 41 ++++++---
 combo/apps/search/models.py                   | 50 +++++++++--
 .../combo/manager/add-engine-form.html        | 18 ++++
 .../combo/manager/search-cell-form.html       | 10 +--
 .../templates/combo/search-cell-results.html  |  2 +-
 combo/apps/search/utils.py                    |  7 +-
 tests/test_search.py                          | 86 ++++++++++++++++++-
 8 files changed, 220 insertions(+), 24 deletions(-)
 create mode 100644 combo/apps/search/templates/combo/manager/add-engine-form.html

diff --git a/combo/apps/search/forms.py b/combo/apps/search/forms.py
index 0c2f230..f4077ca 100644
--- a/combo/apps/search/forms.py
+++ b/combo/apps/search/forms.py
@@ -15,7 +15,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from django import forms
+from django.utils.translation import ugettext_lazy as _
 
+from combo.data.models import Page
 from .models import SearchCell
 
 
@@ -23,3 +25,31 @@ class SearchCellForm(forms.ModelForm):
     class Meta:
         model = SearchCell
         fields = ('autofocus', )
+
+
+class SelectPageForm(forms.ModelForm):
+    selected_page = forms.ModelChoiceField(
+        label=_('Page'),
+        required=False,
+        queryset=Page.objects.none(),
+        help_text=_("Select a page to limit the search on this page and sub pages contents."),
+    )
+
+    class Meta:
+        model = SearchCell
+        fields = []
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        used_slugs = [
+            e['slug'].replace('_text_page_', '') for e in self.instance.search_services
+            if e['slug'].startswith('_text_page_')]
+        pages_queryset = (
+            Page.objects
+            .filter(snapshot__isnull=True, sub_slug='')
+            .exclude(slug__in=used_slugs)
+            .order_by('title'))
+        self.fields['selected_page'].queryset = pages_queryset
+        # if '_text' without page is already selected, page is required
+        if any(e['slug'] == '_text' for e in self.instance.search_services):
+            self.fields['selected_page'].required = True
diff --git a/combo/apps/search/manager_views.py b/combo/apps/search/manager_views.py
index 4d9a6f5..273c07d 100644
--- a/combo/apps/search/manager_views.py
+++ b/combo/apps/search/manager_views.py
@@ -17,9 +17,11 @@
 from django.http import HttpResponse
 from django.http import HttpResponseRedirect
 from django.shortcuts import get_object_or_404
+from django.shortcuts import render
 from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 
+from combo.apps.search.forms import SelectPageForm
 from combo.apps.search.models import SearchCell
 from combo.data.models import PageSnapshot
 
@@ -27,16 +29,35 @@ from combo.data.models import PageSnapshot
 def page_search_cell_add_engine(request, page_pk, cell_reference, engine_slug):
     cell = get_object_or_404(SearchCell, pk=cell_reference.split('-')[1], page=page_pk)
 
-    if engine_slug in cell.available_engines:
-        if not cell._search_services or not cell._search_services.get('data'):
-            cell._search_services = {'data': []}
-        cell._search_services['data'].append(engine_slug)
-        cell.save()
-        PageSnapshot.take(cell.page, request=request, comment=_('changed cell "%s"') % cell)
-
-    return HttpResponseRedirect('%s#cell-%s' % (
-        reverse('combo-manager-page-view', kwargs={'pk': page_pk}),
-        cell_reference))
+    def add_slug(slug):
+        if slug in cell.available_engines or slug.startswith('_text_page'):
+            if not cell._search_services or not cell._search_services.get('data'):
+                cell._search_services = {'data': []}
+            cell._search_services['data'].append(slug)
+            cell.save()
+            PageSnapshot.take(cell.page, request=request, comment=_('changed cell "%s"') % cell)
+        return HttpResponseRedirect('%s#cell-%s' % (
+            reverse('combo-manager-page-view', kwargs={'pk': page_pk}),
+            cell_reference))
+
+    if engine_slug != '_text':
+        # add engine without intermediary form and popup
+        return add_slug(engine_slug)
+
+    if request.method == 'POST':
+        form = SelectPageForm(instance=cell, data=request.POST)
+        if form.is_valid():
+            slug = '_text'
+            if form.cleaned_data['selected_page'] is not None:
+                slug = '_text_page_%s' % form.cleaned_data['selected_page'].slug
+            return add_slug(slug)
+    else:
+        form = SelectPageForm(instance=cell)
+    context = {
+        'form': form,
+        'cell': cell,
+    }
+    return render(request, 'combo/manager/add-engine-form.html', context)
 
 
 def page_search_cell_delete_engine(request, page_pk, cell_reference, engine_slug):
diff --git a/combo/apps/search/models.py b/combo/apps/search/models.py
index 0c08376..e20d050 100644
--- a/combo/apps/search/models.py
+++ b/combo/apps/search/models.py
@@ -36,6 +36,17 @@ from combo.utils import get_templated_url
 from . import engines
 
 
+def get_root_page_and_children(service_slug):
+    if not service_slug.startswith('_text_page_'):
+        return
+    page_slug = service_slug.replace('_text_page_', '')
+    try:
+        root_page = Page.objects.get(slug=page_slug, sub_slug='')
+    except (Page.DoesNotExist, Page.MultipleObjectsReturned):
+        return
+    return root_page.get_descendants_and_me()
+
+
 @register_cell_class
 class SearchCell(CellBase):
     template_name = 'combo/search-cell.html'
@@ -68,6 +79,8 @@ class SearchCell(CellBase):
         services = []
         for service_slug in self._search_services.get('data') or []:
             service = engines.get(service_slug)
+            if service_slug.startswith('_text_page_'):
+                service = engines.get('_text')
             if service and (service.get('url') or service.get('function')):
                 service['slug'] = service_slug
                 services.append(service)
@@ -76,9 +89,28 @@ class SearchCell(CellBase):
     @cached_property
     def available_engines(self):
         all_engines = engines.get_engines()
-        current_engines = [e['slug'] for e in self.search_services]
+        # always remove _text engine: we can add search on page and sub pages
+        current_engines = [e['slug'] for e in self.search_services if e['slug'] != '_text']
         return {k: v for k, v in all_engines.items() if k not in current_engines}
 
+    def get_search_services_for_display(self):
+        # get pages for _text engines
+        page_slugs = [
+            e['slug'].replace('_text_page_', '') for e in self.search_services
+            if e['slug'].startswith('_text_page_')]
+        pages = (
+            Page.objects
+            .filter(snapshot__isnull=True, sub_slug='', slug__in=page_slugs)
+            .values('slug', 'title'))
+        pages_by_slug = {'_text_page_%s' % p['slug']: p['title'] for p in pages}
+        services = []
+        for service in self.search_services:
+            label = service['label']
+            if service['slug'] in pages_by_slug:
+                label = _('Page "%(page)s" and sub pages Contents') % {'page': pages_by_slug[service['slug']]}
+            services.append((service['slug'], label))
+        return services
+
     @property
     def has_multiple_search_services(self):
         return len(self._search_services.get('data') or []) > 1
@@ -124,15 +156,21 @@ class SearchCell(CellBase):
 
         query = request.GET.get('q')
 
-        def render_response(service={}, results={'err': 0, 'data': []}):
+        def render_response(service=None, results=None, pages=None):
+            service = service or {}
+            results = results or {'err': 0, 'data': []}
             template_names = ['combo/search-cell-results.html']
             if cell.slug:
                 template_names.insert(0, 'combo/cells/%s/search-cell-results.html' % cell.slug)
             tmpl = template.loader.select_template(template_names)
+            service_label = service.get('label')
+            if pages:
+                service_label = _('Page "%(page)s" and sub pages Contents') % {'page': pages[0].title}
             context = {
                 'cell': cell,
                 'results': results,
                 'search_service': service,
+                'search_service_label': service_label,
                 'query': query
             }
             return HttpResponse(tmpl.render(context, request), content_type='text/html')
@@ -146,8 +184,10 @@ class SearchCell(CellBase):
         if not query:
             return render_response(service)
 
+        pages = None
         if service.get('function'):  # internal search engine
-            results = {'data': service['function'](request, query)}
+            pages = get_root_page_and_children(service_slug)
+            results = {'data': service['function'](request, query, pages=pages)}
         else:
             url = get_templated_url(service['url'],
                     context={'request': request, 'q': query, 'search_service': service})
@@ -188,10 +228,10 @@ class SearchCell(CellBase):
                 for k, v in hit_templates.items():
                     hit[k] = v.render(RequestContext(request, hit))
 
-        return render_response(service, results)
+        return render_response(service, results, pages=pages)
 
     def has_text_search_service(self):
-        return '_text' in self._search_services.get('data', [])
+        return any(key.startswith('_text') for key in self._search_services.get('data', []))
 
     def missing_index(self):
         return IndexedCell.objects.all().count() == 0
diff --git a/combo/apps/search/templates/combo/manager/add-engine-form.html b/combo/apps/search/templates/combo/manager/add-engine-form.html
new file mode 100644
index 0000000..c593f28
--- /dev/null
+++ b/combo/apps/search/templates/combo/manager/add-engine-form.html
@@ -0,0 +1,18 @@
+{% extends "combo/manager_base.html" %}
+{% load i18n %}
+
+{% block appbar %}
+<h2>{% trans 'Add a "Page Contents" engine' %}</h2>
+{% endblock %}
+
+{% block content %}
+
+<form method="post" enctype="multipart/form-data">
+  {% csrf_token %}
+  {{ form.as_p }}
+  <div class="buttons">
+    <button class="submit-button">{% trans "Save" %}</button>
+    <a class="cancel" href="{% url 'combo-manager-page-view' pk=cell.page_id %}">{% trans 'Cancel' %}</a>
+  </div>
+</form>
+{% endblock %}
diff --git a/combo/apps/search/templates/combo/manager/search-cell-form.html b/combo/apps/search/templates/combo/manager/search-cell-form.html
index 6e03542..8a297ce 100644
--- a/combo/apps/search/templates/combo/manager/search-cell-form.html
+++ b/combo/apps/search/templates/combo/manager/search-cell-form.html
@@ -13,16 +13,16 @@
 </div>
 {% endif %}
 {{ form.as_p }}
-{% with cell.search_services as engines %}
+{% with cell.get_search_services_for_display as engines %}
 {% if engines %}
 <p><label>{% trans "Engines:" %}</label></p>
 <div>
   <ul class="objects-list list-of-links" id="list-of-links-{{ cell.pk }}"
      data-link-list-order-url="{% url 'combo-manager-search-engines-order' page_pk=page.pk cell_reference=cell.get_reference %}">
     {% for engine in engines %}
-    <li data-link-item-id="{{ engine.slug }}"><span class="handle">⣿</span>
-      <span>{{ engine.label }}</span>
-      <a title="{% trans "Delete" %}" class="link-action-icon delete" href="{% url 'combo-manager-page-search-cell-delete-engine' page_pk=page.pk cell_reference=cell.get_reference engine_slug=engine.slug %}">{% trans "Delete" %}</a>
+    <li data-link-item-id="{{ engine.0 }}"><span class="handle">⣿</span>
+      <span>{{ engine.1 }}</span>
+      <a title="{% trans "Delete" %}" class="link-action-icon delete" href="{% url 'combo-manager-page-search-cell-delete-engine' page_pk=page.pk cell_reference=cell.get_reference engine_slug=engine.0 %}">{% trans "Delete" %}</a>
     </li>
     {% endfor %}
   </ul>
@@ -51,7 +51,7 @@ $(function () {
 <div class="buttons">
   {% trans "Add an engine:" %}
   {% for key, engine in cell.available_engines.items %}
-  <a href="{% url 'combo-manager-page-search-cell-add-engine' page_pk=page.pk cell_reference=cell.get_reference engine_slug=key %}">{{ engine.label }}</a> {% if not forloop.last %}|{% endif %}
+  <a {% if key == '_text' %}rel="popup"{% endif %} href="{% url 'combo-manager-page-search-cell-add-engine' page_pk=page.pk cell_reference=cell.get_reference engine_slug=key %}">{{ engine.label }}</a> {% if not forloop.last %}|{% endif %}
   {% endfor %}
 </div>
 {% endif %}
diff --git a/combo/apps/search/templates/combo/search-cell-results.html b/combo/apps/search/templates/combo/search-cell-results.html
index dae4acb..2e36640 100644
--- a/combo/apps/search/templates/combo/search-cell-results.html
+++ b/combo/apps/search/templates/combo/search-cell-results.html
@@ -1,6 +1,6 @@
 {% load i18n %}
 {% if query %}
-{% if cell.has_multiple_search_services %}<p class="search-service-label">{{ search_service.label }}</p>{% endif %}
+{% if cell.has_multiple_search_services %}<p class="search-service-label">{{ search_service_label }}</p>{% endif %}
 {% if results.data %}
 <div class="links-list">
 <ul>
diff --git a/combo/apps/search/utils.py b/combo/apps/search/utils.py
index 30d034c..9661365 100644
--- a/combo/apps/search/utils.py
+++ b/combo/apps/search/utils.py
@@ -105,7 +105,9 @@ def index_site():
                     indexed_cell.save()
 
 
-def search_site(request, query):
+def search_site(request, query, pages=None):
+    pages = pages or []
+
     if connection.vendor == 'postgresql':
         config = settings.POSTGRESQL_FTS_SEARCH_CONFIG
         vector = SearchVector('title', config=config, weight='A') + SearchVector('indexed_text', config=config, weight='A')
@@ -122,6 +124,9 @@ def search_site(request, query):
                 Q(restricted_groups__in=request.user.groups.all()))
         qs = qs.exclude(excluded_groups__in=request.user.groups.all())
 
+    if pages:
+        qs = qs.filter(page__in=pages)
+
     hits = []
     seen = {}
     for hit in qs:
diff --git a/tests/test_search.py b/tests/test_search.py
index 11da8af..bfa6049 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -15,7 +15,7 @@ from django.utils.http import urlencode
 from combo.apps.search.engines import engines
 from combo.apps.search.models import SearchCell, IndexedCell
 from combo.apps.search.utils import index_site, search_site
-from combo.data.models import Page, JsonCell, TextCell, MenuCell, LinkCell
+from combo.data.models import Page, JsonCell, TextCell, MenuCell, LinkCell, PageSnapshot
 
 from .test_manager import login
 
@@ -312,6 +312,61 @@ def test_search_api(app):
     assert resp.text.count('<li') == 0
 
 
+def test_search_on_root_page_api(app):
+    # not indexed: with sub_slug
+    page = Page.objects.create(title='example page', slug='example-page', sub_slug='foo')
+    TextCell.objects.create(page=page, placeholder='content', text='<p>foobar baz</p>', order=0)
+
+    second_page = Page.objects.create(title='second page', slug='second-page')
+    TextCell.objects.create(page=second_page, placeholder='content', text='<p>other baz</p>', order=0)
+    sub_second_page = Page.objects.create(parent=second_page, title='sub second page', slug='sub-second-page')
+    TextCell.objects.create(page=sub_second_page, placeholder='content', text='<p>other baz</p>', order=0)
+    # not indexed: with snapshot
+    third_page = Page.objects.create(title='second page', slug='third-page')
+    TextCell.objects.create(page=third_page, placeholder='content', text='<p>other baz again</p>', order=0)
+    third_page.snapshot = PageSnapshot.objects.create(page=third_page)
+    third_page.save()
+
+    index_site()
+
+    cell = SearchCell.objects.create(page=page, placeholder='content', _search_services={'data': ['_text']}, order=1)
+
+    resp = app.get('/ajax/search/%s/_text/?q=baz' % cell.pk, status=200)
+    assert resp.text.count('<li') == 2
+    cell._search_services = {'data': ['_text_page_second-page']}
+    cell.save()
+    resp = app.get('/ajax/search/%s/_text_page_second-page/?q=baz' % cell.pk, status=200)
+    assert resp.text.count('<li') == 2
+    cell._search_services = {'data': ['_text_page_sub-second-page']}
+    cell.save()
+    resp = app.get('/ajax/search/%s/_text_page_sub-second-page/?q=baz' % cell.pk, status=200)
+    assert resp.text.count('<li') == 1
+    # invalid page, search everywhere
+    # with sub_slug
+    cell._search_services = {'data': ['_text_page_example-page']}
+    cell.save()
+    resp = app.get('/ajax/search/%s/_text_page_example-page/?q=baz' % cell.pk, status=200)
+    assert resp.text.count('<li') == 2
+    # with snapshot
+    cell._search_services = {'data': ['_text_page_third-page']}
+    cell.save()
+    resp = app.get('/ajax/search/%s/_text_page_third-page/?q=baz' % cell.pk, status=200)
+    assert resp.text.count('<li') == 2
+    # page does not exists, search everywhere
+    cell._search_services = {'data': ['_text_page_foo']}
+    cell.save()
+    resp = app.get('/ajax/search/%s/_text_page_foo/?q=baz' % cell.pk, status=200)
+    assert resp.text.count('<li') == 2
+    # slug is not unique, search everywhere
+    page.slug = 'sub-second-page'
+    page.sub_slug = ''
+    page.save()
+    cell._search_services = {'data': ['_text_page_sub-second-page']}
+    cell.save()
+    resp = app.get('/ajax/search/%s/_text_page_sub-second-page/?q=baz' % cell.pk, status=200)
+    assert resp.text.count('<li') == 2
+
+
 def test_search_external_links(app):
     page = Page(title='example page', slug='example-page')
     page.save()
@@ -372,6 +427,7 @@ def test_manager_search_cell(settings, app, admin_user):
 
     # add engines
     resp = resp.click(href='.*/search_searchcell-%s/engine/_text/add/' % cell.pk)
+    resp = resp.form.submit('submit')
     assert resp.status_int == 302
     assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
     resp = app.get('/manage/pages/%s/' % page.pk)
@@ -381,7 +437,8 @@ def test_manager_search_cell(settings, app, admin_user):
     cell.refresh_from_db()
     assert cell._search_services == {'data': ['_text', 'search1', 'search_tmpl']}
     resp = app.get('/manage/pages/%s/' % page.pk)
-    assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/add/' % (page.pk, cell.pk) not in resp.text
+    # '_text' is always available
+    assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/add/' % (page.pk, cell.pk) in resp.text
     assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search1/add/' % (page.pk, cell.pk) not in resp.text
     assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_tmpl/add/' % (page.pk, cell.pk) not in resp.text
     assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_alternate_key/add/' % (page.pk, cell.pk) in resp.text
@@ -407,6 +464,30 @@ def test_manager_search_cell(settings, app, admin_user):
     assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_tmpl/delete/' % (page.pk, cell.pk) not in resp.text
     assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/search_alternate_key/delete/' % (page.pk, cell.pk) not in resp.text
 
+    # add engines on page and sub pages
+    resp = resp.click(href='.*/search_searchcell-%s/engine/_text/add/' % cell.pk)
+    assert list(resp.context['form']['selected_page'].field.queryset) == [page]
+    resp = resp.form.submit('submit')
+    assert resp.status_int == 302
+    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
+    cell.refresh_from_db()
+    assert cell._search_services == {'data': ['search1', 'search_tmpl', '_text']}
+    resp = app.get('/manage/pages/%s/' % page.pk)
+    assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/add/' % (page.pk, cell.pk) in resp.text
+    assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/delete/' % (page.pk, cell.pk) in resp.text
+    assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text_page_one/delete/' % (page.pk, cell.pk) not in resp.text
+    resp = resp.click(href='.*/search_searchcell-%s/engine/_text/add/' % cell.pk)
+    resp.form['selected_page'] = page.pk
+    resp = resp.form.submit('submit')
+    assert resp.status_int == 302
+    assert resp.location.endswith('/manage/pages/%s/#cell-%s' % (page.pk, cell.get_reference()))
+    cell.refresh_from_db()
+    assert cell._search_services == {'data': ['search1', 'search_tmpl', '_text', '_text_page_one']}
+    resp = app.get('/manage/pages/%s/' % page.pk)
+    assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/add/' % (page.pk, cell.pk) in resp.text
+    assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text/delete/' % (page.pk, cell.pk) in resp.text
+    assert '/manage/search/pages/%s/cell/search_searchcell-%s/engine/_text_page_one/delete/' % (page.pk, cell.pk) in resp.text
+
 
 def test_manager_search_cell_order(settings, app, admin_user):
     settings.COMBO_SEARCH_SERVICES = SEARCH_SERVICES
@@ -435,6 +516,7 @@ def test_manager_waiting_index_message(app, admin_user):
     assert 'Content indexing has been scheduled' not in resp.text
 
     resp = resp.click(href='.*/search_searchcell-%s/engine/_text/add/' % cell.pk)
+    resp = resp.form.submit('submit')
     resp = app.get('/manage/pages/%s/' % page.pk)
     assert 'Content indexing has been scheduled' in resp.text
 
-- 
2.20.1

