From 9f85b90c074865fe5812a7bb42f60fd73b6769b5 Mon Sep 17 00:00:00 2001 From: Thomas NOEL Date: Fri, 20 Jul 2018 15:42:39 +0200 Subject: [PATCH 01/11] search: handle several search engines (#23534) --- combo/apps/search/README | 2 + .../management/commands/update_index.py | 2 +- .../migrations/0002_auto_20180720_1511.py | 22 +++++ .../migrations/0003_create_search_services.py | 35 +++++++ .../0004_remove_searchcell__search_service.py | 18 ++++ combo/apps/search/models.py | 94 ++++++++++++------- .../templates/combo/search-cell-results.html | 3 + .../search/templates/combo/search-cell.html | 24 +++-- combo/apps/search/urls.py | 2 +- combo/public/views.py | 2 +- tests/test_search.py | 24 ++--- 11 files changed, 168 insertions(+), 60 deletions(-) create mode 100644 combo/apps/search/migrations/0002_auto_20180720_1511.py create mode 100644 combo/apps/search/migrations/0003_create_search_services.py create mode 100644 combo/apps/search/migrations/0004_remove_searchcell__search_service.py diff --git a/combo/apps/search/README b/combo/apps/search/README index e738b15..5ef850c 100644 --- a/combo/apps/search/README +++ b/combo/apps/search/README @@ -4,6 +4,8 @@ Configure search services in settings.COMBO_SEARCH_SERVICES: 'user': { 'label': 'Search a user', 'url': 'https://.../api/user/?q=%(q)s', + # 'cache_duration': 60, # in seconds, default is 0 + # 'signature': True, # boolean, default is False }, } diff --git a/combo/apps/search/management/commands/update_index.py b/combo/apps/search/management/commands/update_index.py index 75a2f92..ca61cd1 100644 --- a/combo/apps/search/management/commands/update_index.py +++ b/combo/apps/search/management/commands/update_index.py @@ -30,7 +30,7 @@ class Command(UpdateIndexCommand): dest='skip_external_links_collection') def handle(self, **options): - if SearchCell.objects.filter(_search_service='_text').count() == 0: + if not any(SearchCell.get_cells_by_search_service('_text')): # do not index site if there's no matching search cell return if not options.get('skip_external_links_collection', False): diff --git a/combo/apps/search/migrations/0002_auto_20180720_1511.py b/combo/apps/search/migrations/0002_auto_20180720_1511.py new file mode 100644 index 0000000..5ee7df3 --- /dev/null +++ b/combo/apps/search/migrations/0002_auto_20180720_1511.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('search', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='searchcell', + name='_search_services', + field=jsonfield.fields.JSONField(default=dict, + verbose_name='Search Services', + blank=True), + ), + ] diff --git a/combo/apps/search/migrations/0003_create_search_services.py b/combo/apps/search/migrations/0003_create_search_services.py new file mode 100644 index 0000000..66eb545 --- /dev/null +++ b/combo/apps/search/migrations/0003_create_search_services.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + + +def create_search_services(apps, schema_editor): + SearchCell = apps.get_model('search', 'SearchCell') + for searchcell in SearchCell.objects.all(): + if searchcell._search_service: + searchcell._search_services = {'data': [searchcell._search_service]} + else: + searchcell._search_services = {'data': []} + searchcell.save() + + +def back_to_search_service(apps, schema_editor): + SearchCell = apps.get_model('search', 'SearchCell') + for searchcell in SearchCell.objects.all(): + if searchcell._search_services.get('data'): + searchcell._search_service = searchcell._search_services.get('data')[0] + else: + searchcell._search_service = '' + searchcell.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('search', '0002_auto_20180720_1511'), + ] + + operations = [ + migrations.RunPython(create_search_services, back_to_search_service), + ] diff --git a/combo/apps/search/migrations/0004_remove_searchcell__search_service.py b/combo/apps/search/migrations/0004_remove_searchcell__search_service.py new file mode 100644 index 0000000..22852d5 --- /dev/null +++ b/combo/apps/search/migrations/0004_remove_searchcell__search_service.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('search', '0003_create_search_services'), + ] + + operations = [ + migrations.RemoveField( + model_name='searchcell', + name='_search_service', + ), + ] diff --git a/combo/apps/search/models.py b/combo/apps/search/models.py index 23edefd..28af4c4 100644 --- a/combo/apps/search/models.py +++ b/combo/apps/search/models.py @@ -15,44 +15,35 @@ # along with this program. If not, see . from django.conf import settings -from django.db import models from django.utils.translation import ugettext_lazy as _ from django import template from django.http import HttpResponse from django.core.exceptions import PermissionDenied from django.core.urlresolvers import reverse -from django.forms import models as model_forms, Select from django.utils.http import quote +from jsonfield import JSONField + from combo.utils import requests from combo.data.models import CellBase from combo.data.library import register_cell_class -from combo.utils import NothingInCacheException, get_templated_url +from combo.utils import get_templated_url + @register_cell_class class SearchCell(CellBase): template_name = 'combo/search-cell.html' - _search_service = models.CharField(verbose_name=_('Search Service'), max_length=64) + _search_services = JSONField(_('Search Services'), default=dict, blank=True) class Meta: verbose_name = _('Search') def is_visible(self, user=None): - if not self.search_service: + if not self.search_services: return False return super(SearchCell, self).is_visible(user=user) - def get_default_form_class(self): - search_services = [(None, _('Not configured'))] - search_services.append(('_text', _('Page Contents'))) - search_services.extend([(code, service['label']) - for code, service in settings.COMBO_SEARCH_SERVICES.items()]) - widgets = {'_search_service': Select(choices=search_services)} - return model_forms.modelform_factory(self.__class__, - fields=['_search_service'], - widgets=widgets) - @property def varname(self): if self.slug: @@ -61,10 +52,27 @@ class SearchCell(CellBase): return '' @property - def search_service(self): - if self._search_service == '_text': - return {'url': reverse('api-search') + '?q=%(q)s', 'label': _('Page Contents')} - return settings.COMBO_SEARCH_SERVICES.get(self._search_service) or {} + def search_services(self): + services = [] + for service_slug in self._search_services.get('data') or []: + if service_slug == '_text': + service = {'url': reverse('api-search') + '?q=%(q)s', 'label': _('Page Contents')} + else: + service = settings.COMBO_SEARCH_SERVICES.get(service_slug) + if service and service.get('url'): + service['slug'] = service_slug + services.append(service) + return services + + @property + def has_multiple_search_services(self): + return len(self._search_services.get('data') or []) > 1 + + @classmethod + def get_cells_by_search_service(cls, search_service): + for cell in cls.objects.all(): + if search_service in (cell._search_services.get('data') or []): + yield cell def modify_global_context(self, context, request): # if self.varname is in the query string (of the page), @@ -94,25 +102,39 @@ class SearchCell(CellBase): return extra_context @classmethod - def ajax_results_view(cls, request, cell_pk): + def ajax_results_view(cls, request, cell_pk, service_slug): cell = cls.objects.get(pk=cell_pk) if not cell.is_visible(request.user) or not cell.page.is_visible(request.user): raise PermissionDenied - query = request.GET.get('q') - if query and cell.search_service.get('url'): - url = cell.search_service.get('url') - url = get_templated_url(url) - url = url % {'q': quote(query.encode('utf-8'))} - if url.startswith('/'): - url = request.build_absolute_uri(url) - results = requests.get(url, cache_duration=0).json() + def render_response(service={}, results={'err': 0, 'data': []}): + template_names = ['combo/search-cell-results.html'] + if cell.slug: + template_names.insert(0, 'combo/cells/%s/search-cell-results.html' % cell.slug) + tmpl = template.loader.select_template(template_names) + context = {'cell': cell, 'results': results, 'search_service': service} + return HttpResponse(tmpl.render(context, request), content_type='text/html') + + for service in cell.search_services: + if service.get('slug') == service_slug: + break else: - results = {'err': 0, 'data': []} - - template_names = ['combo/search-cell-results.html'] - if cell.slug: - template_names.insert(0, 'combo/cells/%s/search-cell-results.html' % cell.slug) - tmpl = template.loader.select_template(template_names) - context= {'cell': cell, 'results': results} - return HttpResponse(tmpl.render(context, request), content_type='text/html') + return render_response() + + query = request.GET.get('q') + if not query: + return render_response(service) + + url = get_templated_url(service['url'], context={'q': query, 'search_service': service}) + url = url % {'q': quote(query.encode('utf-8'))} # if url contains %(q)s + if url.startswith('/'): + url = request.build_absolute_uri(url) + + if not url: + return render_response(service) + + kwargs = {} + kwargs['cache_duration'] = service.get('cache_duration', 0) + kwargs['remote_service'] = 'auto' if service.get('signature') else None + results = requests.get(url, **kwargs).json() + return render_response(service, results) diff --git a/combo/apps/search/templates/combo/search-cell-results.html b/combo/apps/search/templates/combo/search-cell-results.html index 2956f5d..e1a7f50 100644 --- a/combo/apps/search/templates/combo/search-cell-results.html +++ b/combo/apps/search/templates/combo/search-cell-results.html @@ -1,3 +1,5 @@ +{% if results.data %} +{% if cell.has_multiple_search_services %}

{{ search_service.label }}

{% endif %}
    {% for item in results.data %} @@ -7,3 +9,4 @@ {% endfor %}
+{% endif %} diff --git a/combo/apps/search/templates/combo/search-cell.html b/combo/apps/search/templates/combo/search-cell.html index 27f2195..243b73f 100644 --- a/combo/apps/search/templates/combo/search-cell.html +++ b/combo/apps/search/templates/combo/search-cell.html @@ -2,30 +2,36 @@ {% block cell-content %}
- {# initial_query_string pass some context to ajax call #} +
-
+{% for search_service in cell.search_services %} +
+{% endfor %}