0001-search-handle-several-search-engines-23534.patch
combo/apps/search/management/commands/update_index.py | ||
---|---|---|
30 | 30 |
dest='skip_external_links_collection') |
31 | 31 | |
32 | 32 |
def handle(self, **options): |
33 |
if SearchCell.objects.filter(_search_service='_text').count() == 0: |
|
33 |
if SearchCell.objects.filter(_search_services__contains='_text').count() == 0:
|
|
34 | 34 |
# do not index site if there's no matching search cell |
35 | 35 |
return |
36 | 36 |
if not options.get('skip_external_links_collection', False): |
combo/apps/search/migrations/0002_auto_20180720_1511.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
from __future__ import unicode_literals |
|
3 | ||
4 |
from django.db import migrations |
|
5 |
import jsonfield.fields |
|
6 | ||
7 | ||
8 |
class Migration(migrations.Migration): |
|
9 | ||
10 |
dependencies = [ |
|
11 |
('search', '0001_initial'), |
|
12 |
] |
|
13 | ||
14 |
operations = [ |
|
15 |
migrations.AddField( |
|
16 |
model_name='searchcell', |
|
17 |
name='_search_services', |
|
18 |
field=jsonfield.fields.JSONField(default=list, |
|
19 |
help_text='JSON list of search service names', |
|
20 |
verbose_name='Search Services', |
|
21 |
blank=True), |
|
22 |
), |
|
23 |
] |
combo/apps/search/migrations/0003_create_search_services.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
from __future__ import unicode_literals |
|
3 | ||
4 |
from django.db import migrations |
|
5 | ||
6 | ||
7 |
def create_search_services(apps, schema_editor): |
|
8 |
SearchCell = apps.get_model('search', 'SearchCell') |
|
9 |
for searchcell in SearchCell.objects.all(): |
|
10 |
if searchcell._search_service: |
|
11 |
searchcell._search_services = [searchcell._search_service] |
|
12 |
else: |
|
13 |
searchcell._search_services = [] |
|
14 |
searchcell.save() |
|
15 | ||
16 | ||
17 |
def create_search_service(apps, schema_editor): |
|
18 |
SearchCell = apps.get_model('search', 'SearchCell') |
|
19 |
for searchcell in SearchCell.objects.all(): |
|
20 |
if searchcell._search_services: |
|
21 |
searchcell._search_service = searchcell._search_services[0] |
|
22 |
else: |
|
23 |
searchcell._search_service = '' |
|
24 |
searchcell.save() |
|
25 | ||
26 | ||
27 |
class Migration(migrations.Migration): |
|
28 | ||
29 |
dependencies = [ |
|
30 |
('search', '0002_auto_20180720_1511'), |
|
31 |
] |
|
32 | ||
33 |
operations = [ |
|
34 |
migrations.RunPython(create_search_services, create_search_service), |
|
35 |
] |
combo/apps/search/migrations/0004_remove_searchcell__search_service.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
from __future__ import unicode_literals |
|
3 | ||
4 |
from django.db import migrations, models |
|
5 | ||
6 | ||
7 |
class Migration(migrations.Migration): |
|
8 | ||
9 |
dependencies = [ |
|
10 |
('search', '0003_create_search_services'), |
|
11 |
] |
|
12 | ||
13 |
operations = [ |
|
14 |
migrations.RemoveField( |
|
15 |
model_name='searchcell', |
|
16 |
name='_search_service', |
|
17 |
), |
|
18 |
] |
combo/apps/search/models.py | ||
---|---|---|
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 | 17 |
from django.conf import settings |
18 |
from django.db import models |
|
19 | 18 |
from django.utils.translation import ugettext_lazy as _ |
20 | 19 |
from django import template |
21 | 20 |
from django.http import HttpResponse |
22 | 21 |
from django.core.exceptions import PermissionDenied |
23 | 22 |
from django.core.urlresolvers import reverse |
24 |
from django.forms import models as model_forms, Select |
|
25 | 23 |
from django.utils.http import quote |
26 | 24 | |
25 |
from jsonfield import JSONField |
|
26 | ||
27 | 27 |
from combo.utils import requests |
28 | 28 |
from combo.data.models import CellBase |
29 | 29 |
from combo.data.library import register_cell_class |
30 |
from combo.utils import NothingInCacheException, get_templated_url |
|
30 |
from combo.utils import get_templated_url |
|
31 | ||
31 | 32 | |
32 | 33 |
@register_cell_class |
33 | 34 |
class SearchCell(CellBase): |
34 | 35 |
template_name = 'combo/search-cell.html' |
35 | 36 | |
36 |
_search_service = models.CharField(verbose_name=_('Search Service'), max_length=64) |
|
37 |
_search_services = JSONField(_('Search Services'), default=list, blank=True, |
|
38 |
help_text=_('JSON list of search service names')) |
|
37 | 39 | |
38 | 40 |
class Meta: |
39 | 41 |
verbose_name = _('Search') |
40 | 42 | |
41 | 43 |
def is_visible(self, user=None): |
42 |
if not self.search_service: |
|
44 |
if not self.search_services:
|
|
43 | 45 |
return False |
44 | 46 |
return super(SearchCell, self).is_visible(user=user) |
45 | 47 | |
46 |
def get_default_form_class(self): |
|
47 |
search_services = [(None, _('Not configured'))] |
|
48 |
search_services.append(('_text', _('Page Contents'))) |
|
49 |
search_services.extend([(code, service['label']) |
|
50 |
for code, service in settings.COMBO_SEARCH_SERVICES.items()]) |
|
51 |
widgets = {'_search_service': Select(choices=search_services)} |
|
52 |
return model_forms.modelform_factory(self.__class__, |
|
53 |
fields=['_search_service'], |
|
54 |
widgets=widgets) |
|
55 | ||
56 | 48 |
@property |
57 | 49 |
def varname(self): |
58 | 50 |
if self.slug: |
... | ... | |
61 | 53 |
return '' |
62 | 54 | |
63 | 55 |
@property |
64 |
def search_service(self): |
|
65 |
if self._search_service == '_text': |
|
66 |
return {'url': reverse('api-search') + '?q=%(q)s', 'label': _('Page Contents')} |
|
67 |
return settings.COMBO_SEARCH_SERVICES.get(self._search_service) or {} |
|
56 |
def search_services(self): |
|
57 |
services = [] |
|
58 |
for service_slug in self._search_services or []: |
|
59 |
if service_slug == '_text': |
|
60 |
service = {'url': reverse('api-search') + '?q=%(q)s', 'label': _('Page Contents')} |
|
61 |
else: |
|
62 |
service = settings.COMBO_SEARCH_SERVICES.get(service_slug) |
|
63 |
if service and service.get('url'): |
|
64 |
service['slug'] = service_slug |
|
65 |
services.append(service) |
|
66 |
return services |
|
68 | 67 | |
69 | 68 |
def modify_global_context(self, context, request): |
70 | 69 |
# if self.varname is in the query string (of the page), |
... | ... | |
94 | 93 |
return extra_context |
95 | 94 | |
96 | 95 |
@classmethod |
97 |
def ajax_results_view(cls, request, cell_pk): |
|
96 |
def ajax_results_view(cls, request, cell_pk, service_slug):
|
|
98 | 97 |
cell = cls.objects.get(pk=cell_pk) |
99 | 98 |
if not cell.is_visible(request.user) or not cell.page.is_visible(request.user): |
100 | 99 |
raise PermissionDenied |
101 | 100 | |
102 |
query = request.GET.get('q') |
|
103 |
if query and cell.search_service.get('url'): |
|
104 |
url = cell.search_service.get('url') |
|
105 |
url = get_templated_url(url) |
|
106 |
url = url % {'q': quote(query.encode('utf-8'))} |
|
107 |
if url.startswith('/'): |
|
108 |
url = request.build_absolute_uri(url) |
|
109 |
results = requests.get(url, cache_duration=0).json() |
|
101 |
def render_response(service={}, results={'err': 0, 'data': []}): |
|
102 |
template_names = ['combo/search-cell-results.html'] |
|
103 |
if cell.slug: |
|
104 |
template_names.insert(0, 'combo/cells/%s/search-cell-results.html' % cell.slug) |
|
105 |
tmpl = template.loader.select_template(template_names) |
|
106 |
context = {'cell': cell, 'results': results, 'search_service': service} |
|
107 |
return HttpResponse(tmpl.render(context, request), content_type='text/html') |
|
108 | ||
109 |
for service in cell.search_services: |
|
110 |
if service.get('slug') == service_slug: |
|
111 |
break |
|
110 | 112 |
else: |
111 |
results = {'err': 0, 'data': []} |
|
112 | ||
113 |
template_names = ['combo/search-cell-results.html'] |
|
114 |
if cell.slug: |
|
115 |
template_names.insert(0, 'combo/cells/%s/search-cell-results.html' % cell.slug) |
|
116 |
tmpl = template.loader.select_template(template_names) |
|
117 |
context= {'cell': cell, 'results': results} |
|
118 |
return HttpResponse(tmpl.render(context, request), content_type='text/html') |
|
113 |
return render_response() |
|
114 | ||
115 |
query = request.GET.get('q') |
|
116 |
if not query: |
|
117 |
return render_response(service) |
|
118 | ||
119 |
url = get_templated_url(service['url'], context={'q': query, 'search_service': service}) |
|
120 |
url = url % {'q': quote(query.encode('utf-8'))} # if url contains %(q)s |
|
121 |
if url.startswith('/'): |
|
122 |
url = request.build_absolute_uri(url) |
|
123 | ||
124 |
if not url: |
|
125 |
return render_response(service) |
|
126 | ||
127 |
results = requests.get(url, cache_duration=0).json() |
|
128 |
return render_response(service, results) |
combo/apps/search/templates/combo/search-cell-results.html | ||
---|---|---|
1 |
{% if results.data %} |
|
2 |
<p class="search-service-label">{{ search_service.label }}</p> |
|
1 | 3 |
<div class="links-list"> |
2 | 4 |
<ul> |
3 | 5 |
{% for item in results.data %} |
... | ... | |
7 | 9 |
{% endfor %} |
8 | 10 |
</ul> |
9 | 11 |
</div> |
12 |
{% endif %} |
combo/apps/search/templates/combo/search-cell.html | ||
---|---|---|
2 | 2 |
{% block cell-content %} |
3 | 3 | |
4 | 4 |
<form id="combo-search-form-{{ cell.pk }}" class="combo-search-form"> |
5 |
<input type="text" name="q" autocomplete="off" id="combo-search-input-{{ cell.pk }}" class="combo-search-input" |
|
6 |
data-autocomplete-json="{% url 'combo-search-ajax-results' cell_pk=cell.pk %}{% if initial_query_string %}?{{ initial_query_string }}{% endif %}" />{# initial_query_string pass some context to ajax call #} |
|
5 |
<input type="text" name="q" autocomplete="off" id="combo-search-input-{{ cell.pk }}" class="combo-search-input" /> |
|
7 | 6 |
<button class="submit-button">{% trans "Search" %}</button> |
8 | 7 |
</form> |
9 | 8 | |
10 |
<div id="combo-search-results-{{ cell.pk }}" class="combo-search-results"></div> |
|
9 |
{% for search_service in cell.search_services %} |
|
10 |
<div id="combo-search-results-{{ cell.pk }}-{{ forloop.counter }}" class="combo-search-results combo-search-results-{{ search_service.slug }}"></div> |
|
11 |
{% endfor %} |
|
11 | 12 | |
12 | 13 |
<script> |
13 | 14 |
$(function() { |
14 | 15 |
var combo_search_timeout_{{ cell.pk }}; |
15 | 16 |
var combo_search_form_{{ cell.pk }} = $('#combo-search-form-{{ cell.pk }}'); |
16 | 17 |
var combo_search_input_{{ cell.pk }} = $('#combo-search-input-{{ cell.pk }}'); |
17 |
var combo_search_results_{{ cell.pk }} = $('#combo-search-results-{{ cell.pk }}'); |
|
18 |
var xhr = null; |
|
18 |
{% for search_service in cell.search_services %} |
|
19 |
var combo_search_results_{{ cell.pk }}_{{ forloop.counter }} = $('#combo-search-results-{{ cell.pk }}-{{ forloop.counter }}'); |
|
20 |
var xhr_{{ forloop.counter }} = null; |
|
21 |
var url_{{ forloop.counter }} = '{% url 'combo-search-ajax-results' cell_pk=cell.pk service_slug=search_service.slug %}{% if initial_query_string %}?{{ intial_query_string }}{% endif %}'; |
|
22 |
{% endfor %} |
|
19 | 23 | |
20 | 24 |
function combo_search_update_{{ cell.pk }}() { |
21 |
if (xhr) xhr.abort(); |
|
22 |
xhr = $.get(combo_search_input_{{ cell.pk }}.data('autocomplete-json'), |
|
25 |
{% for search_service in cell.search_services %} |
|
26 |
if (xhr_{{ forloop.counter }}) xhr_{{ forloop.counter }}.abort(); |
|
27 |
xhr_{{ forloop.counter }} = $.get(url_{{ forloop.counter }}, |
|
23 | 28 |
{'q': combo_search_input_{{ cell.pk }}.val()}, |
24 | 29 |
function (response) { |
25 |
xhr = null; |
|
26 |
combo_search_results_{{ cell.pk }}.html(response); |
|
30 |
xhr_{{ forloop.counter }} = null;
|
|
31 |
combo_search_results_{{ cell.pk }}_{{ forloop.counter }}.html(response);
|
|
27 | 32 |
} |
28 | 33 |
); |
34 |
{% endfor %} |
|
29 | 35 |
}; |
30 | 36 | |
31 | 37 |
combo_search_input_{{ cell.pk }}.on('paste keyup', function() { |
combo/apps/search/urls.py | ||
---|---|---|
19 | 19 |
from .models import SearchCell |
20 | 20 | |
21 | 21 |
urlpatterns = [ |
22 |
url(r'^ajax/search/(?P<cell_pk>\w+)/$', SearchCell.ajax_results_view, |
|
22 |
url(r'^ajax/search/(?P<cell_pk>\w+)/(?P<service_slug>\w+)/$', SearchCell.ajax_results_view,
|
|
23 | 23 |
name='combo-search-ajax-results'), |
24 | 24 |
] |
combo/public/views.py | ||
---|---|---|
436 | 436 | |
437 | 437 | |
438 | 438 |
def api_search(request): |
439 |
for cell in SearchCell.objects.filter(_search_service='_text'): |
|
439 |
for cell in SearchCell.objects.filter(_search_services__contains='_text'):
|
|
440 | 440 |
if not cell.is_visible(request.user): |
441 | 441 |
continue |
442 | 442 |
break |
tests/test_search.py | ||
---|---|---|
49 | 49 |
page.save() |
50 | 50 | |
51 | 51 |
cell = SearchCell(page=page, placeholder='content', order=0) |
52 |
cell._search_service = 'search1'
|
|
52 |
cell._search_services = ['search1']
|
|
53 | 53 |
cell.save() |
54 | 54 | |
55 | 55 |
resp = cell.render({}) |
... | ... | |
67 | 67 |
mock_json = mock.Mock() |
68 | 68 |
mock_json.json.return_value = response |
69 | 69 |
requests_get.return_value = mock_json |
70 |
resp = client.get('/ajax/search/%s/?q=foo' % cell.pk, status=200) |
|
70 |
resp = client.get('/ajax/search/%s/search1/?q=foo' % cell.pk, status=200)
|
|
71 | 71 |
assert requests_get.call_args[0][0] == 'http://www.example.net/search/?q=foo' |
72 | 72 |
assert '<li>' not in resp.content |
73 | 73 | |
74 |
resp = client.get('/ajax/search/%s/?q=foo%%23bar' % cell.pk, status=200) |
|
74 |
resp = client.get('/ajax/search/%s/search1/?q=foo%%23bar' % cell.pk, status=200)
|
|
75 | 75 |
assert requests_get.call_args[0][0] == 'http://www.example.net/search/?q=foo%23bar' |
76 | 76 |
assert '<li>' not in resp.content |
77 | 77 | |
78 | 78 |
response['data'] = [{'url': 'http://test', 'text': 'barbarbar'}] |
79 |
resp = client.get('/ajax/search/%s/?q=foo' % cell.pk, status=200) |
|
79 |
resp = client.get('/ajax/search/%s/search1/?q=foo' % cell.pk, status=200)
|
|
80 | 80 |
assert resp.content.count('<li>') == 1 |
81 | 81 |
assert '<li><a href="http://test">barbarbar</a>' in resp.content |
82 | 82 | |
83 | 83 |
response['data'] = [{'url': 'http://test', 'text': 'barbarbar', |
84 | 84 |
'description': 'this is <b>html</b>'}] |
85 |
resp = client.get('/ajax/search/%s/?q=foo' % cell.pk, status=200) |
|
85 |
resp = client.get('/ajax/search/%s/search1/?q=foo' % cell.pk, status=200)
|
|
86 | 86 |
assert resp.content.count('<li>') == 1 |
87 | 87 |
assert '<li><a href="http://test">barbarbar</a>' in resp.content |
88 | 88 |
assert 'this is <b>html</b>' in resp.content |
89 | 89 | |
90 | 90 |
with override_settings(TEMPLATE_VARS=TEMPLATE_VARS): |
91 |
cell._search_service = 'search_tmpl'
|
|
91 |
cell._search_services = ['search_tmpl']
|
|
92 | 92 |
cell.save() |
93 | 93 |
with mock.patch('combo.apps.search.models.requests.get') as requests_get: |
94 | 94 |
response = {'err': 0, 'data': []} |
95 | 95 |
mock_json = mock.Mock() |
96 | 96 |
mock_json.json.return_value = response |
97 | 97 |
requests_get.return_value = mock_json |
98 |
resp = client.get('/ajax/search/%s/?q=foo' % cell.pk, status=200) |
|
98 |
resp = client.get('/ajax/search/%s/search_tmpl/?q=foo' % cell.pk, status=200)
|
|
99 | 99 |
assert requests_get.call_args[0][0] == 'http://search.example.net/?q=foo' |
100 | 100 | |
101 | 101 |
# TEMPLATE_VARS are accessible in template |
... | ... | |
104 | 104 |
templates_settings = [settings.TEMPLATES[0].copy()] |
105 | 105 |
templates_settings[0]['DIRS'] = ['%s/templates-1' % os.path.abspath(os.path.dirname(__file__))] |
106 | 106 |
with override_settings(TEMPLATES=templates_settings): |
107 |
resp = client.get('/ajax/search/%s/?q=bar' % cell.pk, status=200) |
|
107 |
resp = client.get('/ajax/search/%s/search_tmpl/?q=bar' % cell.pk, status=200)
|
|
108 | 108 |
assert requests_get.call_args[0][0] == 'http://search.example.net/?q=bar' |
109 | 109 |
assert 'searchfoo results.data=[]' in resp.content |
110 | 110 |
assert 'search_url=http://search.example.net/' in resp.content |
... | ... | |
114 | 114 |
page = Page(title='Search', slug='search_page', template_name='standard') |
115 | 115 |
page.save() |
116 | 116 |
cell = SearchCell(page=page, placeholder='content', order=0) |
117 |
cell._search_service = 'search1'
|
|
117 |
cell._search_services = ['search1']
|
|
118 | 118 |
cell.save() |
119 | 119 |
assert cell.varname == '' |
120 | 120 | |
... | ... | |
144 | 144 |
cell = SearchCell(page=page, order=0) |
145 | 145 |
assert not cell.is_visible() |
146 | 146 | |
147 |
cell._search_service = '_text'
|
|
147 |
cell._search_services = ['_text']
|
|
148 | 148 |
assert cell.is_visible() |
149 | 149 | |
150 | 150 |
def test_search_contents(): |
... | ... | |
219 | 219 | |
220 | 220 |
resp = app.get('/api/search/?q=foobar', status=404) |
221 | 221 | |
222 |
cell = SearchCell(page=page, _search_service='_text', order=0)
|
|
222 |
cell = SearchCell(page=page, _search_services=['_text'], order=0)
|
|
223 | 223 |
cell.save() |
224 | 224 | |
225 | 225 |
resp = app.get('/api/search/?q=foobar', status=200) |
... | ... | |
243 | 243 |
page = Page(title='example page', slug='example-page') |
244 | 244 |
page.save() |
245 | 245 | |
246 |
cell = SearchCell(page=page, _search_service='_text', order=0)
|
|
246 |
cell = SearchCell(page=page, _search_services=['_text'], order=0)
|
|
247 | 247 |
cell.save() |
248 | 248 | |
249 | 249 |
call_command('update_index') |
250 |
- |