0002-seo-add-robots_txt-views-20263.patch
hobo/environment/utils.py | ||
---|---|---|
78 | 78 |
name=name, |
79 | 79 |
defaults={ |
80 | 80 |
'auto': True, |
81 | 81 |
'value': settings.VARIABLE_SETTINGS_DEFAULTS.get(name) or '' |
82 | 82 |
}) |
83 | 83 |
return variable |
84 | 84 | |
85 | 85 | |
86 |
def set_variable(name, value): |
|
87 |
from .models import Variable |
|
88 |
variable, created = Variable.objects.get_or_create( |
|
89 |
name=name, defaults={'auto': True}) |
|
90 |
variable.value = value |
|
91 |
variable.save() |
|
92 | ||
93 | ||
86 | 94 |
def create_base_url(hostname, service): |
87 | 95 |
""" |
88 | 96 |
Distinguish mutualised domains (matching a "-" in the first part of the netloc) |
89 | 97 |
from the normal scenario with a dedicated parent domain. |
90 | 98 |
""" |
91 | 99 |
ph = urlparse(hostname) |
92 | 100 |
parts = ph.netloc.split('.') |
93 | 101 |
if '-' in parts[0]: |
hobo/seo/forms.py | ||
---|---|---|
1 |
# hobo - portal to configure and deploy applications |
|
2 |
# Copyright (C) 2019 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software: you can redistribute it and/or modify it |
|
5 |
# under the terms of the GNU Affero General Public License as published |
|
6 |
# by the Free Software Foundation, either version 3 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU Affero General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU Affero General Public License |
|
15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
from django import forms |
|
18 |
from django.utils.translation import ugettext_lazy as _ |
|
19 | ||
20 | ||
21 |
class RobotsTxtForm(forms.Form): |
|
22 |
content = forms.CharField( |
|
23 |
label=_('Content of robots.txt file'), |
|
24 |
required=False, |
|
25 |
widget=forms.Textarea) |
|
26 | ||
27 | ||
28 |
class EnableForm(forms.Form): |
|
29 |
pass |
hobo/seo/templates/hobo/robots_txt.html | ||
---|---|---|
1 |
{% extends "hobo/seo_home.html" %} |
|
2 |
{% load i18n %} |
|
3 | ||
4 |
{% block appbar %} |
|
5 |
<h2>{% trans 'Robots Exclusion Protocol' %}</h2> |
|
6 |
{% endblock %} |
|
7 | ||
8 |
{% block content %} |
|
9 |
<form method="post"> |
|
10 |
{% csrf_token %} |
|
11 |
{{ form.as_p }} |
|
12 |
<p> |
|
13 | ||
14 |
<div class="buttons"> |
|
15 |
<button class="submit-button">{% trans "Save" %}</button> |
|
16 |
<a class="cancel" href="{% url 'seo-home' %}">{% trans "Cancel" %}</a> |
|
17 |
</div> |
|
18 |
</form> |
|
19 | ||
20 |
{% endblock %} |
hobo/seo/templates/hobo/seo_home.html | ||
---|---|---|
1 |
{% extends "hobo/base.html" %} |
|
2 |
{% load i18n %} |
|
3 | ||
4 |
{% block breadcrumb %} |
|
5 |
{{ block.super }} |
|
6 |
<a href="{% url 'seo-home' %}">Se</a> |
|
7 |
{% endblock %} |
|
8 | ||
9 |
{% block appbar %} |
|
10 |
<h2>{% trans 'Indexing Settings' %}</h2> |
|
11 |
<span class="actions"> |
|
12 |
<a |
|
13 |
rel="popup" href="{% url 'robots-customize' %}"> |
|
14 |
{% if mode == 'customize' %}{% trans "Change custom settings" %} |
|
15 |
{% else %}{% trans 'Use custom settings' %}{% endif %} |
|
16 |
</a> |
|
17 |
<a |
|
18 |
{% if mode == 'disallow' %}class="disabled"{% endif %} |
|
19 |
href="{% url 'robots-disallow' %}"> |
|
20 |
{% trans 'Do not allow indexing' %} |
|
21 |
</a> |
|
22 |
<a |
|
23 |
{% if mode == 'allow' %}class="disabled"{% endif %} |
|
24 |
href="{% url 'robots-allow' %}"> |
|
25 |
{% trans 'Allow indexing' %} |
|
26 |
</a> |
|
27 |
</span> |
|
28 |
{% endblock %} |
|
29 | ||
30 |
{% block content %} |
|
31 |
<div class="infonotice"> |
|
32 |
{% blocktrans %} |
|
33 |
Robots are programs that automatically traverse web sites. |
|
34 |
Web site owners can use a <tt>robots.txt</tt> file to give |
|
35 |
instructions about their site. |
|
36 |
{% endblocktrans %} |
|
37 |
</div> |
|
38 | ||
39 |
{% if mode == 'allow' %} |
|
40 |
<div class="successnotice"> |
|
41 |
{% trans "Web Robots are currently allowed." %} |
|
42 |
</div> |
|
43 |
{% endif %} |
|
44 | ||
45 |
{% if mode == 'disallow' %} |
|
46 |
<div class="warningnotice"> |
|
47 |
{% trans "Web Robots are currently not allowed." %} |
|
48 |
</div> |
|
49 |
{% endif %} |
|
50 | ||
51 |
{% if mode == 'customize' or mode == 'disallow' %} |
|
52 |
<div class="section padded"> |
|
53 |
<span>{% trans "Contents of robots.txt file:" %}</span> |
|
54 |
<pre>{{ robots_txt }}</pre> |
|
55 |
</div> |
|
56 |
{% endif %} |
|
57 | ||
58 |
{% endblock %} |
hobo/seo/urls.py | ||
---|---|---|
1 |
# hobo - portal to configure and deploy applications |
|
2 |
# Copyright (C) 2019 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software: you can redistribute it and/or modify it |
|
5 |
# under the terms of the GNU Affero General Public License as published |
|
6 |
# by the Free Software Foundation, either version 3 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU Affero General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU Affero General Public License |
|
15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
from django.conf.urls import url |
|
18 | ||
19 |
from . import views |
|
20 | ||
21 |
urlpatterns = [ |
|
22 |
url(r'^$', views.home, name='seo-home'), |
|
23 |
url(r'^allow$', views.allow, name='robots-allow'), |
|
24 |
url(r'^disallow$', views.disallow, name='robots-disallow'), |
|
25 |
url(r'^customize$', views.customize, name='robots-customize'), |
|
26 |
] |
hobo/seo/views.py | ||
---|---|---|
1 |
# hobo - portal to configure and deploy applications |
|
2 |
# Copyright (C) 2020 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software: you can redistribute it and/or modify it |
|
5 |
# under the terms of the GNU Affero General Public License as published |
|
6 |
# by the Free Software Foundation, either version 3 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU Affero General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU Affero General Public License |
|
15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
from django.core.urlresolvers import reverse_lazy |
|
18 |
from django.http import HttpResponseRedirect |
|
19 |
from django.views.generic import FormView, TemplateView |
|
20 | ||
21 |
from hobo.environment.utils import get_variable, set_variable |
|
22 |
from .forms import RobotsTxtForm |
|
23 | ||
24 | ||
25 |
ALLOW = "" |
|
26 |
DISALLOW = """User-agent: * |
|
27 |
Disallow: /""" |
|
28 | ||
29 | ||
30 |
def get_mode(content): |
|
31 |
content = content.strip().replace('\r', '') |
|
32 |
if content == ALLOW: |
|
33 |
return 'allow' |
|
34 |
if content == DISALLOW: |
|
35 |
return 'disallow' |
|
36 |
return 'customize' |
|
37 | ||
38 | ||
39 |
class HomeView(TemplateView): |
|
40 |
template_name = 'hobo/seo_home.html' |
|
41 |
success_url = reverse_lazy('seo-home') |
|
42 | ||
43 |
def get_context_data(self, **kwargs): |
|
44 |
context = super(HomeView, self).get_context_data(**kwargs) |
|
45 |
context['robots_txt'] = get_variable('robots_txt').value |
|
46 |
context['mode'] = get_mode(context['robots_txt']) |
|
47 |
return context |
|
48 | ||
49 |
home = HomeView.as_view() |
|
50 | ||
51 | ||
52 |
def allow(request): |
|
53 |
set_variable('robots_txt', ALLOW) |
|
54 |
return HttpResponseRedirect(reverse_lazy('seo-home')) |
|
55 | ||
56 | ||
57 |
def disallow(request): |
|
58 |
set_variable('robots_txt', DISALLOW) |
|
59 |
return HttpResponseRedirect(reverse_lazy('seo-home')) |
|
60 | ||
61 | ||
62 |
class CustomizeView(FormView): |
|
63 |
template_name = 'hobo/robots_txt.html' |
|
64 |
form_class = RobotsTxtForm |
|
65 |
success_url = reverse_lazy('seo-home') |
|
66 | ||
67 |
def get_initial(self): |
|
68 |
initial = super(CustomizeView, self).get_initial() |
|
69 |
initial['content'] = get_variable('robots_txt').value |
|
70 |
return initial |
|
71 | ||
72 |
def form_valid(self, form): |
|
73 |
set_variable('robots_txt', form.cleaned_data['content']) |
|
74 |
return super(CustomizeView, self).form_valid(form) |
|
75 | ||
76 |
customize = CustomizeView.as_view() |
hobo/settings.py | ||
---|---|---|
41 | 41 |
'rest_framework', |
42 | 42 |
'mellon', |
43 | 43 |
'gadjo', |
44 | 44 |
'hobo.debug', |
45 | 45 |
'hobo.environment', |
46 | 46 |
'hobo.franceconnect', |
47 | 47 |
'hobo.matomo', |
48 | 48 |
'hobo.profile', |
49 |
'hobo.seo', |
|
49 | 50 |
'hobo.theme', |
50 | 51 |
'hobo.emails', |
51 | 52 |
'hobo.deploy', |
52 | 53 |
) |
53 | 54 | |
54 | 55 |
MIDDLEWARE_CLASSES = ( |
55 | 56 |
'hobo.middleware.xforwardedfor.XForwardedForMiddleware', |
56 | 57 |
'django.contrib.sessions.middleware.SessionMiddleware', |
hobo/templates/hobo/home.html | ||
---|---|---|
10 | 10 |
<li><a href="{% url 'profile-home' %}">{% trans 'User Profile' %}</a></li> |
11 | 11 |
{% endif %} |
12 | 12 |
<li><a href="{% url 'theme-home' %}">{% trans 'Theme' %}</a></li> |
13 | 13 |
<li><a href="{% url 'emails-home' %}">{% trans 'Emails' %}</a></li> |
14 | 14 |
{% if has_authentic %} |
15 | 15 |
<li><a href="{% url 'franceconnect-home' %}">FranceConnect</a></li> |
16 | 16 |
{% endif %} |
17 | 17 |
<li><a href="{% url 'matomo-home' %}">{% trans 'User tracking' %}</a></li> |
18 |
<li><a href="{% url 'seo-home' %}">{% trans 'Indexing' %}</a></li> |
|
18 | 19 |
<li><a href="{% url 'environment-home' %}">{% trans 'Services' %}</a></li> |
19 | 20 |
<li><a href="{% url 'environment-variables' %}">{% trans 'Variables' %}</a></li> |
20 | 21 |
<li><a href="{% url 'debug-home' %}">{% trans 'Debugging' %}</a></li> |
21 | 22 |
</ul> |
22 | 23 |
</span> |
23 | 24 |
{% endblock %} |
24 | 25 | |
25 | 26 |
{% block content %} |
hobo/urls.py | ||
---|---|---|
23 | 23 |
from .urls_utils import decorated_includes |
24 | 24 |
from .environment.urls import urlpatterns as environment_urls |
25 | 25 |
from .franceconnect.urls import urlpatterns as franceconnect_urls |
26 | 26 |
from .matomo.urls import urlpatterns as matomo_urls |
27 | 27 |
from .profile.urls import urlpatterns as profile_urls |
28 | 28 |
from .theme.urls import urlpatterns as theme_urls |
29 | 29 |
from .emails.urls import urlpatterns as emails_urls |
30 | 30 |
from .debug.urls import urlpatterns as debug_urls |
31 |
from .seo.urls import urlpatterns as seo_urls |
|
31 | 32 | |
32 | 33 |
admin.autodiscover() |
33 | 34 | |
34 | 35 |
urlpatterns = [ |
35 | 36 |
url(r'^$', home, name='home'), |
36 | 37 |
url(r'^sites/', decorated_includes(admin_required, include(environment_urls))), |
37 | 38 |
url(r'^profile/', decorated_includes(admin_required, include(profile_urls))), |
38 | 39 |
url(r'^franceconnect/', decorated_includes(admin_required, include(franceconnect_urls))), |
39 | 40 |
url(r'^visits-tracking/', decorated_includes(admin_required, include(matomo_urls))), |
40 | 41 |
url(r'^theme/', decorated_includes(admin_required, include(theme_urls))), |
41 | 42 |
url(r'^emails/', decorated_includes(admin_required, include(emails_urls))), |
43 |
url(r'^seo/', decorated_includes(admin_required, include(seo_urls))), |
|
42 | 44 |
url(r'^debug/', decorated_includes(admin_required, include(debug_urls))), |
43 | 45 |
url(r'^api/health/$', health_json, name='health-json'), |
44 | 46 |
url(r'^menu.json$', menu_json, name='menu_json'), |
45 | 47 |
url(r'^hobos.json$', hobo), |
46 | 48 |
url(r'^admin/', admin.site.urls), |
47 | 49 |
] |
48 | 50 | |
49 | 51 |
# add authentication patterns |
tests/test_seo.py | ||
---|---|---|
13 | 13 |
# |
14 | 14 |
# You should have received a copy of the GNU Affero General Public License |
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>.import pytest |
16 | 16 | |
17 | 17 |
import pytest |
18 | 18 | |
19 | 19 |
from django.test import override_settings |
20 | 20 | |
21 |
from hobo.environment.utils import get_variable |
|
22 |
from hobo.seo.views import ALLOW, DISALLOW |
|
23 | ||
24 |
from test_manager import login |
|
25 | ||
21 | 26 | |
22 | 27 |
pytestmark = pytest.mark.django_db |
23 | 28 | |
29 | ||
24 | 30 |
def test_middleware(app): |
25 | 31 |
with override_settings(TEMPLATE_VARS={'robots_txt': 'xxx'}): |
26 | 32 |
resp = app.get('/robots.txt', status=200) |
27 | 33 |
assert resp.text == 'xxx' |
28 | 34 | |
29 | 35 |
with override_settings(): |
30 | 36 |
resp = app.get('/robots.txt', status=404) |
37 | ||
38 | ||
39 |
def test_unlogged_access(app): |
|
40 |
# connect while not being logged in |
|
41 |
resp = app.get('/seo/', status=302) |
|
42 |
assert resp.location.endswith('/login/?next=/seo/') |
|
43 | ||
44 | ||
45 |
def test_home(app, admin_user): |
|
46 |
login(app) |
|
47 |
resp = app.get('/seo/', status=200) |
|
48 |
assert [ |
|
49 |
x['href'] for x in resp.html.find('span', {'class': 'actions'}).find_all('a') |
|
50 |
] == ['/seo/customize', '/seo/disallow', '/seo/allow'] |
|
51 | ||
52 |
variable = get_variable('robots_txt') |
|
53 |
variable.value = ALLOW |
|
54 |
variable.save() |
|
55 |
resp = app.get('/seo/', status=200) |
|
56 |
assert 'disabled' in resp.html.find('a', {'href': '/seo/allow'})['class'] |
|
57 | ||
58 |
variable = get_variable('robots_txt') |
|
59 |
variable.value = DISALLOW |
|
60 |
variable.save() |
|
61 |
resp = app.get('/seo/', status=200) |
|
62 |
assert 'disabled' in resp.html.find('a', {'href': '/seo/disallow'})['class'] |
|
63 |
assert not resp.html.button |
|
64 | ||
65 | ||
66 |
def test_allow(app, admin_user): |
|
67 |
login(app) |
|
68 |
variable = get_variable('robots_txt') |
|
69 |
variable.value = "some content" |
|
70 |
variable.save() |
|
71 |
resp = app.get('/seo/allow', status=302) |
|
72 |
assert resp.location.endswith('/seo/') |
|
73 |
assert get_variable('robots_txt').value == ALLOW |
|
74 | ||
75 | ||
76 |
def test_disallow(app, admin_user): |
|
77 |
login(app) |
|
78 |
variable = get_variable('robots_txt') |
|
79 |
variable.value = "some content" |
|
80 |
variable.save() |
|
81 |
resp = app.get('/seo/disallow', status=302) |
|
82 |
assert resp.location.endswith('/seo/') |
|
83 |
assert get_variable('robots_txt').value == DISALLOW |
|
84 |
resp = resp.follow() |
|
85 |
assert resp.html.pre.text == DISALLOW |
|
86 | ||
87 | ||
88 |
def test_custom(app, admin_user): |
|
89 |
login(app) |
|
90 |
variable = get_variable('robots_txt') |
|
91 |
variable.value = "some content" |
|
92 |
variable.save() |
|
93 |
resp = app.get('/seo/customize', status=200) |
|
94 |
assert resp.html.textarea['name'] == 'content' |
|
95 |
assert resp.html.textarea.text.strip() == 'some content' |
|
96 |
resp.form['content'] = 'some new content' |
|
97 |
resp = resp.form.submit() |
|
98 |
assert get_variable('robots_txt').value == 'some new content' |
|
99 |
resp = resp.follow() |
|
100 |
assert resp.html.pre.text == 'some new content' |
|
31 |
- |