From 9f7e0d325866fe583526b55de35974af43ad29f7 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Tue, 7 Apr 2020 18:46:48 +0200
Subject: [PATCH 2/2] seo: add robots views (#20263)

---
 hobo/seo/__init__.py                          |  0
 hobo/seo/forms.py                             | 29 ++++++
 hobo/seo/templates/hobo/robots_customize.html | 20 ++++
 hobo/seo/templates/hobo/seo_home.html         | 66 +++++++++++++
 hobo/seo/urls.py                              | 26 +++++
 hobo/seo/views.py                             | 99 +++++++++++++++++++
 hobo/settings.py                              |  1 +
 hobo/templates/hobo/home.html                 |  1 +
 hobo/urls.py                                  |  2 +
 tests/test_seo.py                             | 71 +++++++++++++
 10 files changed, 315 insertions(+)
 create mode 100644 hobo/seo/__init__.py
 create mode 100644 hobo/seo/forms.py
 create mode 100644 hobo/seo/templates/hobo/robots_customize.html
 create mode 100644 hobo/seo/templates/hobo/seo_home.html
 create mode 100644 hobo/seo/urls.py
 create mode 100644 hobo/seo/views.py

diff --git a/hobo/seo/__init__.py b/hobo/seo/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/hobo/seo/forms.py b/hobo/seo/forms.py
new file mode 100644
index 0000000..7c91a61
--- /dev/null
+++ b/hobo/seo/forms.py
@@ -0,0 +1,29 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2019  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django import forms
+from django.utils.translation import ugettext_lazy as _
+
+
+class SettingsForm(forms.Form):
+    content = forms.CharField(
+        label=_('content of /robots.txt file'),
+        required=False,
+        widget=forms.Textarea)
+
+
+class EnableForm(forms.Form):
+    pass
diff --git a/hobo/seo/templates/hobo/robots_customize.html b/hobo/seo/templates/hobo/robots_customize.html
new file mode 100644
index 0000000..596f6ca
--- /dev/null
+++ b/hobo/seo/templates/hobo/robots_customize.html
@@ -0,0 +1,20 @@
+{% extends "hobo/seo_home.html" %}
+{% load i18n %}
+
+{% block appbar %}
+  <h2>{% trans 'Robots Exclusion Protocol' %}</h2>
+{% endblock %}
+
+{% block content %}
+<form method="post">
+  {% csrf_token %}
+  {{ form.as_p }}
+<p>
+
+<div class="buttons">
+<button class="submit-button">{% trans "Enable" %}</button>
+<a class="cancel" href="{% url 'seo-home' %}">{% trans "Cancel" %}</a>
+</div>
+</form>
+
+{% endblock %}
diff --git a/hobo/seo/templates/hobo/seo_home.html b/hobo/seo/templates/hobo/seo_home.html
new file mode 100644
index 0000000..9bf6745
--- /dev/null
+++ b/hobo/seo/templates/hobo/seo_home.html
@@ -0,0 +1,66 @@
+{% extends "hobo/base.html" %}
+{% load i18n %}
+
+{% block breadcrumb %}
+{{ block.super }}
+<a href="{% url 'seo-home' %}">Se</a>
+{% endblock %}
+
+{% block appbar %}
+<h2>{% trans 'Search Engine Optimization' %}</h2>
+<span class="actions">
+  <a
+    {% if mode == 'customize' %}class="disabled"{% endif %}
+    rel="popup" href="{% url 'robots-customize' %}">
+    {% trans 'Customize' %}
+  </a>
+  <a
+    {% if mode == 'disallow' %}class="disabled"{% endif %}
+    href="{% url 'robots-disallow' %}">
+    {% trans 'Do not allow indexation' %}
+  </a>
+  <a
+    {% if mode == 'allow' %}class="disabled"{% endif %}
+    href="{% url 'robots-allow' %}">
+    {% trans 'Allow indexation' %}
+  </a>
+</span>
+{% endblock %}
+
+{% block content %}
+{% if mode == 'allow' %}
+<div class="infonotice">
+  <p>
+    {% blocktrans %}
+    Web Robots (also called "Wanderers" or "Spiders") are Web client programs that automatically traverse the Web's hypertext structure by retrieving a document, and recursively retrieving all documents that are referenced.
+    Web site owners use the /robots.txt file to give instructions about their site to web robots; this is called The Robots Exclusion Protocol.
+    {% endblocktrans %}
+  </p>
+</div>
+{% endif %}
+
+{% if mode == 'allow' %}
+<div class="warningnotice">
+  {% trans "Web Robots are currently allowed." %}
+</div>
+{% endif %}
+
+{% if mode == 'disallow' %}
+<div class="successnotice">
+  {% trans "Web Robots are currently not allowed." %}
+</div>
+{% endif %}
+
+{% if mode == 'customize' or mode == 'disallow' %}
+<form method="post">
+  {% csrf_token %}
+  {{ form.as_p }}
+
+  {% if mode == 'customize' %}
+  <div class="buttons">
+    <button class="submit-button">{% trans "Save" %}</button>
+  </div>
+  {% endif %}
+</form>
+{% endif %}
+{% endblock %}
diff --git a/hobo/seo/urls.py b/hobo/seo/urls.py
new file mode 100644
index 0000000..d0884ef
--- /dev/null
+++ b/hobo/seo/urls.py
@@ -0,0 +1,26 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2019  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.conf.urls import url
+
+from . import views
+
+urlpatterns = [
+    url(r'^$', views.home, name='seo-home'),
+    url(r'^allow$', views.allow, name='robots-allow'),
+    url(r'^disallow$', views.disallow, name='robots-disallow'),
+    url(r'^customize$', views.customize, name='robots-customize'),
+]
diff --git a/hobo/seo/views.py b/hobo/seo/views.py
new file mode 100644
index 0000000..bd96075
--- /dev/null
+++ b/hobo/seo/views.py
@@ -0,0 +1,99 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2020  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.core.urlresolvers import reverse_lazy
+from django.views.generic import FormView, RedirectView
+
+from hobo.environment.utils import get_variable
+from .forms import SettingsForm
+
+
+ALLOW = ""
+DISALLOW = """User-agent: *
+Disallow: /"""
+
+def put_robots_txt(content):
+    variable = get_variable('robots_txt')
+    variable.value = content
+    variable.save()
+
+def get_mode(content):
+    content = content.strip().replace('\r', '')
+    if content == ALLOW:
+        return 'allow'
+    elif content == DISALLOW:
+        return 'disallow'
+    else:
+        return 'customize'
+
+class HomeView(FormView):
+    template_name = 'hobo/seo_home.html'
+    form_class = SettingsForm
+    success_url = reverse_lazy('seo-home')
+
+    def get_initial(self):
+        initial = super(HomeView, self).get_initial()
+        initial['content'] = get_variable('robots_txt').value
+        initial['mode'] = get_mode(initial['content'])
+        return initial
+
+    def form_valid(self, form):
+        content = form.cleaned_data['content']
+        put_robots_txt(content)
+        return super(HomeView, self).form_valid(form)
+
+    def get_context_data(self, **kwargs):
+        context = super(HomeView, self).get_context_data(**kwargs)
+        mode = get_mode(get_variable('robots_txt').value)
+        context['form'].fields['content'].disabled = mode != 'customize'
+        context['mode'] = mode
+        return context
+
+home = HomeView.as_view()
+
+
+class AllowView(RedirectView):
+    def get_redirect_url(self):
+        put_robots_txt(ALLOW)
+        return reverse_lazy('seo-home')
+
+allow = AllowView.as_view()
+
+
+class DisallowView(RedirectView):
+    def get_redirect_url(self):
+        put_robots_txt(DISALLOW)
+        return reverse_lazy('seo-home')
+
+disallow = DisallowView.as_view()
+
+
+class CustomizeView(FormView):
+    template_name = 'hobo/robots_customize.html'
+    form_class = SettingsForm
+    success_url = reverse_lazy('seo-home')
+
+    def get_initial(self):
+        initial = super(CustomizeView, self).get_initial()
+        initial['content'] = get_variable('robots_txt').value
+        return initial
+
+    def form_valid(self, form):
+        content = form.cleaned_data['content']
+        put_robots_txt(content)
+        return super(CustomizeView, self).form_valid(form)
+
+customize = CustomizeView.as_view()
diff --git a/hobo/settings.py b/hobo/settings.py
index deeeaf2..1feb8f2 100644
--- a/hobo/settings.py
+++ b/hobo/settings.py
@@ -41,16 +41,17 @@ INSTALLED_APPS = (
     'rest_framework',
     'mellon',
     'gadjo',
     'hobo.debug',
     'hobo.environment',
     'hobo.franceconnect',
     'hobo.matomo',
     'hobo.profile',
+    'hobo.seo',
     'hobo.theme',
     'hobo.emails',
     'hobo.deploy',
 )
 
 MIDDLEWARE_CLASSES = (
     'hobo.middleware.seo.RobotMiddleware',
     'hobo.middleware.xforwardedfor.XForwardedForMiddleware',
diff --git a/hobo/templates/hobo/home.html b/hobo/templates/hobo/home.html
index 2f966ee..eba41cb 100644
--- a/hobo/templates/hobo/home.html
+++ b/hobo/templates/hobo/home.html
@@ -10,16 +10,17 @@
     <li><a href="{% url 'profile-home' %}">{% trans 'User Profile' %}</a></li>
     {% endif %}
     <li><a href="{% url 'theme-home' %}">{% trans 'Theme' %}</a></li>
     <li><a href="{% url 'emails-home' %}">{% trans 'Emails' %}</a></li>
     {% if has_authentic %}
     <li><a href="{% url 'franceconnect-home' %}">FranceConnect</a></li>
     {% endif %}
     <li><a href="{% url 'matomo-home' %}">{% trans 'User tracking' %}</a></li>
+    <li><a href="{% url 'seo-home' %}">{% trans 'Search Engine Optimization' %}</a></li>
     <li><a href="{% url 'environment-home' %}">{% trans 'Services' %}</a></li>
     <li><a href="{% url 'environment-variables' %}">{% trans 'Variables' %}</a></li>
     <li><a href="{% url 'debug-home' %}">{% trans 'Debugging' %}</a></li>
   </ul>
   </span>
 {% endblock %}
 
 {% block content %}
diff --git a/hobo/urls.py b/hobo/urls.py
index f81a7f4..7993d50 100644
--- a/hobo/urls.py
+++ b/hobo/urls.py
@@ -23,27 +23,29 @@ from .views import admin_required, login, login_local, logout, home, health_json
 from .urls_utils import decorated_includes
 from .environment.urls import urlpatterns as environment_urls
 from .franceconnect.urls import urlpatterns as franceconnect_urls
 from .matomo.urls import urlpatterns as matomo_urls
 from .profile.urls import urlpatterns as profile_urls
 from .theme.urls import urlpatterns as theme_urls
 from .emails.urls import urlpatterns as emails_urls
 from .debug.urls import urlpatterns as debug_urls
+from .seo.urls import urlpatterns as seo_urls
 
 admin.autodiscover()
 
 urlpatterns = [
     url(r'^$', home, name='home'),
     url(r'^sites/', decorated_includes(admin_required, include(environment_urls))),
     url(r'^profile/', decorated_includes(admin_required, include(profile_urls))),
     url(r'^franceconnect/', decorated_includes(admin_required, include(franceconnect_urls))),
     url(r'^visits-tracking/', decorated_includes(admin_required, include(matomo_urls))),
     url(r'^theme/', decorated_includes(admin_required, include(theme_urls))),
     url(r'^emails/', decorated_includes(admin_required, include(emails_urls))),
+    url(r'^seo/', decorated_includes(admin_required, include(seo_urls))),
     url(r'^debug/', decorated_includes(admin_required, include(debug_urls))),
     url(r'^api/health/$', health_json, name='health-json'),
     url(r'^menu.json$', menu_json, name='menu_json'),
     url(r'^hobos.json$', hobo),
     url(r'^admin/', admin.site.urls),
 ]
 
 # add authentication patterns
diff --git a/tests/test_seo.py b/tests/test_seo.py
index f976834..1dcec66 100644
--- a/tests/test_seo.py
+++ b/tests/test_seo.py
@@ -13,16 +13,18 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.import pytest
 import pytest
 
 from hobo.environment.utils import get_variable
 from hobo.seo.views import ALLOW, DISALLOW
 
+from test_manager import login
+
 
 pytestmark = pytest.mark.django_db
 
 def test_middleware(app):
     variable = get_variable('robots_txt')
     variable.value = DISALLOW
     variable.save()
     resp = app.get('/robots.txt', status=200)
@@ -33,8 +35,77 @@ def test_middleware(app):
     variable.save()
     resp = app.get('/robots.txt', status=200)
     assert resp.text == ALLOW
 
     variable = get_variable('robots_txt')
     variable.delete()
     resp = app.get('/robots.txt', status=200)
     assert resp.text == ALLOW
+
+def test_unlogged_access(app):
+    # connect while not being logged in
+    resp = app.get('/seo/', status=302)
+    assert resp.location.endswith('/login/?next=/seo/')
+
+def test_home(app, admin_user):
+    login(app)
+    resp = app.get('/seo/', status=200)
+    assert [
+        x['href'] for x in resp.html.find('span', {'class': 'actions'}).find_all('a')
+    ] == ['/seo/customize', '/seo/disallow', '/seo/allow']
+
+    variable = get_variable('robots_txt')
+    variable.value = ALLOW
+    variable.save()
+    resp = app.get('/seo/', status=200)
+    assert 'disabled' in resp.html.find('a', {'href': '/seo/allow'})['class']
+    assert not resp.html.textarea
+
+    variable = get_variable('robots_txt')
+    variable.value = DISALLOW
+    variable.save()
+    resp = app.get('/seo/', status=200)
+    assert 'disabled' in resp.html.find('a', {'href': '/seo/disallow'})['class']
+    assert resp.html.textarea.text.strip() == DISALLOW
+    assert resp.html.textarea.has_attr('disabled')
+    assert not resp.html.button
+
+    variable = get_variable('robots_txt')
+    variable.value = "some content"
+    variable.save()
+    resp = app.get('/seo/', status=200)
+    assert 'disabled' in resp.html.find('a', {'href': '/seo/customize'})['class']
+    assert resp.html.textarea.text.strip() == 'some content'
+    assert not resp.html.textarea.has_attr('disabled')
+    resp.form['content'] = 'some new content'
+    resp = resp.form.submit()
+    assert get_variable('robots_txt').value == 'some new content'
+
+def test_allow(app, admin_user):
+    login(app)
+    variable = get_variable('robots_txt')
+    variable.value = "some content"
+    variable.save()
+    resp = app.get('/seo/allow', status=302)
+    assert resp.location.endswith('/seo/')
+    assert get_variable('robots_txt').value == ALLOW
+
+def test_disallow(app, admin_user):
+    login(app)
+    variable = get_variable('robots_txt')
+    variable.value = "some content"
+    variable.save()
+    resp = app.get('/seo/disallow', status=302)
+    assert resp.location.endswith('/seo/')
+    assert get_variable('robots_txt').value == DISALLOW
+
+def test_custom(app, admin_user):
+    login(app)
+    variable = get_variable('robots_txt')
+    variable.value = "some content"
+    variable.save()
+    resp = app.get('/seo/customize', status=200)
+    assert resp.html.textarea['name'] == 'content'
+    assert resp.html.textarea.text.strip() == 'some content'
+    resp.form['content'] = 'some new content'
+    resp = resp.form.submit()
+    assert get_variable('robots_txt').value == 'some new content'
-- 
2.25.1

