From 142bb0d1c7bc87fe1a7dfab42ed620492f577cbc Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Tue, 7 Apr 2020 18:46:48 +0200
Subject: [PATCH 2/3] seo: add robots_txt views (#20263)

---
 hobo/environment/utils.py               |  8 +++
 hobo/seo/__init__.py                    |  0
 hobo/seo/forms.py                       | 29 ++++++++++
 hobo/seo/templates/hobo/robots_txt.html | 20 +++++++
 hobo/seo/templates/hobo/seo_home.html   | 58 +++++++++++++++++++
 hobo/seo/urls.py                        | 26 +++++++++
 hobo/seo/views.py                       | 76 +++++++++++++++++++++++++
 hobo/settings.py                        |  1 +
 hobo/templates/hobo/home.html           |  1 +
 hobo/urls.py                            |  2 +
 tests/test_seo.py                       | 70 +++++++++++++++++++++++
 11 files changed, 291 insertions(+)
 create mode 100644 hobo/seo/__init__.py
 create mode 100644 hobo/seo/forms.py
 create mode 100644 hobo/seo/templates/hobo/robots_txt.html
 create mode 100644 hobo/seo/templates/hobo/seo_home.html
 create mode 100644 hobo/seo/urls.py
 create mode 100644 hobo/seo/views.py

diff --git a/hobo/environment/utils.py b/hobo/environment/utils.py
index ca07029..034b4b7 100644
--- a/hobo/environment/utils.py
+++ b/hobo/environment/utils.py
@@ -78,16 +78,24 @@ def get_variable(name):
         name=name,
         defaults={
             'auto': True,
             'value': settings.VARIABLE_SETTINGS_DEFAULTS.get(name) or ''
         })
     return variable
 
 
+def set_variable(name, value):
+    from .models import Variable
+    variable, created = Variable.objects.get_or_create(
+            name=name, defaults={'auto': True})
+    variable.value = value
+    variable.save()
+
+
 def create_base_url(hostname, service):
     """
     Distinguish mutualised domains (matching a "-" in the first part of the netloc)
     from the normal scenario with a dedicated parent domain.
     """
     ph = urlparse(hostname)
     parts = ph.netloc.split('.')
     if '-' in parts[0]:
diff --git a/hobo/seo/__init__.py b/hobo/seo/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/hobo/seo/forms.py b/hobo/seo/forms.py
new file mode 100644
index 0000000..9d27c8d
--- /dev/null
+++ b/hobo/seo/forms.py
@@ -0,0 +1,29 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2019  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django import forms
+from django.utils.translation import ugettext_lazy as _
+
+
+class RobotsTxtForm(forms.Form):
+    content = forms.CharField(
+        label=_('Content of robots.txt file'),
+        required=False,
+        widget=forms.Textarea)
+
+
+class EnableForm(forms.Form):
+    pass
diff --git a/hobo/seo/templates/hobo/robots_txt.html b/hobo/seo/templates/hobo/robots_txt.html
new file mode 100644
index 0000000..18bcf26
--- /dev/null
+++ b/hobo/seo/templates/hobo/robots_txt.html
@@ -0,0 +1,20 @@
+{% extends "hobo/seo_home.html" %}
+{% load i18n %}
+
+{% block appbar %}
+  <h2>{% trans 'Robots Exclusion Protocol' %}</h2>
+{% endblock %}
+
+{% block content %}
+<form method="post">
+  {% csrf_token %}
+  {{ form.as_p }}
+<p>
+
+<div class="buttons">
+<button class="submit-button">{% trans "Save" %}</button>
+<a class="cancel" href="{% url 'seo-home' %}">{% trans "Cancel" %}</a>
+</div>
+</form>
+
+{% endblock %}
diff --git a/hobo/seo/templates/hobo/seo_home.html b/hobo/seo/templates/hobo/seo_home.html
new file mode 100644
index 0000000..32b5e27
--- /dev/null
+++ b/hobo/seo/templates/hobo/seo_home.html
@@ -0,0 +1,58 @@
+{% extends "hobo/base.html" %}
+{% load i18n %}
+
+{% block breadcrumb %}
+{{ block.super }}
+<a href="{% url 'seo-home' %}">Se</a>
+{% endblock %}
+
+{% block appbar %}
+<h2>{% trans 'Indexing Settings' %}</h2>
+<span class="actions">
+  <a
+    rel="popup" href="{% url 'robots-customize' %}">
+    {% if mode == 'customize' %}{% trans "Change custom settings" %}
+    {% else %}{% trans 'Use custom settings' %}{% endif %}
+  </a>
+  <a
+    {% if mode == 'disallow' %}class="disabled"{% endif %}
+    href="{% url 'robots-disallow' %}">
+    {% trans 'Do not allow indexing' %}
+  </a>
+  <a
+    {% if mode == 'allow' %}class="disabled"{% endif %}
+    href="{% url 'robots-allow' %}">
+    {% trans 'Allow indexing' %}
+  </a>
+</span>
+{% endblock %}
+
+{% block content %}
+<div class="infonotice">
+    {% blocktrans %}
+    Robots are programs that automatically traverse web sites.
+    Web site owners can use a <tt>robots.txt</tt> file to give
+    instructions about their site.
+    {% endblocktrans %}
+</div>
+
+{% if mode == 'allow' %}
+<div class="successnotice">
+  {% trans "Web Robots are currently allowed." %}
+</div>
+{% endif %}
+
+{% if mode == 'disallow' %}
+<div class="warningnotice">
+  {% trans "Web Robots are currently not allowed." %}
+</div>
+{% endif %}
+
+{% if mode == 'customize' or mode == 'disallow' %}
+<div class="section padded">
+  <span>{% trans "Contents of robots.txt file:" %}</span>
+  <pre>{{ robots_txt }}</pre>
+</div>
+{% endif %}
+
+{% endblock %}
diff --git a/hobo/seo/urls.py b/hobo/seo/urls.py
new file mode 100644
index 0000000..d0884ef
--- /dev/null
+++ b/hobo/seo/urls.py
@@ -0,0 +1,26 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2019  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.conf.urls import url
+
+from . import views
+
+urlpatterns = [
+    url(r'^$', views.home, name='seo-home'),
+    url(r'^allow$', views.allow, name='robots-allow'),
+    url(r'^disallow$', views.disallow, name='robots-disallow'),
+    url(r'^customize$', views.customize, name='robots-customize'),
+]
diff --git a/hobo/seo/views.py b/hobo/seo/views.py
new file mode 100644
index 0000000..c1103e3
--- /dev/null
+++ b/hobo/seo/views.py
@@ -0,0 +1,76 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2020  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.core.urlresolvers import reverse_lazy
+from django.http import HttpResponseRedirect
+from django.views.generic import FormView, TemplateView
+
+from hobo.environment.utils import get_variable, set_variable
+from .forms import RobotsTxtForm
+
+
+ALLOW = ""
+DISALLOW = """User-agent: *
+Disallow: /"""
+
+
+def get_mode(content):
+    content = content.strip().replace('\r', '')
+    if content == ALLOW:
+        return 'allow'
+    if content == DISALLOW:
+        return 'disallow'
+    return 'customize'
+
+
+class HomeView(TemplateView):
+    template_name = 'hobo/seo_home.html'
+    success_url = reverse_lazy('seo-home')
+
+    def get_context_data(self, **kwargs):
+        context = super(HomeView, self).get_context_data(**kwargs)
+        context['robots_txt'] = get_variable('robots_txt').value
+        context['mode'] = get_mode(context['robots_txt'])
+        return context
+
+home = HomeView.as_view()
+
+
+def allow(request):
+    set_variable('robots_txt', ALLOW)
+    return HttpResponseRedirect(reverse_lazy('seo-home'))
+
+
+def disallow(request):
+    set_variable('robots_txt', DISALLOW)
+    return HttpResponseRedirect(reverse_lazy('seo-home'))
+
+
+class CustomizeView(FormView):
+    template_name = 'hobo/robots_txt.html'
+    form_class = RobotsTxtForm
+    success_url = reverse_lazy('seo-home')
+
+    def get_initial(self):
+        initial = super(CustomizeView, self).get_initial()
+        initial['content'] = get_variable('robots_txt').value
+        return initial
+
+    def form_valid(self, form):
+        set_variable('robots_txt', form.cleaned_data['content'])
+        return super(CustomizeView, self).form_valid(form)
+
+customize = CustomizeView.as_view()
diff --git a/hobo/settings.py b/hobo/settings.py
index bc76dbc..f0363b8 100644
--- a/hobo/settings.py
+++ b/hobo/settings.py
@@ -41,16 +41,17 @@ INSTALLED_APPS = (
     'rest_framework',
     'mellon',
     'gadjo',
     'hobo.debug',
     'hobo.environment',
     'hobo.franceconnect',
     'hobo.matomo',
     'hobo.profile',
+    'hobo.seo',
     'hobo.theme',
     'hobo.emails',
     'hobo.deploy',
 )
 
 MIDDLEWARE_CLASSES = (
     'hobo.middleware.xforwardedfor.XForwardedForMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
diff --git a/hobo/templates/hobo/home.html b/hobo/templates/hobo/home.html
index 2f966ee..07fd9d1 100644
--- a/hobo/templates/hobo/home.html
+++ b/hobo/templates/hobo/home.html
@@ -10,16 +10,17 @@
     <li><a href="{% url 'profile-home' %}">{% trans 'User Profile' %}</a></li>
     {% endif %}
     <li><a href="{% url 'theme-home' %}">{% trans 'Theme' %}</a></li>
     <li><a href="{% url 'emails-home' %}">{% trans 'Emails' %}</a></li>
     {% if has_authentic %}
     <li><a href="{% url 'franceconnect-home' %}">FranceConnect</a></li>
     {% endif %}
     <li><a href="{% url 'matomo-home' %}">{% trans 'User tracking' %}</a></li>
+    <li><a href="{% url 'seo-home' %}">{% trans 'Indexing' %}</a></li>
     <li><a href="{% url 'environment-home' %}">{% trans 'Services' %}</a></li>
     <li><a href="{% url 'environment-variables' %}">{% trans 'Variables' %}</a></li>
     <li><a href="{% url 'debug-home' %}">{% trans 'Debugging' %}</a></li>
   </ul>
   </span>
 {% endblock %}
 
 {% block content %}
diff --git a/hobo/urls.py b/hobo/urls.py
index f81a7f4..7993d50 100644
--- a/hobo/urls.py
+++ b/hobo/urls.py
@@ -23,27 +23,29 @@ from .views import admin_required, login, login_local, logout, home, health_json
 from .urls_utils import decorated_includes
 from .environment.urls import urlpatterns as environment_urls
 from .franceconnect.urls import urlpatterns as franceconnect_urls
 from .matomo.urls import urlpatterns as matomo_urls
 from .profile.urls import urlpatterns as profile_urls
 from .theme.urls import urlpatterns as theme_urls
 from .emails.urls import urlpatterns as emails_urls
 from .debug.urls import urlpatterns as debug_urls
+from .seo.urls import urlpatterns as seo_urls
 
 admin.autodiscover()
 
 urlpatterns = [
     url(r'^$', home, name='home'),
     url(r'^sites/', decorated_includes(admin_required, include(environment_urls))),
     url(r'^profile/', decorated_includes(admin_required, include(profile_urls))),
     url(r'^franceconnect/', decorated_includes(admin_required, include(franceconnect_urls))),
     url(r'^visits-tracking/', decorated_includes(admin_required, include(matomo_urls))),
     url(r'^theme/', decorated_includes(admin_required, include(theme_urls))),
     url(r'^emails/', decorated_includes(admin_required, include(emails_urls))),
+    url(r'^seo/', decorated_includes(admin_required, include(seo_urls))),
     url(r'^debug/', decorated_includes(admin_required, include(debug_urls))),
     url(r'^api/health/$', health_json, name='health-json'),
     url(r'^menu.json$', menu_json, name='menu_json'),
     url(r'^hobos.json$', hobo),
     url(r'^admin/', admin.site.urls),
 ]
 
 # add authentication patterns
diff --git a/tests/test_seo.py b/tests/test_seo.py
index f0e0530..84b77c9 100644
--- a/tests/test_seo.py
+++ b/tests/test_seo.py
@@ -13,18 +13,88 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.import pytest
 
 import pytest
 
 from django.test import override_settings
 
+from hobo.environment.utils import get_variable
+from hobo.seo.views import ALLOW, DISALLOW
+
+from test_manager import login
+
 
 pytestmark = pytest.mark.django_db
 
+
 def test_middleware(app):
     with override_settings(TEMPLATE_VARS={'robots_txt': 'xxx'}):
         resp = app.get('/robots.txt', status=200)
         assert resp.text == 'xxx'
 
     with override_settings():
         resp = app.get('/robots.txt', status=404)
+
+
+def test_unlogged_access(app):
+    # connect while not being logged in
+    resp = app.get('/seo/', status=302)
+    assert resp.location.endswith('/login/?next=/seo/')
+
+
+def test_home(app, admin_user):
+    login(app)
+    resp = app.get('/seo/', status=200)
+    assert [
+        x['href'] for x in resp.html.find('span', {'class': 'actions'}).find_all('a')
+    ] == ['/seo/customize', '/seo/disallow', '/seo/allow']
+
+    variable = get_variable('robots_txt')
+    variable.value = ALLOW
+    variable.save()
+    resp = app.get('/seo/', status=200)
+    assert 'disabled' in resp.html.find('a', {'href': '/seo/allow'})['class']
+
+    variable = get_variable('robots_txt')
+    variable.value = DISALLOW
+    variable.save()
+    resp = app.get('/seo/', status=200)
+    assert 'disabled' in resp.html.find('a', {'href': '/seo/disallow'})['class']
+    assert not resp.html.button
+
+
+def test_allow(app, admin_user):
+    login(app)
+    variable = get_variable('robots_txt')
+    variable.value = "some content"
+    variable.save()
+    resp = app.get('/seo/allow', status=302)
+    assert resp.location.endswith('/seo/')
+    assert get_variable('robots_txt').value == ALLOW
+
+
+def test_disallow(app, admin_user):
+    login(app)
+    variable = get_variable('robots_txt')
+    variable.value = "some content"
+    variable.save()
+    resp = app.get('/seo/disallow', status=302)
+    assert resp.location.endswith('/seo/')
+    assert get_variable('robots_txt').value == DISALLOW
+    resp = resp.follow()
+    assert resp.html.pre.text == DISALLOW
+
+
+def test_custom(app, admin_user):
+    login(app)
+    variable = get_variable('robots_txt')
+    variable.value = "some content"
+    variable.save()
+    resp = app.get('/seo/customize', status=200)
+    assert resp.html.textarea['name'] == 'content'
+    assert resp.html.textarea.text.strip() == 'some content'
+    resp.form['content'] = 'some new content'
+    resp = resp.form.submit()
+    assert get_variable('robots_txt').value == 'some new content'
+    resp = resp.follow()
+    assert resp.html.pre.text == 'some new content'
-- 
2.25.1

