From 22b426f9028ed940724e0893157bee430097ceb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Wed, 28 Nov 2018 19:59:45 +0100 Subject: [PATCH 1/2] add caching to health API (#26836) --- hobo/environment/models.py | 19 +++++++++ hobo/environment/urls.py | 1 + hobo/environment/views.py | 7 +++- hobo/rest/serializers.py | 27 ------------ hobo/rest/urls.py | 22 ---------- hobo/rest/views.py | 46 --------------------- hobo/urls.py | 2 - {hobo/rest => tests}/__init__.py | 0 tests/test_health_api.py | 70 ++++++++++++++++++++++---------- 9 files changed, 75 insertions(+), 119 deletions(-) delete mode 100644 hobo/rest/serializers.py delete mode 100644 hobo/rest/urls.py delete mode 100644 hobo/rest/views.py rename {hobo/rest => tests}/__init__.py (100%) diff --git a/hobo/environment/models.py b/hobo/environment/models.py index 01ab76f..a3de536 100644 --- a/hobo/environment/models.py +++ b/hobo/environment/models.py @@ -1,9 +1,11 @@ import datetime import json +import random import requests import socket from django.conf import settings +from django.core.cache import cache from django.db import models from django.utils.crypto import get_random_string from django.utils.encoding import force_text @@ -189,6 +191,23 @@ class ServiceBase(models.Model): r = requests.get(self.get_admin_zones()[0].href, verify=False) return bool(r.status_code is 200) + def get_health_dict(self): + properties = [ + ('is_resolvable', 120), + ('has_valid_certificate', 3600), + ('is_running', 60), + ('is_operational', 60), + ] + result = {} + for name, cache_duration in properties: + cache_key = '%s_%s' % (self.slug, name) + value = cache.get(cache_key) + if value is None: + value = getattr(self, name)() + cache.set(cache_key, value, cache_duration * (0.5 + random.random())) + result[name] = value + return result + class Authentic(ServiceBase): use_as_idp_for_self = models.BooleanField( diff --git a/hobo/environment/urls.py b/hobo/environment/urls.py index 58f5519..1254a52 100644 --- a/hobo/environment/urls.py +++ b/hobo/environment/urls.py @@ -19,4 +19,5 @@ urlpatterns = [ url(r'^new-variable-(?P\w+)/(?P[\w-]+)$', VariableCreateView.as_view(), name='new-variable-service',), url(r'^debug.json$', debug_json, name='debug-json'), + url(r'^health.json$', health_json, name='health-json'), ] diff --git a/hobo/environment/views.py b/hobo/environment/views.py index 9603c3d..aa9d646 100644 --- a/hobo/environment/views.py +++ b/hobo/environment/views.py @@ -4,7 +4,7 @@ import string from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.urlresolvers import reverse_lazy -from django.http import HttpResponse, HttpResponseRedirect, Http404 +from django.http import HttpResponse, HttpResponseRedirect, Http404, JsonResponse from django.shortcuts import get_object_or_404 from django.views.generic.base import TemplateView from django.views.generic.edit import CreateView, UpdateView, DeleteView @@ -200,3 +200,8 @@ def debug_json(request): response = HttpResponse(content_type='application/json') json.dump((utils.get_installed_services_dict(),), response, indent=2) return response + + +def health_json(request): + data = {x.slug: x.get_health_dict() for x in utils.get_installed_services() if not x.secondary} + return JsonResponse({'data': data}) diff --git a/hobo/rest/serializers.py b/hobo/rest/serializers.py deleted file mode 100644 index 06b11f7..0000000 --- a/hobo/rest/serializers.py +++ /dev/null @@ -1,27 +0,0 @@ -# hobo - portal to configure and deploy applications -# Copyright (C) 2015-2018 Entr'ouvert -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - -from rest_framework import serializers - - -class HealthBaseSerializer(serializers.Serializer): - title = serializers.CharField() - slug = serializers.SlugField() - base_url = serializers.CharField() - is_resolvable = serializers.BooleanField() - has_valid_certificate = serializers.BooleanField() - is_running = serializers.BooleanField() - is_operational = serializers.BooleanField() diff --git a/hobo/rest/urls.py b/hobo/rest/urls.py deleted file mode 100644 index 75675b7..0000000 --- a/hobo/rest/urls.py +++ /dev/null @@ -1,22 +0,0 @@ -# hobo - portal to configure and deploy applications -# Copyright (C) 2015-2018 Entr'ouvert -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - -from django.conf.urls import url -from .views import HealthList - -urlpatterns = [ - url(r'^health/$', HealthList.as_view(), name='api-health-list'), -] diff --git a/hobo/rest/views.py b/hobo/rest/views.py deleted file mode 100644 index bd95735..0000000 --- a/hobo/rest/views.py +++ /dev/null @@ -1,46 +0,0 @@ -# hobo - portal to configure and deploy applications -# Copyright (C) 2015-2018 Entr'ouvert -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - -from rest_framework import generics -from rest_framework import permissions -from rest_framework.response import Response - -from hobo.environment.utils import get_installed_services -from hobo.rest.serializers import HealthBaseSerializer - - -class HealthList(generics.ListAPIView): - serializer_class = HealthBaseSerializer - permission_classes = (permissions.IsAuthenticatedOrReadOnly,) - - def get_queryset(self): - return [x for x in get_installed_services() if not x.secondary] - - def list(self, request, *args, **kwargs): - """ - Custom dictionary object - """ - self.object_list = self.filter_queryset(self.get_queryset()) - - # Switch between paginated or standard style responses - page = self.paginate_queryset(self.object_list) - if page is not None: - serializer = self.get_pagination_serializer(page) - else: - serializer = self.get_serializer(self.object_list, many=True) - - response = {d['slug']: d for d in serializer.data} - return Response({'data': response}) diff --git a/hobo/urls.py b/hobo/urls.py index d247818..c0af2d4 100644 --- a/hobo/urls.py +++ b/hobo/urls.py @@ -10,7 +10,6 @@ from .environment.urls import urlpatterns as environment_urls from .profile.urls import urlpatterns as profile_urls from .theme.urls import urlpatterns as theme_urls from .emails.urls import urlpatterns as emails_urls -from .rest.urls import urlpatterns as rest_urls urlpatterns = [ url(r'^$', home, name='home'), @@ -21,7 +20,6 @@ urlpatterns = [ url(r'^theme/', decorated_includes(admin_required, include(theme_urls))), url(r'^emails/', decorated_includes(admin_required, include(emails_urls))), - url(r'^api/', include(rest_urls)), url(r'^menu.json$', menu_json, name='menu_json'), url(r'^hobos.json$', hobo), url(r'^admin/', include(admin.site.urls)), diff --git a/hobo/rest/__init__.py b/tests/__init__.py similarity index 100% rename from hobo/rest/__init__.py rename to tests/__init__.py diff --git a/tests/test_health_api.py b/tests/test_health_api.py index e941eac..9ca4484 100644 --- a/tests/test_health_api.py +++ b/tests/test_health_api.py @@ -4,10 +4,15 @@ import pytest import requests import socket +from django.core.cache import cache from django.utils import timezone +from httmock import urlmatch, remember_called, HTTMock + from hobo.environment.models import Authentic, Combo +from test_manager import admin_user, login + pytestmark = pytest.mark.django_db @@ -22,17 +27,19 @@ def services(request): c.save() -def test_response(app, services, monkeypatch): +def test_response(app, admin_user, services, monkeypatch): + cache.clear() monkeypatch.setattr(socket, 'gethostbyname', lambda x: '176.31.123.109') monkeypatch.setattr(requests, 'get', lambda x, verify: MagicMock(status_code=200)) - response = app.get('/api/health/') + response = login(app).get('/sites/health.json') assert response.status_code == 200 content = json.loads(response.content) assert 'blues' in content['data'].keys() assert 'jazz' in content['data'].keys() -def test_is_resolvable(app, services, monkeypatch): +def test_is_resolvable(app, admin_user, services, monkeypatch): + cache.clear() def gethostname(netloc): if netloc == "jazz.example.publik": return '176.31.123.109' @@ -40,7 +47,7 @@ def test_is_resolvable(app, services, monkeypatch): raise socket.gaierror monkeypatch.setattr(socket, 'gethostbyname', gethostname) monkeypatch.setattr(requests, 'get', lambda x, verify: MagicMock(status_code=200)) - response = app.get('/api/health/') + response = login(app).get('/sites/health.json') content = json.loads(response.content) blues = content['data']['blues'] jazz = content['data']['jazz'] @@ -48,26 +55,46 @@ def test_is_resolvable(app, services, monkeypatch): assert jazz['is_resolvable'] -def test_is_running(app, services, monkeypatch): - def get(url, verify): - if url == 'https://jazz.example.publik/manage/': - return MagicMock(status_code=200) - else: - return MagicMock(status_code=404) +def test_is_running(app, admin_user, services, monkeypatch): + cache.clear() monkeypatch.setattr(socket, 'gethostbyname', lambda x: '176.31.123.109') - monkeypatch.setattr(requests, 'get', get) - response = app.get('/api/health/') - content = json.loads(response.content) - blues = content['data']['blues'] - jazz = content['data']['jazz'] - assert not blues['is_running'] - assert jazz['is_running'] + + @urlmatch(netloc='jazz.example.publik') + @remember_called + def jazz_mock(url, request): + return {'status_code': 200} + + @urlmatch(netloc='blues.example.publik') + @remember_called + def blues_mock(url, request): + return {'status_code': 404} + + with HTTMock(blues_mock, jazz_mock) as mock: + response = login(app).get('/sites/health.json') + content = json.loads(response.content) + blues = content['data']['blues'] + jazz = content['data']['jazz'] + assert not blues['is_running'] + assert jazz['is_running'] + assert blues_mock.call['count'] == 2 + assert jazz_mock.call['count'] == 2 + + # check it gets results from cache + response = login(app).get('/sites/health.json') + content = json.loads(response.content) + blues = content['data']['blues'] + jazz = content['data']['jazz'] + assert not blues['is_running'] + assert jazz['is_running'] + assert blues_mock.call['count'] == 2 + assert jazz_mock.call['count'] == 2 -def test_is_operational(app, services, monkeypatch): +def test_is_operational(app, admin_user, services, monkeypatch): + cache.clear() monkeypatch.setattr(socket, 'gethostbyname', lambda x: '176.31.123.109') monkeypatch.setattr(requests, 'get', lambda x, verify: MagicMock(status_code=200)) - response = app.get('/api/health/') + response = login(app).get('/sites/health.json') content = json.loads(response.content) blues = content['data']['blues'] jazz = content['data']['jazz'] @@ -75,7 +102,8 @@ def test_is_operational(app, services, monkeypatch): assert not jazz['is_operational'] -def test_has_valid_certificate(app, services, monkeypatch): +def test_has_valid_certificate(app, admin_user, services, monkeypatch): + cache.clear() def get(url, verify): if 'blues.example.publik' in url or not verify: return MagicMock(status_code=200) @@ -83,7 +111,7 @@ def test_has_valid_certificate(app, services, monkeypatch): raise requests.exceptions.SSLError monkeypatch.setattr(socket, 'gethostbyname', lambda x: '176.31.123.109') monkeypatch.setattr(requests, 'get', get) - response = app.get('/api/health/') + response = login(app).get('/sites/health.json') content = json.loads(response.content) blues = content['data']['blues'] jazz = content['data']['jazz'] -- 2.20.0.rc1