From d5984fa6c3b10dd3c2aef6579142cad4cb466679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Wed, 28 Nov 2018 19:59:45 +0100 Subject: [PATCH 1/2] add caching to health API (#26836) --- hobo/environment/models.py | 19 ++++++++++++ hobo/rest/__init__.py | 0 hobo/rest/serializers.py | 27 ----------------- hobo/rest/urls.py | 22 -------------- hobo/rest/views.py | 46 ----------------------------- hobo/urls.py | 5 ++-- hobo/views.py | 8 ++++- tests/test_health_api.py | 60 +++++++++++++++++++++++++++----------- 8 files changed, 71 insertions(+), 116 deletions(-) delete mode 100644 hobo/rest/__init__.py delete mode 100644 hobo/rest/serializers.py delete mode 100644 hobo/rest/urls.py delete mode 100644 hobo/rest/views.py diff --git a/hobo/environment/models.py b/hobo/environment/models.py index 01ab76f..a3de536 100644 --- a/hobo/environment/models.py +++ b/hobo/environment/models.py @@ -1,9 +1,11 @@ import datetime import json +import random import requests import socket from django.conf import settings +from django.core.cache import cache from django.db import models from django.utils.crypto import get_random_string from django.utils.encoding import force_text @@ -189,6 +191,23 @@ class ServiceBase(models.Model): r = requests.get(self.get_admin_zones()[0].href, verify=False) return bool(r.status_code is 200) + def get_health_dict(self): + properties = [ + ('is_resolvable', 120), + ('has_valid_certificate', 3600), + ('is_running', 60), + ('is_operational', 60), + ] + result = {} + for name, cache_duration in properties: + cache_key = '%s_%s' % (self.slug, name) + value = cache.get(cache_key) + if value is None: + value = getattr(self, name)() + cache.set(cache_key, value, cache_duration * (0.5 + random.random())) + result[name] = value + return result + class Authentic(ServiceBase): use_as_idp_for_self = models.BooleanField( diff --git a/hobo/rest/__init__.py b/hobo/rest/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/hobo/rest/serializers.py b/hobo/rest/serializers.py deleted file mode 100644 index 06b11f7..0000000 --- a/hobo/rest/serializers.py +++ /dev/null @@ -1,27 +0,0 @@ -# hobo - portal to configure and deploy applications -# Copyright (C) 2015-2018 Entr'ouvert -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - -from rest_framework import serializers - - -class HealthBaseSerializer(serializers.Serializer): - title = serializers.CharField() - slug = serializers.SlugField() - base_url = serializers.CharField() - is_resolvable = serializers.BooleanField() - has_valid_certificate = serializers.BooleanField() - is_running = serializers.BooleanField() - is_operational = serializers.BooleanField() diff --git a/hobo/rest/urls.py b/hobo/rest/urls.py deleted file mode 100644 index 75675b7..0000000 --- a/hobo/rest/urls.py +++ /dev/null @@ -1,22 +0,0 @@ -# hobo - portal to configure and deploy applications -# Copyright (C) 2015-2018 Entr'ouvert -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - -from django.conf.urls import url -from .views import HealthList - -urlpatterns = [ - url(r'^health/$', HealthList.as_view(), name='api-health-list'), -] diff --git a/hobo/rest/views.py b/hobo/rest/views.py deleted file mode 100644 index bd95735..0000000 --- a/hobo/rest/views.py +++ /dev/null @@ -1,46 +0,0 @@ -# hobo - portal to configure and deploy applications -# Copyright (C) 2015-2018 Entr'ouvert -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - -from rest_framework import generics -from rest_framework import permissions -from rest_framework.response import Response - -from hobo.environment.utils import get_installed_services -from hobo.rest.serializers import HealthBaseSerializer - - -class HealthList(generics.ListAPIView): - serializer_class = HealthBaseSerializer - permission_classes = (permissions.IsAuthenticatedOrReadOnly,) - - def get_queryset(self): - return [x for x in get_installed_services() if not x.secondary] - - def list(self, request, *args, **kwargs): - """ - Custom dictionary object - """ - self.object_list = self.filter_queryset(self.get_queryset()) - - # Switch between paginated or standard style responses - page = self.paginate_queryset(self.object_list) - if page is not None: - serializer = self.get_pagination_serializer(page) - else: - serializer = self.get_serializer(self.object_list, many=True) - - response = {d['slug']: d for d in serializer.data} - return Response({'data': response}) diff --git a/hobo/urls.py b/hobo/urls.py index d247818..08b9d42 100644 --- a/hobo/urls.py +++ b/hobo/urls.py @@ -4,13 +4,12 @@ from django.conf.urls import include, url from django.contrib import admin admin.autodiscover() -from .views import admin_required, login, login_local, logout, home, menu_json, hobo +from .views import admin_required, login, login_local, logout, home, health_json, menu_json, hobo from .urls_utils import decorated_includes from .environment.urls import urlpatterns as environment_urls from .profile.urls import urlpatterns as profile_urls from .theme.urls import urlpatterns as theme_urls from .emails.urls import urlpatterns as emails_urls -from .rest.urls import urlpatterns as rest_urls urlpatterns = [ url(r'^$', home, name='home'), @@ -21,7 +20,7 @@ urlpatterns = [ url(r'^theme/', decorated_includes(admin_required, include(theme_urls))), url(r'^emails/', decorated_includes(admin_required, include(emails_urls))), - url(r'^api/', include(rest_urls)), + url(r'^api/health/$', health_json, name='health-json'), url(r'^menu.json$', menu_json, name='menu_json'), url(r'^hobos.json$', hobo), url(r'^admin/', include(admin.site.urls)), diff --git a/hobo/views.py b/hobo/views.py index e293165..d869378 100644 --- a/hobo/views.py +++ b/hobo/views.py @@ -2,7 +2,7 @@ import json import urllib from django.conf import settings -from django.http import HttpResponse, HttpResponseRedirect +from django.http import HttpResponse, HttpResponseRedirect, JsonResponse from django.utils.translation import ugettext as _ from django.views.generic.base import TemplateView from django.views.generic import edit @@ -119,6 +119,12 @@ def logout(request, next_page=None): return HttpResponseRedirect(request.GET.get('next') or request.build_absolute_uri('/')) + +def health_json(request): + data = {x.slug: x.get_health_dict() for x in get_installed_services() if not x.secondary} + return JsonResponse({'data': data}) + + @admin_required def menu_json(request): response = HttpResponse(content_type='application/json') diff --git a/tests/test_health_api.py b/tests/test_health_api.py index e941eac..073ee9c 100644 --- a/tests/test_health_api.py +++ b/tests/test_health_api.py @@ -4,8 +4,11 @@ import pytest import requests import socket +from django.core.cache import cache from django.utils import timezone +from httmock import urlmatch, remember_called, HTTMock + from hobo.environment.models import Authentic, Combo pytestmark = pytest.mark.django_db @@ -22,7 +25,8 @@ def services(request): c.save() -def test_response(app, services, monkeypatch): +def test_response(app, admin_user, services, monkeypatch): + cache.clear() monkeypatch.setattr(socket, 'gethostbyname', lambda x: '176.31.123.109') monkeypatch.setattr(requests, 'get', lambda x, verify: MagicMock(status_code=200)) response = app.get('/api/health/') @@ -32,7 +36,8 @@ def test_response(app, services, monkeypatch): assert 'jazz' in content['data'].keys() -def test_is_resolvable(app, services, monkeypatch): +def test_is_resolvable(app, admin_user, services, monkeypatch): + cache.clear() def gethostname(netloc): if netloc == "jazz.example.publik": return '176.31.123.109' @@ -48,23 +53,43 @@ def test_is_resolvable(app, services, monkeypatch): assert jazz['is_resolvable'] -def test_is_running(app, services, monkeypatch): - def get(url, verify): - if url == 'https://jazz.example.publik/manage/': - return MagicMock(status_code=200) - else: - return MagicMock(status_code=404) +def test_is_running(app, admin_user, services, monkeypatch): + cache.clear() monkeypatch.setattr(socket, 'gethostbyname', lambda x: '176.31.123.109') - monkeypatch.setattr(requests, 'get', get) - response = app.get('/api/health/') - content = json.loads(response.content) - blues = content['data']['blues'] - jazz = content['data']['jazz'] - assert not blues['is_running'] - assert jazz['is_running'] + + @urlmatch(netloc='jazz.example.publik') + @remember_called + def jazz_mock(url, request): + return {'status_code': 200} + + @urlmatch(netloc='blues.example.publik') + @remember_called + def blues_mock(url, request): + return {'status_code': 404} + + with HTTMock(blues_mock, jazz_mock) as mock: + response = app.get('/api/health/') + content = json.loads(response.content) + blues = content['data']['blues'] + jazz = content['data']['jazz'] + assert not blues['is_running'] + assert jazz['is_running'] + assert blues_mock.call['count'] == 2 + assert jazz_mock.call['count'] == 2 + + # check it gets results from cache + response = app.get('/api/health/') + content = json.loads(response.content) + blues = content['data']['blues'] + jazz = content['data']['jazz'] + assert not blues['is_running'] + assert jazz['is_running'] + assert blues_mock.call['count'] == 2 + assert jazz_mock.call['count'] == 2 -def test_is_operational(app, services, monkeypatch): +def test_is_operational(app, admin_user, services, monkeypatch): + cache.clear() monkeypatch.setattr(socket, 'gethostbyname', lambda x: '176.31.123.109') monkeypatch.setattr(requests, 'get', lambda x, verify: MagicMock(status_code=200)) response = app.get('/api/health/') @@ -75,7 +100,8 @@ def test_is_operational(app, services, monkeypatch): assert not jazz['is_operational'] -def test_has_valid_certificate(app, services, monkeypatch): +def test_has_valid_certificate(app, admin_user, services, monkeypatch): + cache.clear() def get(url, verify): if 'blues.example.publik' in url or not verify: return MagicMock(status_code=200) -- 2.20.0.rc1