From be6a56aa2706f6063eb654d2403ff5a6a452b4f6 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 16 Dec 2019 16:23:10 +0100 Subject: [PATCH 2/2] misc: add visualization import/export (#30854) --- bijoe/management/__init__.py | 0 bijoe/management/commands/__init__.py | 0 bijoe/management/commands/export_site.py | 38 ++++++++ bijoe/management/commands/import_site.py | 44 +++++++++ bijoe/templates/bijoe/homepage.html | 14 ++- bijoe/templates/bijoe/visualization.html | 1 + bijoe/templates/bijoe/visualizations.html | 9 +- .../bijoe/visualizations_import.html | 20 ++++ bijoe/templates/bijoe/warehouse.html | 3 + bijoe/utils.py | 29 +++++- bijoe/visualization/forms.py | 4 + bijoe/visualization/models.py | 17 ++++ bijoe/visualization/urls.py | 5 + bijoe/visualization/views.py | 63 ++++++++++++- tests/test_import_export.py | 92 +++++++++++++++++++ tests/test_views.py | 71 ++++++++++++++ 16 files changed, 404 insertions(+), 6 deletions(-) create mode 100644 bijoe/management/__init__.py create mode 100644 bijoe/management/commands/__init__.py create mode 100644 bijoe/management/commands/export_site.py create mode 100644 bijoe/management/commands/import_site.py create mode 100644 bijoe/templates/bijoe/visualizations_import.html create mode 100644 tests/test_import_export.py diff --git a/bijoe/management/__init__.py b/bijoe/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bijoe/management/commands/__init__.py b/bijoe/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bijoe/management/commands/export_site.py b/bijoe/management/commands/export_site.py new file mode 100644 index 0000000..12b1bae --- /dev/null +++ b/bijoe/management/commands/export_site.py @@ -0,0 +1,38 @@ +# bijoe - BI dashboard +# Copyright (C) 2015 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +import sys + +from django.core.management.base import BaseCommand, CommandError + +from bijoe.utils import export_site + + +class Command(BaseCommand): + help = 'Export the site' + + def add_arguments(self, parser): + parser.add_argument( + '--output', metavar='FILE', default=None, + help='name of a file to write output to') + + def handle(self, *args, **options): + if options['output']: + output = open(options['output'], 'w') + else: + output = sys.stdout + json.dump(export_site(), output, indent=4) diff --git a/bijoe/management/commands/import_site.py b/bijoe/management/commands/import_site.py new file mode 100644 index 0000000..f7bb0b4 --- /dev/null +++ b/bijoe/management/commands/import_site.py @@ -0,0 +1,44 @@ +# bijoe - BI dashboard +# Copyright (C) 2015 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +import sys + +from django.core.management.base import BaseCommand + +from bijoe.utils import import_site + + +class Command(BaseCommand): + help = 'Import an exported site' + + def add_arguments(self, parser): + parser.add_argument( + 'filename', metavar='FILENAME', type=str, + help='name of file to import') + parser.add_argument( + '--clean', action='store_true', default=False, + help='Clean site before importing') + parser.add_argument( + '--if-empty', action='store_true', default=False, + help='Import only if site is empty') + + def handle(self, filename, **options): + if filename == '-': + fd = sys.stdin + else: + fd = open(filename) + import_site(json.load(fd), if_empty=options['if_empty'], clean=options['clean']) diff --git a/bijoe/templates/bijoe/homepage.html b/bijoe/templates/bijoe/homepage.html index 0eecec2..2fe0b01 100644 --- a/bijoe/templates/bijoe/homepage.html +++ b/bijoe/templates/bijoe/homepage.html @@ -6,10 +6,22 @@ {% trans "Homepage" %} {% endblock %} +{% block appbar %} +

{% trans "Visualizations" %}

+ + + + +{% endblock %} + {% block content %} {% if visualizations %} -

{% trans "Visualizations" %}

{% include "bijoe/visualizations_list.html" %} + {% else %} + {% trans "No visualizations to display yet." %} {% endif %} {% if warehouses %}

{% trans "Data sources" %}

diff --git a/bijoe/templates/bijoe/visualization.html b/bijoe/templates/bijoe/visualization.html index 9692295..d81c6e7 100644 --- a/bijoe/templates/bijoe/visualization.html +++ b/bijoe/templates/bijoe/visualization.html @@ -16,6 +16,7 @@ {% trans "Rename" %} {% trans "Delete" %} {% trans "Export as ODS" %} + {% trans "Export as JSON" %} {% trans "URL for IFRAME" %} {% endblock %} diff --git a/bijoe/templates/bijoe/visualizations.html b/bijoe/templates/bijoe/visualizations.html index bd3f1c4..c821dbd 100644 --- a/bijoe/templates/bijoe/visualizations.html +++ b/bijoe/templates/bijoe/visualizations.html @@ -7,7 +7,14 @@ {% endblock %} {% block appbar %} -

{% trans "Visualizations" %}

+

{% trans "Visualizations" %}

+ + + + {% endblock %} {% block content %} diff --git a/bijoe/templates/bijoe/visualizations_import.html b/bijoe/templates/bijoe/visualizations_import.html new file mode 100644 index 0000000..70d2f53 --- /dev/null +++ b/bijoe/templates/bijoe/visualizations_import.html @@ -0,0 +1,20 @@ +{% extends "bijoe/base.html" %} +{% load i18n %} + +{% block appbar %} +

{% trans "Visualizations Import" %}

+{% endblock %} + +{% block content %} +
+ {% csrf_token %} + {{ form.as_p }} +
+ + {% trans 'Cancel' %} +
+
+{% endblock %} + + + diff --git a/bijoe/templates/bijoe/warehouse.html b/bijoe/templates/bijoe/warehouse.html index 6625292..a3f05ae 100644 --- a/bijoe/templates/bijoe/warehouse.html +++ b/bijoe/templates/bijoe/warehouse.html @@ -28,4 +28,7 @@ {% endfor %} +{% if not visualizations_exist %} + {% trans 'Import' %} +{% endif %} {% endblock %} diff --git a/bijoe/utils.py b/bijoe/utils.py index a4afa53..faa9600 100644 --- a/bijoe/utils.py +++ b/bijoe/utils.py @@ -19,14 +19,13 @@ import glob import json from django.conf import settings -from django.db import connection +from django.db import connection, transaction from django.utils.translation import ugettext as _ try: from functools import lru_cache except ImportError: from django.utils.lru_cache import lru_cache - from .schemas import Warehouse @@ -63,3 +62,29 @@ def human_join(l): if len(l) > 2: l = u', '.join(l[:-1]), l[-1] return _(u'{0} and {1}').format(l[0], l[1]) + + +def export_site(): + from bijoe.visualization.models import Visualization + + return {'visualizations': [v.export_json() for v in Visualization.objects.all()]} + + +def import_site(data, if_empty=False, clean=False): + from bijoe.visualization.models import Visualization + + if if_empty and Visualization.objects.exists(): + return + + if clean: + Visualization.objects.all().delete() + + results = {'created': 0, 'updated': 0} + with transaction.atomic(): + for data in data.get('visualizations', []): + created = Visualization.import_json(data) + if created: + results['created'] += 1 + else: + results['updated'] += 1 + return results diff --git a/bijoe/visualization/forms.py b/bijoe/visualization/forms.py index cefd33a..c4e36fe 100644 --- a/bijoe/visualization/forms.py +++ b/bijoe/visualization/forms.py @@ -235,3 +235,7 @@ class CubeForm(forms.Form): raise ValidationError({'loop': _('You cannot use the same dimension for looping and' ' grouping')}) return cleaned_data + + +class VisualizationsImportForm(forms.Form): + visualizations_json = forms.FileField(label=_('Visualizations Export File')) diff --git a/bijoe/visualization/models.py b/bijoe/visualization/models.py index 98d16a1..4bbfa08 100644 --- a/bijoe/visualization/models.py +++ b/bijoe/visualization/models.py @@ -54,6 +54,23 @@ class Visualization(models.Model): def natural_key(self): return (self.slug,) + def export_json(self): + visualization = { + 'slug': self.slug, + 'name': self.name, + 'parameters': self.parameters + } + return visualization + + @classmethod + def import_json(cls, data): + defaults = { + 'name': data['name'], + 'parameters': data['parameters'] + } + _, created = cls.objects.update_or_create(slug=data['slug'], defaults=defaults) + return created + def save(self, *args, **kwargs): if not self.slug: slug = base_slug = slugify(self.name)[:40].strip('-') diff --git a/bijoe/visualization/urls.py b/bijoe/visualization/urls.py index d55313b..9833d13 100644 --- a/bijoe/visualization/urls.py +++ b/bijoe/visualization/urls.py @@ -23,6 +23,10 @@ urlpatterns = [ views.visualizations, name='visualizations'), url(r'^json/$', views.visualizations_json, name='visualizations-json'), + url(r'^import/$', + views.visualizations_import, name='visualizations-import'), + url(r'^export$', + views.visualizations_export, name='visualizations-export'), url(r'^warehouse/(?P[^/]*)/$', views.warehouse, name='warehouse'), url(r'^warehouse/(?P[^/]*)/(?P[^/]*)/$', views.cube, name='cube'), url(r'^warehouse/(?P[^/]*)/(?P[^/]*)/iframe/$', views.cube_iframe, @@ -36,4 +40,5 @@ urlpatterns = [ url(r'(?P\d+)/ods/$', views.visualization_ods, name='visualization-ods'), url(r'(?P\d+)/rename/$', views.rename_visualization, name='rename-visualization'), url(r'(?P\d+)/delete/$', views.delete_visualization, name='delete-visualization'), + url(r'(?P\d+)/export$', views.export_visualization, name='export-visualization'), ] diff --git a/bijoe/visualization/views.py b/bijoe/visualization/views.py index a9b113a..b2118ad 100644 --- a/bijoe/visualization/views.py +++ b/bijoe/visualization/views.py @@ -19,9 +19,11 @@ import hashlib import json from django.conf import settings +from django.contrib import messages from django.utils.encoding import force_text from django.utils.text import slugify -from django.views.generic.edit import CreateView, DeleteView, UpdateView +from django.utils.translation import ungettext, ugettext as _ +from django.views.generic.edit import CreateView, DeleteView, UpdateView, FormView from django.views.generic.list import MultipleObjectMixin from django.views.generic import DetailView, ListView, View, TemplateView from django.shortcuts import redirect @@ -33,7 +35,7 @@ from django.views.decorators.clickjacking import xframe_options_exempt from rest_framework import generics from rest_framework.response import Response -from ..utils import get_warehouses +from bijoe.utils import get_warehouses, import_site, export_site from ..engine import Engine from . import models, forms, signature from .utils import Visualization @@ -52,6 +54,7 @@ class WarehouseView(views.AuthorizationMixin, TemplateView): raise Http404 ctx['warehouse'] = Engine(warehouse) ctx['cubes'] = sorted(ctx['warehouse'].cubes, key=lambda cube: cube.label.strip().lower()) + ctx['visualizations_exist'] = models.Visualization.objects.exists() return ctx @@ -341,6 +344,59 @@ class VisualizationJSONView(generics.GenericAPIView): }) +class ExportVisualizationView(views.AuthorizationMixin, DetailView): + model = models.Visualization + + def get(self, request, *args, **kwargs): + response = HttpResponse(content_type='application/json') + json.dump({'visualizations': [self.get_object().export_json()]}, response, indent=2) + return response + + +class VisualizationsImportView(views.AuthorizationMixin, FormView): + form_class = forms.VisualizationsImportForm + template_name = 'bijoe/visualizations_import.html' + success_url = reverse_lazy('homepage') + + def form_valid(self, form): + try: + visualizations_json = json.loads( + force_text(self.request.FILES['visualizations_json'].read())) + except ValueError: + form.add_error('visualizations_json', _('File is not in the expected JSON format.')) + return self.form_invalid(form) + + results = import_site(visualizations_json) + + if results.get('created') == 0 and results.get('updated') == 0: + messages.info(self.request, _('No visualizations were found.')) + else: + if results.get('created') == 0: + message1 = _('No visualization created.') + else: + message1 = ungettext('A visualization has been created.', + '%(count)d visualizations have been created.', results['created']) % { + 'count': results['created']} + + if results.get('updated') == 0: + message2 = _('No visualization updated.') + else: + message2 = ungettext('A visualization has been updated.', + '%(count)d visualizations have been updated.', results['updated']) % { + 'count': results['updated']} + messages.info(self.request, u'%s %s' % (message1, message2)) + + return super(VisualizationsImportView, self).form_valid(form) + + +class VisualizationsExportView(views.AuthorizationMixin, View): + + def get(self, request, *args, **kwargs): + response = HttpResponse(content_type='application/json') + json.dump(export_site(), response, indent=2) + return response + + warehouse = WarehouseView.as_view() cube = CubeView.as_view() cube_iframe = xframe_options_exempt(CubeIframeView.as_view()) @@ -349,11 +405,14 @@ visualizations_json = VisualizationsJSONView.as_view() create_visualization = CreateVisualizationView.as_view() delete_visualization = DeleteVisualizationView.as_view() rename_visualization = RenameVisualization.as_view() +export_visualization = ExportVisualizationView.as_view() visualization = VisualizationView.as_view() visualization_iframe = xframe_options_exempt(VisualizationIframeView.as_view()) visualization_geojson = VisualizationGeoJSONView.as_view() visualization_ods = VisualizationODSView.as_view() visualization_json = VisualizationJSONView.as_view() +visualizations_import = VisualizationsImportView.as_view() +visualizations_export = VisualizationsExportView.as_view() cube_iframe.mellon_no_passive = True visualization_iframe.mellon_no_passive = True diff --git a/tests/test_import_export.py b/tests/test_import_export.py new file mode 100644 index 0000000..e251f23 --- /dev/null +++ b/tests/test_import_export.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +import json +import os +import shutil +import sys +import tempfile + +import pytest +from django.core.management import call_command +from django.utils.encoding import force_bytes +from django.utils.six import StringIO + +from bijoe.visualization.models import Visualization +from bijoe.utils import import_site + +pytestmark = pytest.mark.django_db + + +def get_output_of_command(command, *args, **kwargs): + old_stdout = sys.stdout + output = sys.stdout = StringIO() + call_command(command, *args, **kwargs) + sys.stdout = old_stdout + return output.getvalue() + + +def test_import_export(schema1, app): + parameters = { + 'cube': 'facts1', + 'warehouse': 'schema1', + 'measure': 'duration', + 'representation': 'table', + 'loop': '', + 'filters': {}, + 'drilldown_x': 'date__yearmonth' + } + + def create_visu(i=0): + Visualization.objects.create(name='test' + str(i), parameters=parameters) + + for i in range(3): + create_visu(i) + output = get_output_of_command('export_site') + assert len(json.loads(output)['visualizations']) == 3 + + import_site(data={}, clean=True) + empty_output = get_output_of_command('export_site') + assert len(json.loads(empty_output)['visualizations']) == 0 + + create_visu() + old_stdin = sys.stdin + sys.stdin = StringIO(json.dumps({})) + assert Visualization.objects.count() == 1 + try: + call_command('import_site', '-', clean=True) + finally: + sys.stdin = old_stdin + assert Visualization.objects.count() == 0 + + with tempfile.NamedTemporaryFile() as f: + f.write(force_bytes(output)) + f.flush() + call_command('import_site', f.name) + assert Visualization.objects.count() == 3 + for i in range(3): + visu = Visualization.objects.get(name='test' + str(i)) + assert visu.parameters == parameters + + visu = Visualization.objects.get(name='test0') + slug = visu.slug + visu_json = visu.export_json() + visu_json['name'] = 'new_name' + visu_json['parameters']['measure'] = 'test' + result = import_site(data={'visualizations': [visu_json]}) + assert result['created'] == 0 and result['updated'] == 1 + visu = Visualization.objects.get(slug=slug) + assert visu.name == 'new_name' + new_params = visu.parameters + assert new_params.pop('measure') == 'test' + assert new_params == {k: v for k, v in parameters.items() if k != 'measure'} + + import_site(data={}, if_empty=True) + assert Visualization.objects.count() == 3 + + import_site(data={}, clean=True) + tempdir = tempfile.mkdtemp('bijoe-test') + empty_output = get_output_of_command('export_site', output=os.path.join(tempdir, 't.json')) + assert os.path.exists(os.path.join(tempdir, 't.json')) + shutil.rmtree(tempdir) diff --git a/tests/test_views.py b/tests/test_views.py index f3e1c72..ac1ace6 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -14,7 +14,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import copy +import json + import pytest +from webtest import Upload from django.core.urlresolvers import reverse @@ -97,3 +101,70 @@ def test_missing_data(schema1, app, admin, visualization): response = app.get('/') assert response.pyquery('ul li a.disabled').text() == visualization.name + + +def test_import_visualization(schema1, app, admin, visualization): + login(app, admin) + resp = app.get('/visualization/%s/' % visualization.id) + resp = resp.click('Export as JSON') + assert resp.headers['content-type'] == 'application/json' + visualization_export = resp.text + + # invalid json + resp = app.get('/', status=200) + resp = resp.click('Import') + resp.form['visualizations_json'] = Upload('export.json', b'garbage', 'application/json') + resp = resp.form.submit() + assert 'File is not in the expected JSON format.' in resp.text + + # empty json + resp = app.get('/', status=200) + resp = resp.click('Import') + resp.form['visualizations_json'] = Upload('export.json', b'{}', 'application/json') + resp = resp.form.submit().follow() + assert 'No visualizations were found.' in resp.text + + # existing visualization + resp = app.get('/', status=200) + resp = resp.click('Import') + resp.form['visualizations_json'] = Upload('export.json', visualization_export.encode('utf-8'), 'application/json') + resp = resp.form.submit().follow() + assert 'No visualization created. A visualization has been updated.' in resp.text + assert Visualization.objects.count() == 1 + + # new visualization + Visualization.objects.all().delete() + resp = app.get('/').follow() + resp = resp.click('Import') + resp.form['visualizations_json'] = Upload('export.json', visualization_export.encode('utf-8'), 'application/json') + resp = resp.form.submit().follow() + assert 'A visualization has been created. No visualization updated.' in resp.text + assert Visualization.objects.count() == 1 + + # multiple visualizations + visualizations = json.loads(visualization_export) + visualizations['visualizations'].append(copy.copy(visualizations['visualizations'][0])) + visualizations['visualizations'].append(copy.copy(visualizations['visualizations'][0])) + visualizations['visualizations'][1]['name'] = 'test 2' + visualizations['visualizations'][1]['slug'] = 'test-2' + visualizations['visualizations'][2]['name'] = 'test 3' + visualizations['visualizations'][2]['slug'] = 'test-3' + + resp = app.get('/', status=200) + resp = resp.click('Import') + resp.form['visualizations_json'] = Upload('export.json', json.dumps(visualizations).encode('utf-8'), 'application/json') + resp = resp.form.submit().follow() + assert '2 visualizations have been created. A visualization has been updated.' in resp.text + assert Visualization.objects.count() == 3 + + # global export/import + resp = app.get('/').click('Export') + visualizations_export = resp.text + Visualization.objects.all().delete() + + resp = app.get('/').follow() + resp = resp.click('Import') + resp.form['visualizations_json'] = Upload('export.json', visualizations_export.encode('utf-8'), 'application/json') + resp = resp.form.submit().follow() + assert '3 visualizations have been created. No visualization updated.' in resp.text + assert Visualization.objects.count() == 3 -- 2.20.1