From a9e5e41d9651283d9aced4bc275478e384df6714 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Sat, 25 Jul 2020 20:54:33 +0200
Subject: [PATCH] manager: add tar format for site export/import (#45128)

---
 combo/manager/forms.py                        |  6 +-
 .../manager/templates/combo/manager_home.html |  2 +-
 .../manager/templates/combo/site_export.html  | 17 ++++++
 combo/manager/views.py                        | 55 ++++++++++++-----
 tests/test_manager.py                         | 59 +++++++++++++++++--
 5 files changed, 116 insertions(+), 23 deletions(-)
 create mode 100644 combo/manager/templates/combo/site_export.html

diff --git a/combo/manager/forms.py b/combo/manager/forms.py
index d98ba08e..4ad0fa8a 100644
--- a/combo/manager/forms.py
+++ b/combo/manager/forms.py
@@ -187,9 +187,13 @@ class CellVisibilityForm(forms.Form):
             self.instance.public = False
             self.instance.restricted_to_unlogged = True
             self.instance.groups.set(self.cleaned_data['groups'])
         self.instance.save()
         return self.instance
 
 
 class SiteImportForm(forms.Form):
-    site_json = forms.FileField(label=_('Site Export File'))
+    site_file = forms.FileField(label=_('Site Export File'))
+
+
+class SiteExportForm(forms.Form):
+    include_asset = forms.BooleanField(label=_('Include assets into the export'), required=False)
diff --git a/combo/manager/templates/combo/manager_home.html b/combo/manager/templates/combo/manager_home.html
index 42ec558a..adf50f67 100644
--- a/combo/manager/templates/combo/manager_home.html
+++ b/combo/manager/templates/combo/manager_home.html
@@ -2,17 +2,17 @@
 {% load i18n %}
 
 {% block appbar %}
 <h2>{% trans 'Pages' %}</h2>
 <span class="actions">
 <a class="extra-actions-menu-opener"></a>
 <a rel="popup" href="{% url 'combo-manager-page-add' %}">{% trans 'New' %}</a>
 <ul class="extra-actions-menu">
- <li><a download href="{% url 'combo-manager-site-export' %}">{% trans 'Export Site' %}</a></li>
+ <li><a href="{% url 'combo-manager-site-export' %}" rel="popup" data-autoclose-dialog="true">{% trans 'Export Site' %}</a></li>
  <li><a href="{% url 'combo-manager-site-import' %}">{% trans 'Import Site' %}</a></li>
  <li><a href="{% url 'combo-manager-invalid-cell-report' %}">{% trans 'Anomaly report' %}</a></li>
  {% for extra_action in extra_actions %}
  <li><a href="{{ extra_action.href }}">{{ extra_action.text }}</a></li>
  {% endfor %}
 </ul>
 </span>
 {% endblock %}
diff --git a/combo/manager/templates/combo/site_export.html b/combo/manager/templates/combo/site_export.html
new file mode 100644
index 00000000..1eaa8fd6
--- /dev/null
+++ b/combo/manager/templates/combo/site_export.html
@@ -0,0 +1,17 @@
+{% extends "combo/manager_base.html" %}
+{% load i18n %}
+
+{% block appbar %}
+<h2>{% trans "Site Export" %}</h2>
+{% endblock %}
+
+{% block content %}
+<form method="post">
+  {% csrf_token %}
+  {{ form.as_p }}
+  <div class="buttons">
+    <button>{% trans 'Export' %}</button>
+    <a class="cancel" href="{% url 'combo-manager-homepage' %}">{% trans 'Cancel' %}</a>
+  </div>
+</form>
+{% endblock %}
diff --git a/combo/manager/views.py b/combo/manager/views.py
index 1b1517a3..9c9e1ec1 100644
--- a/combo/manager/views.py
+++ b/combo/manager/views.py
@@ -12,40 +12,44 @@
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import hashlib
 import json
 import os
+import tarfile
+
 from operator import attrgetter
 
 from django.conf import settings
 from django.contrib import messages
 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
 from django.urls import reverse, reverse_lazy
 from django.http import HttpResponse, HttpResponseRedirect, Http404
 from django.shortcuts import redirect
 from django.shortcuts import render
 from django.shortcuts import get_object_or_404
 from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import force_text, force_bytes
 from django.utils.formats import date_format
+from django.utils.six import BytesIO
 from django.utils.timezone import localtime
 from django.views.decorators.csrf import requires_csrf_token
 from django.views.generic import (RedirectView, DetailView,
-        CreateView, UpdateView, ListView, DeleteView, FormView)
+        CreateView, UpdateView, ListView, DeleteView, FormView, TemplateView)
 
 from combo.data.models import Page, CellBase, ParentContentCell, PageSnapshot, LinkListCell
 from combo.data.library import get_cell_class
-from combo.data.utils import export_site, import_site, MissingGroups
+from combo.data.utils import (export_site, export_site_tar, import_site, import_site_tar,
+        ImportSiteError, MissingGroups)
 from combo import plugins
 
-from .forms import (PageEditTitleForm, PageVisibilityForm, SiteImportForm,
+from .forms import (PageEditTitleForm, PageVisibilityForm, SiteImportForm, SiteExportForm,
         PageEditRedirectionForm, PageSelectTemplateForm, PageEditSlugForm,
         PageEditPictureForm, PageEditIncludeInNavigationForm,
         PageEditDescriptionForm, CellVisibilityForm, PageDuplicateForm)
 
 
 class HomepageView(ListView):
     model = Page
     template_name = 'combo/manager_home.html'
@@ -56,43 +60,64 @@ class HomepageView(ListView):
         context['extra_actions'] = plugins.get_extra_manager_actions()
         context['collapse_pages'] = settings.COMBO_MANAGE_HOME_COLLAPSE_PAGES
         return context
 
 
 homepage = HomepageView.as_view()
 
 
-class SiteExportView(ListView):
-    model = Page
+class SiteExportView(FormView):
+    form_class = SiteExportForm
+    template_name = 'combo/site_export.html'
 
-    def render_to_response(self, context, **response_kwargs):
-        response = HttpResponse(content_type='application/json')
-        json.dump(export_site(), response, indent=2)
+    def post(self, request, *args, **kwargs):
+        if request.POST.get('include_asset'):
+            fd = BytesIO()
+            export_site_tar(fd)
+            response = HttpResponse(content=fd.getvalue(), content_type='application/x-tar')
+            response['Content-Disposition'] = 'attachment; filename="site-export.tar"'
+        else:
+            response = HttpResponse(content_type='application/json')
+            response['Content-Disposition'] = 'attachment; filename="site-export.json"'
+            json.dump(export_site(), response, indent=2)
         return response
 
+
 site_export = SiteExportView.as_view()
 
 
 class SiteImportView(FormView):
     form_class = SiteImportForm
     template_name = 'combo/site_import.html'
     success_url = reverse_lazy('combo-manager-homepage')
 
     def form_valid(self, form):
+        fd = self.request.FILES['site_file'].file
         try:
-            json_site = json.loads(force_text(self.request.FILES['site_json'].read()))
-        except ValueError:
-            form.add_error('site_json', _('File is not in the expected JSON format.'))
-            return self.form_invalid(form)
-
+            tarfile.open(mode='r', fileobj=fd)
+        except tarfile.TarError as e:
+            try:
+                fd.seek(0)
+                json_site = json.loads(force_text(fd.read()))
+            except ValueError:
+                form.add_error('site_file', _('File is not in the expected TAR or JSON format.'))
+                return self.form_invalid(form)
+            else:
+                format = 'json'
+        else:
+            format = 'tar'
+            fd.seek(0)
         try:
-            import_site(json_site, request=self.request)
+            if format == 'json':
+                import_site(json_site, request=self.request)
+            else:
+                import_site_tar(fd, request=self.request)
         except MissingGroups as e:
-            form.add_error('site_json', force_text(e))
+            form.add_error('site_file', force_text(e))
             return self.form_invalid(form)
 
         return super(SiteImportView, self).form_valid(form)
 
 
 site_import = SiteImportView.as_view()
 
 
diff --git a/tests/test_manager.py b/tests/test_manager.py
index 2fe303f5..fea7cc46 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -1,16 +1,17 @@
 import base64
 import datetime
 import json
 import mock
 import os
 import re
 import shutil
 
+from django.core.files import File
 from django.core.files.storage import default_storage
 from django.urls import reverse
 from django.conf import settings
 from django.contrib.auth.models import Group
 from django.db import connection
 from django.template import TemplateSyntaxError
 from django.test import override_settings
 from django.test.client import RequestFactory
@@ -551,17 +552,17 @@ def test_export_page_order():
     random_list = [page3, page4, page1, page2]
     ordered_list = Page.get_as_reordered_flat_hierarchy(random_list)
     assert ordered_list[0] == page1
     assert ordered_list[1] == page4
     assert ordered_list[2] == page2
     assert ordered_list[3] == page3
 
 
-def test_site_export_import(app, admin_user):
+def test_site_export_import_json(app, admin_user):
     Page.objects.all().delete()
     page1 = Page(title='One', slug='one', template_name='standard')
     page1.save()
     page2 = Page(title='Two', slug='two', parent=page1, template_name='standard')
     page2.save()
     page3 = Page(title='Three', slug='three', parent=page2, template_name='standard')
     page3.save()
     page4 = Page(title='Four', slug='four', parent=page1, template_name='standard')
@@ -574,77 +575,123 @@ def test_site_export_import(app, admin_user):
     cell.save()
 
     cell = LinkCell(page=page2, placeholder='content', link_page=page1, order=0)
     cell.save()
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Export Site')
+    resp = resp.form.submit()
     assert resp.headers['content-type'] == 'application/json'
     site_export = resp.body
 
     Page.objects.all().delete()
     assert LinkCell.objects.count() == 0
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
+    resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json')
     resp = resp.form.submit()
     assert Page.objects.count() == 4
     assert LinkCell.objects.count() == 2
     assert LinkCell.objects.get(page__slug='one').link_page.slug == 'two'
     assert LinkCell.objects.get(page__slug='two').link_page.slug == 'one'
 
     # check with invalid file
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', b'invalid content', 'application/json')
+    resp.form['site_file'] = Upload('site-export.json', b'invalid content', 'application/json')
+    resp = resp.form.submit()
+    assert 'File is not in the expected TAR or JSON format.' in resp.text
+
+
+def test_site_export_import_tar(app, admin_user):
+    Page.objects.all().delete()
+    page1 = Page(title='One', slug='one', template_name='standard')
+    page1.save()
+    cell = TextCell(page=page1, placeholder='content', text='Foobar', order=0)
+    cell.save()
+    Asset(key='collectivity:banner', asset=File(BytesIO(b'test'), 'test.png')).save()
+    path = default_storage.path('')
+    assert open('%s/assets/test.png' % path, 'r').read() == 'test'
+
+    app = login(app)
+    resp = app.get('/manage/')
+    resp = resp.click('Export Site')
+    resp.form['include_asset'] = True
+    resp = resp.form.submit()
+    assert resp.headers['content-type'] == 'application/x-tar'
+    site_export = resp.body
+
+    Page.objects.all().delete()
+    Asset.objects.filter(key='collectivity:banner').delete()
+    assert Page.objects.count() == 0
+    assert TextCell.objects.count() == 0
+    assert Asset.objects.filter(key='collectivity:banner').count() == 0
+    open('%s/assets/test.png' % path, 'w').write('foo')
+    app = login(app)
+    resp = app.get('/manage/')
+    resp = resp.click('Import Site')
+    resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json')
+    resp = resp.form.submit()
+    assert Page.objects.count() == 1
+    assert TextCell.objects.count() == 1
+    assert Asset.objects.filter(key='collectivity:banner').count() == 1
+    assert open('%s/assets/test.png' % path, 'r').read() == 'foo'
+
+    os.remove('%s/assets/test.png' % path)
+    app = login(app)
+    resp = app.get('/manage/')
+    resp = resp.click('Import Site')
+    resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json')
     resp = resp.form.submit()
-    assert 'File is not in the expected JSON format.' in resp.text
+    assert open('%s/assets/test.png' % path, 'r').read() == 'test'
 
 
 def test_site_export_import_missing_group(app, admin_user):
     Page.objects.all().delete()
     group = Group.objects.create(name='foobar')
     page1 = Page(title='One', slug='one', template_name='standard')
     page1.save()
     page1.groups.set([group])
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Export Site')
+    resp = resp.form.submit()
     assert resp.headers['content-type'] == 'application/json'
     site_export = resp.body
 
     Page.objects.all().delete()
     group.delete()
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
+    resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json')
     resp = resp.form.submit()
     assert 'Missing groups: foobar' in resp.text
 
 
 def test_site_export_import_unknown_parent(app, admin_user):
     Page.objects.create(title='One', slug='one', template_name='standard')
     Page.objects.create(title='Two', slug='two', template_name='standard')
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Export Site')
+    resp = resp.form.submit()
     payload = json.loads(force_str(resp.body))
     payload['pages'][0]['fields']['exclude_from_navigation'] = False
     payload['pages'][0]['fields']['parent'] = ['unknown-parent']
 
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', force_bytes(json.dumps(payload)), 'application/json')
+    resp.form['site_file'] = Upload('site-export.json', force_bytes(json.dumps(payload)), 'application/json')
     resp = resp.form.submit().follow()
     assert 'Unknown parent for page &quot;One&quot;; parent has been reset and page was excluded from navigation.' in resp.text
 
 
 def test_invalid_cell_report(app, admin_user):
     app = login(app)
     resp = app.get('/manage/cells/invalid-report/')
     assert resp.context['object_list'] == []
-- 
2.27.0

