From 63d043659f639ffd19401f666db12ded6c1780a1 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Mon, 13 Jul 2020 17:50:44 +0200
Subject: [PATCH] manager: add tar format for site export/import (#39425)

---
 combo/manager/views.py | 36 +++++++++++++++++++++---------
 tests/test_manager.py  | 50 +++++++++++++++++++++++++++++++++++-------
 2 files changed, 68 insertions(+), 18 deletions(-)

diff --git a/combo/manager/views.py b/combo/manager/views.py
index 1b1517a3..2cdcbe64 100644
--- a/combo/manager/views.py
+++ b/combo/manager/views.py
@@ -12,37 +12,41 @@
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import hashlib
 import json
 import os
+import tarfile
+
 from operator import attrgetter
 
 from django.conf import settings
 from django.contrib import messages
 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
 from django.urls import reverse, reverse_lazy
 from django.http import HttpResponse, HttpResponseRedirect, Http404
 from django.shortcuts import redirect
 from django.shortcuts import render
 from django.shortcuts import get_object_or_404
 from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import force_text, force_bytes
 from django.utils.formats import date_format
+from django.utils.six import BytesIO
 from django.utils.timezone import localtime
 from django.views.decorators.csrf import requires_csrf_token
 from django.views.generic import (RedirectView, DetailView,
         CreateView, UpdateView, ListView, DeleteView, FormView)
 
 from combo.data.models import Page, CellBase, ParentContentCell, PageSnapshot, LinkListCell
 from combo.data.library import get_cell_class
-from combo.data.utils import export_site, import_site, MissingGroups
+from combo.data.utils import (export_site_tar, import_site, import_site_tar, ImportSiteError,
+        MissingGroups)
 from combo import plugins
 
 from .forms import (PageEditTitleForm, PageVisibilityForm, SiteImportForm,
         PageEditRedirectionForm, PageSelectTemplateForm, PageEditSlugForm,
         PageEditPictureForm, PageEditIncludeInNavigationForm,
         PageEditDescriptionForm, CellVisibilityForm, PageDuplicateForm)
 
 
@@ -60,37 +64,49 @@ class HomepageView(ListView):
 
 homepage = HomepageView.as_view()
 
 
 class SiteExportView(ListView):
     model = Page
 
     def render_to_response(self, context, **response_kwargs):
-        response = HttpResponse(content_type='application/json')
-        json.dump(export_site(), response, indent=2)
-        return response
+        fd = BytesIO()
+        export_site_tar(fd)
+        return HttpResponse(fd.getvalue(), content_type='application/x-tar')
 
 site_export = SiteExportView.as_view()
 
 
 class SiteImportView(FormView):
     form_class = SiteImportForm
     template_name = 'combo/site_import.html'
     success_url = reverse_lazy('combo-manager-homepage')
 
     def form_valid(self, form):
+        fd = self.request.FILES['site_json'].file
         try:
-            json_site = json.loads(force_text(self.request.FILES['site_json'].read()))
-        except ValueError:
-            form.add_error('site_json', _('File is not in the expected JSON format.'))
-            return self.form_invalid(form)
-
+            tarfile.open(mode='r', fileobj=fd)
+        except tarfile.TarError as e:
+            try:
+                fd.seek(0)
+                json_site = json.loads(force_text(fd.read()))
+            except ValueError:
+                form.add_error('site_json', _('File is not in the expected TAR or JSON format.'))
+                return self.form_invalid(form)
+            else:
+                format = 'json'
+        else:
+            format = 'tar'
+            fd.seek(0)
         try:
-            import_site(json_site, request=self.request)
+            if format == 'json':
+                import_site(json_site, request=self.request)
+            else:
+                import_site_tar(fd, request=self.request)
         except MissingGroups as e:
             form.add_error('site_json', force_text(e))
             return self.form_invalid(form)
 
         return super(SiteImportView, self).form_valid(form)
 
 
 site_import = SiteImportView.as_view()
diff --git a/tests/test_manager.py b/tests/test_manager.py
index 688d29e2..85d2aca1 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -1,15 +1,16 @@
 import base64
 import datetime
 import json
 import mock
 import os
 import re
 import shutil
+import tarfile
 
 from django.core.files.storage import default_storage
 from django.urls import reverse
 from django.conf import settings
 from django.contrib.auth.models import Group
 from django.db import connection
 from django.template import TemplateSyntaxError
 from django.test import override_settings
@@ -24,16 +25,17 @@ from django.utils.six.moves.urllib import parse as urlparse
 import pytest
 from webtest import Upload
 
 from combo.data.forms import LinkCellForm
 from combo.data.models import (
     Page, CellBase, TextCell, LinkCell, ConfigJsonCell, JsonCell, PageSnapshot,
     LinkListCell, ParentContentCell, MenuCell, ValidityInfo)
 from combo.apps.assets.models import Asset
+from combo.apps.assets.utils import add_tar_content
 from combo.apps.family.models import FamilyInfosCell
 from combo.apps.search.models import SearchCell
 
 pytestmark = pytest.mark.django_db
 
 TESTS_DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
 
 
@@ -546,16 +548,41 @@ def test_export_page_order():
     random_list = [page3, page4, page1, page2]
     ordered_list = Page.get_as_reordered_flat_hierarchy(random_list)
     assert ordered_list[0] == page1
     assert ordered_list[1] == page4
     assert ordered_list[2] == page2
     assert ordered_list[3] == page3
 
 
+def test_page_export_import(app, admin_user):
+    Page.objects.all().delete()
+    page1 = Page(title='One', slug='one', template_name='standard')
+    page1.save()
+
+    cell = TextCell(page=page1, placeholder='content', text='Foobar', order=0)
+    cell.save()
+
+    app = login(app)
+    resp = app.get('/manage/pages/%s/' % page1.id)
+    resp = resp.click('Export')
+    assert resp.headers['content-type'] == 'application/json'
+    site_export = resp.body
+
+    Page.objects.all().delete()
+    assert TextCell.objects.count() == 0
+    app = login(app)
+    resp = app.get('/manage/')
+    resp = resp.click('Import Site')
+    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
+    resp = resp.form.submit()
+    assert Page.objects.count() == 1
+    assert TextCell.objects.count() == 1
+
+
 def test_site_export_import(app, admin_user):
     Page.objects.all().delete()
     page1 = Page(title='One', slug='one', template_name='standard')
     page1.save()
     page2 = Page(title='Two', slug='two', parent=page1, template_name='standard')
     page2.save()
     page3 = Page(title='Three', slug='three', parent=page2, template_name='standard')
     page3.save()
@@ -569,77 +596,84 @@ def test_site_export_import(app, admin_user):
     cell.save()
 
     cell = LinkCell(page=page2, placeholder='content', link_page=page1, order=0)
     cell.save()
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Export Site')
-    assert resp.headers['content-type'] == 'application/json'
+    assert resp.headers['content-type'] == 'application/x-tar'
     site_export = resp.body
 
     Page.objects.all().delete()
     assert LinkCell.objects.count() == 0
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
+    resp.form['site_json'] = Upload('site-export.tar', site_export, 'application/x-tar')
     resp = resp.form.submit()
     assert Page.objects.count() == 4
     assert LinkCell.objects.count() == 2
     assert LinkCell.objects.get(page__slug='one').link_page.slug == 'two'
     assert LinkCell.objects.get(page__slug='two').link_page.slug == 'one'
 
     # check with invalid file
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', b'invalid content', 'application/json')
+    resp.form['site_json'] = Upload('site-export.tar', b'invalid content', 'application/x-tar')
     resp = resp.form.submit()
-    assert 'File is not in the expected JSON format.' in resp.text
+    assert 'File is not in the expected TAR or JSON format.' in resp.text
 
 
 def test_site_export_import_missing_group(app, admin_user):
     Page.objects.all().delete()
     group = Group.objects.create(name='foobar')
     page1 = Page(title='One', slug='one', template_name='standard')
     page1.save()
     page1.groups.set([group])
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Export Site')
-    assert resp.headers['content-type'] == 'application/json'
+    assert resp.headers['content-type'] == 'application/x-tar'
     site_export = resp.body
 
     Page.objects.all().delete()
     group.delete()
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
+    resp.form['site_json'] = Upload('site-export.tar', site_export, 'application/x-tar')
     resp = resp.form.submit()
     assert 'Missing groups: foobar' in resp.text
 
 
 def test_site_export_import_unknown_parent(app, admin_user):
     Page.objects.create(title='One', slug='one', template_name='standard')
     Page.objects.create(title='Two', slug='two', template_name='standard')
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Export Site')
-    payload = json.loads(force_str(resp.body))
+    tar = tarfile.open(mode='r', fileobj=BytesIO(resp.body))
+    site_json = tar.extractfile(tar.getmember('_site.json')).read()
+    payload = json.loads(force_str(site_json))
     payload['pages'][0]['fields']['exclude_from_navigation'] = False
     payload['pages'][0]['fields']['parent'] = ['unknown-parent']
 
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', force_bytes(json.dumps(payload)), 'application/json')
+    fd = BytesIO()
+    tar = tarfile.open(mode='w', fileobj=fd)
+    add_tar_content(tar, '_site.json', json.dumps(payload))
+    tar_bytes = fd.getvalue()
+    tar.close()
+    resp.form['site_json'] = Upload('site-export.json', tar_bytes, 'application/x-tar')
     resp = resp.form.submit().follow()
     assert 'Unknown parent for page &quot;One&quot;; parent has been reset and page was excluded from navigation.' in resp.text
 
 
 def test_invalid_cell_report(app, admin_user):
     app = login(app)
     resp = app.get('/manage/cells/invalid-report/')
     assert resp.context['object_list'] == []
-- 
2.27.0

