From d81cc6b798aa6d31223a5410700006d359ec5cf6 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Sun, 16 Feb 2020 17:06:30 +0100
Subject: [PATCH 3/3] general: add import/export tar format for site (#39425)

---
 combo/data/management/commands/export_site.py | 26 ++++++++++----
 combo/data/management/commands/import_site.py | 31 ++++++++++++----
 combo/data/utils.py                           | 34 +++++++++++++++++-
 combo/manager/views.py                        | 35 ++++++++++++------
 tests/test_import_export.py                   | 22 +++++++++++-
 tests/test_manager.py                         | 36 +++++++++++++++----
 tests/test_pages.py                           |  4 +--
 7 files changed, 155 insertions(+), 33 deletions(-)

diff --git a/combo/data/management/commands/export_site.py b/combo/data/management/commands/export_site.py
index cbe95f36..d9f02f2f 100644
--- a/combo/data/management/commands/export_site.py
+++ b/combo/data/management/commands/export_site.py
@@ -12,26 +12,40 @@
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
 import sys
 
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _
 
-from combo.data.utils import export_site
+from combo.data.utils import export_site, export_site_tar
 
 class Command(BaseCommand):
     help = 'Export the site'
 
     def add_arguments(self, parser):
         parser.add_argument(
                 '--output', metavar='FILE', default=None,
                 help='name of a file to write output to')
+        parser.add_argument(
+                '--format-json', action='store_true', default=False,
+                help='use JSON format with no asset files')
 
     def handle(self, *args, **options):
-        if options['output'] and options['output'] != '-':
-            output = open(options['output'], 'w')
+        if options['format_json']:
+            if options['output'] and options['output'] != '-':
+                output = open(options['output'], 'w')
+            else:
+                output = sys.stdout
+            json.dump(export_site(), output, indent=2)
         else:
-            output = sys.stdout
-        json.dump(export_site(), output, indent=2)
+            if options['output'] and options['output'] != '-':
+                try:
+                    output = open(options['output'], 'wb')
+                except IOError as e:
+                    raise CommandError(e)
+                export_site_tar(output)
+            else:
+                raise CommandError(_('TAR format require output filename parameter'))
diff --git a/combo/data/management/commands/import_site.py b/combo/data/management/commands/import_site.py
index 509a5377..9fd7c499 100644
--- a/combo/data/management/commands/import_site.py
+++ b/combo/data/management/commands/import_site.py
@@ -11,38 +11,55 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
 import sys
+import tarfile
 
 from django.core.management.base import BaseCommand, CommandError
-from django.utils.encoding import force_text
 
-from combo.data.utils import import_site, MissingGroups
+from combo.data.utils import import_site, import_site_tar, ImportSiteError
 
 class Command(BaseCommand):
     help = 'Import an exported site'
 
     def add_arguments(self, parser):
         parser.add_argument('filename', metavar='FILENAME', type=str,
                 help='name of file to import')
         parser.add_argument(
                 '--clean', action='store_true', default=False,
                 help='Clean site before importing')
         parser.add_argument(
                 '--if-empty', action='store_true', default=False,
                 help='Import only if site is empty')
 
     def handle(self, filename, *args, **options):
         if filename == '-':
+            format = 'json'
             fd = sys.stdin
         else:
-            fd = open(filename)
+            try:
+                fd = open(filename, 'rb')
+            except IOError as e:
+                raise CommandError(e)
+            try:
+                tarfile.open(mode='r', fileobj=fd)
+            except tarfile.TarError as e:
+                format = 'json'
+                fd = open(filename, 'r')
+            else:
+                format = 'tar'
+                fd = open(filename, 'rb')
         try:
-            import_site(json.load(fd),
-                        if_empty=options['if_empty'],
-                        clean=options['clean'])
-        except MissingGroups as e:
+            if format == 'json':
+                import_site(json.load(fd),
+                            if_empty=options['if_empty'],
+                            clean=options['clean'])
+            else:
+                import_site_tar(fd,
+                                if_empty=options['if_empty'],
+                                clean=options['clean'])
+        except ImportSiteError as e:
             raise CommandError(e)
diff --git a/combo/data/utils.py b/combo/data/utils.py
index adf5ed9b..cce5fc63 100644
--- a/combo/data/utils.py
+++ b/combo/data/utils.py
@@ -9,30 +9,38 @@
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import json
+import tarfile
+
 from django.contrib.auth.models import Group
 from django.db import transaction
 from django.utils import six
 from django.utils.encoding import python_2_unicode_compatible
 from django.utils.translation import ugettext_lazy as _
 
 from combo.apps.assets.models import Asset
+from combo.apps.assets.utils import (add_tar_content, clean_assets_files,
+        untar_assets_files, tar_assets_files)
 from combo.apps.maps.models import MapLayer
 from combo.apps.pwa.models import PwaSettings, PwaNavigationEntry
 from .models import Page
 
 
+class ImportSiteError(Exception):
+    pass
+
 @python_2_unicode_compatible
-class MissingGroups(Exception):
+class MissingGroups(ImportSiteError):
     def __init__(self, names):
         self.names = names
 
     def __str__(self):
         return _('Missing groups: %s') % ', '.join(self.names)
 
 
 def export_site():
@@ -80,8 +88,32 @@ def import_site(data, if_empty=False, clean=False):
 
         MapLayer.load_serialized_objects(data.get('map-layers') or [])
         Asset.load_serialized_objects(data.get('assets') or [])
         Page.load_serialized_pages(data.get('pages') or [])
 
         if data.get('pwa'):
             PwaSettings.load_serialized_settings(data['pwa'].get('settings'))
             PwaNavigationEntry.load_serialized_objects(data['pwa'].get('navigation'))
+
+
+def export_site_tar(fd):
+    tar = tarfile.open(mode='w', fileobj=fd)
+    data = export_site()
+    del data['assets']
+    add_tar_content(tar, '_site.json', json.dumps(data, indent=2))
+    tar_assets_files(tar)
+    tar.close()
+
+
+def import_site_tar(fd, if_empty=False, clean=False):
+    tar = tarfile.open(mode='r', fileobj=fd)
+    try:
+        tarinfo = tar.getmember('_site.json')
+    except KeyError:
+        raise ImportSiteError(_('TAR file should provide _site.json file'))
+    json_site = tar.extractfile(tarinfo).read()
+    data = json.loads(json_site.decode('utf-8'))
+    data.update(untar_assets_files(tar, overwrite=not if_empty))
+    if clean:
+        clean_assets_files()
+    import_site(data, if_empty=if_empty, clean=clean)
+    tar.close()
diff --git a/combo/manager/views.py b/combo/manager/views.py
index b0d30915..5302b3c2 100644
--- a/combo/manager/views.py
+++ b/combo/manager/views.py
@@ -11,35 +11,38 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import hashlib
 import json
+import tarfile
 
 from django.conf import settings
 from django.contrib import messages
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse, reverse_lazy
 from django.http import HttpResponse, HttpResponseRedirect, Http404
 from django.shortcuts import redirect
 from django.shortcuts import get_object_or_404
 from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import force_text, force_bytes
 from django.utils.formats import date_format
+from django.utils.six import BytesIO
 from django.utils.timezone import localtime
 from django.views.decorators.csrf import requires_csrf_token
 from django.views.generic import (RedirectView, DetailView,
         CreateView, UpdateView, ListView, DeleteView, FormView)
 
 from combo.data.models import Page, CellBase, ParentContentCell, PageSnapshot, LinkListCell
 from combo.data.library import get_cell_class
-from combo.data.utils import export_site, import_site, MissingGroups
+from combo.data.utils import (export_site_tar, import_site, import_site_tar, ImportSiteError,
+        MissingGroups)
 from combo import plugins
 
 from .forms import (PageEditTitleForm, PageVisibilityForm, SiteImportForm,
         PageEditRedirectionForm, PageSelectTemplateForm, PageEditSlugForm,
         PageEditPictureForm, PageEditIncludeInNavigationForm,
         PageEditDescriptionForm, CellVisibilityForm)
 
 
@@ -55,37 +58,49 @@ class HomepageView(ListView):
 
 homepage = HomepageView.as_view()
 
 
 class SiteExportView(ListView):
     model = Page
 
     def render_to_response(self, context, **response_kwargs):
-        response = HttpResponse(content_type='application/json')
-        json.dump(export_site(), response, indent=2)
-        return response
+        fd = BytesIO()
+        export_site_tar(fd)
+        return HttpResponse(fd.getvalue(), content_type='application/x-tar')
 
 site_export = SiteExportView.as_view()
 
 
 class SiteImportView(FormView):
     form_class = SiteImportForm
     template_name = 'combo/site_import.html'
     success_url = reverse_lazy('combo-manager-homepage')
 
     def form_valid(self, form):
+        fd = self.request.FILES['site_json'].file
         try:
-            json_site = json.loads(force_text(self.request.FILES['site_json'].read()))
-        except ValueError:
-            form.add_error('site_json', _('File is not in the expected JSON format.'))
-            return self.form_invalid(form)
-
+            tarfile.open(mode='r', fileobj=fd)
+        except tarfile.TarError as e:
+            try:
+                fd.seek(0)
+                json_site = json.loads(force_text(fd.read()))
+            except ValueError:
+                form.add_error('site_json', _('File is not in the expected TAR or JSON format.'))
+                return self.form_invalid(form)
+            else:
+                format = 'json'
+        else:
+            format = 'tar'
+            fd.seek(0)
         try:
-            import_site(json_site)
+            if format == 'json':
+                import_site(json_site)
+            else:
+                import_site_tar(fd)
         except MissingGroups as e:
             form.add_error('site_json', force_text(e))
             return self.form_invalid(form)
 
         return super(SiteImportView, self).form_valid(form)
 
 site_import = SiteImportView.as_view()
 
diff --git a/tests/test_import_export.py b/tests/test_import_export.py
index a4b097fe..660a1abe 100644
--- a/tests/test_import_export.py
+++ b/tests/test_import_export.py
@@ -1,13 +1,14 @@
 import base64
 import json
 import os
 import shutil
 import sys
+import tarfile
 import tempfile
 
 import pytest
 from django.contrib.auth.models import Group
 from django.core.files import File
 from django.core.management import call_command
 from django.core.management.base import CommandError
 from django.utils.encoding import force_bytes, force_text
@@ -42,17 +43,17 @@ def some_map_layers():
 @pytest.fixture
 def some_assets():
     Asset(key='banner', asset=File(BytesIO(b'test'), 'test.png')).save()
     Asset(key='favicon', asset=File(BytesIO(b'test2'), 'test2.png')).save()
 
 def get_output_of_command(command, *args, **kwargs):
     old_stdout = sys.stdout
     output = sys.stdout = StringIO()
-    call_command(command, *args, **kwargs)
+    call_command(command, format_json=True, *args, **kwargs)
     sys.stdout = old_stdout
     return output.getvalue()
 
 def test_import_export(app, some_data):
     output = get_output_of_command('export_site')
     assert len(json.loads(output)['pages']) == 3
     import_site(data={}, clean=True)
     assert Page.objects.all().count() == 0
@@ -270,8 +271,27 @@ def test_import_export_gallery_images(app, some_data):
     output = get_output_of_command('export_site')
     import_site(data={}, clean=True)
     assert Image.objects.all().count() == 0
 
     import_site(data=json.loads(output))
     assert Image.objects.all().count() == 2
     assert image1.title == 'foo'
     assert image1.image == 'path/foo.jpg'
+
+def test_import_export_tar(tmpdir):
+    filename = os.path.join(str(tmpdir), 'file.tar')
+    call_command('export_site', '--output', filename)
+    call_command('import_site', filename, '--clean')
+
+    with pytest.raises(CommandError, match=r'No such file or directory'):
+        call_command('export_site', '--output', '%s/noway/foo.tar' % tmpdir)
+
+    with pytest.raises(CommandError, match='TAR format require output filename parameter'):
+        call_command('export_site', '--output', '-')
+
+    with pytest.raises(CommandError, match=r'No such file or directory'):
+        call_command('import_site', '%s/noway/foo.tar' % tmpdir)
+
+    tar = tarfile.open(filename, 'w')
+    tar.close()
+    with pytest.raises(CommandError, match=r'TAR file should provide _site.json file'):
+        call_command('import_site', filename)
diff --git a/tests/test_manager.py b/tests/test_manager.py
index 82890338..35128f9c 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -449,16 +449,40 @@ def test_export_page_order():
     page4 = Page(title='Four', slug='four', parent=page1, template_name='standard')
     random_list = [page3, page4, page1, page2]
     ordered_list = Page.get_as_reordered_flat_hierarchy(random_list)
     assert ordered_list[0] == page1
     assert ordered_list[1] == page4
     assert ordered_list[2] == page2
     assert ordered_list[3] == page3
 
+def test_page_export_import(app, admin_user):
+    Page.objects.all().delete()
+    page1 = Page(title='One', slug='one', template_name='standard')
+    page1.save()
+
+    cell = TextCell(page=page1, placeholder='content', text='Foobar', order=0)
+    cell.save()
+
+    app = login(app)
+    resp = app.get('/manage/pages/%s/' % page1.id)
+    resp = resp.click('Export')
+    assert resp.headers['content-type'] == 'application/json'
+    site_export = resp.body
+
+    Page.objects.all().delete()
+    assert TextCell.objects.count() == 0
+    app = login(app)
+    resp = app.get('/manage/')
+    resp = resp.click('Import Site')
+    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
+    resp = resp.form.submit()
+    assert Page.objects.count() == 1
+    assert TextCell.objects.count() == 1
+
 def test_site_export_import(app, admin_user):
     Page.objects.all().delete()
     page1 = Page(title='One', slug='one', template_name='standard')
     page1.save()
     page2 = Page(title='Two', slug='two', parent=page1, template_name='standard')
     page2.save()
     page3 = Page(title='Three', slug='three', parent=page2, template_name='standard')
     page3.save()
@@ -472,58 +496,58 @@ def test_site_export_import(app, admin_user):
     cell.save()
 
     cell = LinkCell(page=page2, placeholder='content', link_page=page1, order=0)
     cell.save()
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Export Site')
-    assert resp.headers['content-type'] == 'application/json'
+    assert resp.headers['content-type'] == 'application/x-tar'
     site_export = resp.body
 
     Page.objects.all().delete()
     assert LinkCell.objects.count() == 0
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
+    resp.form['site_json'] = Upload('site-export.tar', site_export, 'application/x-tar')
     resp = resp.form.submit()
     assert Page.objects.count() == 4
     assert LinkCell.objects.count() == 2
     assert LinkCell.objects.get(page__slug='one').link_page.slug == 'two'
     assert LinkCell.objects.get(page__slug='two').link_page.slug == 'one'
 
     # check with invalid file
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', b'invalid content', 'application/json')
+    resp.form['site_json'] = Upload('site-export.tar', b'invalid content', 'application/x-tar')
     resp = resp.form.submit()
-    assert 'File is not in the expected JSON format.' in resp.text
+    assert 'File is not in the expected TAR or JSON format.' in resp.text
 
 def test_site_export_import_missing_group(app, admin_user):
     Page.objects.all().delete()
     group = Group.objects.create(name='foobar')
     page1 = Page(title='One', slug='one', template_name='standard')
     page1.save()
     page1.groups.set([group])
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Export Site')
-    assert resp.headers['content-type'] == 'application/json'
+    assert resp.headers['content-type'] == 'application/x-tar'
     site_export = resp.body
 
     Page.objects.all().delete()
     group.delete()
 
     app = login(app)
     resp = app.get('/manage/')
     resp = resp.click('Import Site')
-    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
+    resp.form['site_json'] = Upload('site-export.tar', site_export, 'application/x-tar')
     resp = resp.form.submit()
     assert 'Missing groups: foobar' in resp.text
 
 
 def test_duplicate_page(app, admin_user):
     page = Page.objects.create(title='One', slug='one', template_name='standard', exclude_from_navigation=False)
     TextCell.objects.create(page=page, placeholder='content', text='Foobar', order=0)
 
diff --git a/tests/test_pages.py b/tests/test_pages.py
index 7d57b562..145c0187 100644
--- a/tests/test_pages.py
+++ b/tests/test_pages.py
@@ -324,23 +324,23 @@ def test_import_export_management_commands():
     cell = TextCell(page=page2, text='bar', order=0, placeholder='content')
     cell.save()
 
     export_filename = os.path.join(settings.MEDIA_ROOT, 'site-export.json')
     if os.path.exists(export_filename):
         os.unlink(export_filename)
 
     cmd = ExportSiteCommand()
-    cmd.handle(output=export_filename)
+    cmd.handle(output=export_filename, format_json=True)
     assert os.path.exists(export_filename)
 
     stdout = sys.stdout
     try:
         sys.stdout = StringIO()
-        cmd.handle(output='-')
+        cmd.handle(output='-', format_json=True)
         assert sys.stdout.getvalue() == open(export_filename).read()
     finally:
         sys.stdout = stdout
 
     Page.objects.all().delete()
 
     cmd = ImportSiteCommand()
     cmd.handle(filename=export_filename, if_empty=False, clean=False)
-- 
2.25.0

