From 4f31d8bbad85a83e9dc7c51e1f95cef31bf97bb9 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Tue, 9 Feb 2021 11:54:43 +0100
Subject: [PATCH] assets: add content exclusion options to export command
 (#50399)

---
 combo/data/management/commands/export_site.py | 39 ++++++++---
 combo/data/utils.py                           |  6 +-
 tests/test_assets.py                          |  1 +
 tests/test_import_export.py                   | 64 +++++++++++++------
 4 files changed, 79 insertions(+), 31 deletions(-)

diff --git a/combo/data/management/commands/export_site.py b/combo/data/management/commands/export_site.py
index 5ec359f2..7b62af47 100644
--- a/combo/data/management/commands/export_site.py
+++ b/combo/data/management/commands/export_site.py
@@ -27,26 +27,47 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
                 '--output', metavar='FILE', default=None,
                 help='name of a file to write output to')
         parser.add_argument(
                 '--format-json', action='store_true', default=False,
                 help='use JSON format with no asset files')
+        parser.add_argument(
+                '--exclude-pages', action='store_true', default=False,
+                help='exclude pages from the export')
+        parser.add_argument(
+                '--exclude-cartography', action='store_true', default=False,
+                help='exclude cartography from the export')
+        parser.add_argument(
+                '--exclude-pwa', action='store_true', default=False,
+                help='exclude pwa from the export')
+        parser.add_argument(
+                '--exclude-assets', action='store_true', default=False,
+                help='exclude assets from the export')
+        parser.add_argument(
+                '--exclude-payment', action='store_true', default=False,
+                help='exclude payment from the export')
 
     def handle(self, *args, **options):
-        if options['format_json']:
-            if options['output'] and options['output'] != '-':
-                output = open(options['output'], 'w')
+        params = {}
+        for content in 'pages', 'cartography', 'pwa', 'assets', 'payment':
+            if options.get('exclude_%s' % content):
+                params[content] = False
+        output = options.get('output') or '-'
+
+        if options.get('format_json'):
+            if output != '-':
+                file_fd = open(output, 'w')
             else:
-                output = sys.stdout
-            json.dump(export_site(), output, indent=2)
+                file_fd = sys.stdout
+            json.dump(export_site(**params), file_fd, indent=2)
         else:
-            if options['output'] and options['output'] != '-':
+            if output != '-':
                 try:
-                    output = open(options['output'], 'wb')
+                    file_fd = open(output, 'wb')
                 except IOError as e:
                     raise CommandError(e)
-                export_site_tar(output)
-                output.close()
+                export_site_tar(file_fd, **params)
+                file_fd.close()
             else:
                 raise CommandError(_('TAR format require output filename parameter'))
diff --git a/combo/data/utils.py b/combo/data/utils.py
index d3c0e030..f87923cd 100644
--- a/combo/data/utils.py
+++ b/combo/data/utils.py
@@ -135,19 +135,21 @@ def import_site(data, if_empty=False, clean=False, request=None):
             except IndexError:
                 raise ImportSiteError(message)
             raise ImportSiteError(_('Unknown page "%s".') % page_slug)
 
 
 def export_site_tar(fd, export_kwargs=None):
     tar = tarfile.open(mode='w', fileobj=fd)
     data = export_site(**(export_kwargs or {}))
-    del data['assets']
+    if data.get('assets'):
+        del data['assets']
     add_tar_content(tar, '_site.json', json.dumps(data, indent=2))
-    tar_assets_files(tar)
+    if not (export_kwargs and export_kwargs.get('exclude_assets')):
+        tar_assets_files(tar)
     tar.close()
 
 
 def import_site_tar(fd, if_empty=False, clean=False, overwrite=False, request=None):
     tar = tarfile.open(mode='r', fileobj=fd)
     try:
         tarinfo = tar.getmember('_site.json')
     except KeyError:
diff --git a/tests/test_assets.py b/tests/test_assets.py
index 41ad5659..ca0688fa 100644
--- a/tests/test_assets.py
+++ b/tests/test_assets.py
@@ -138,8 +138,9 @@ def test_import_export_assets(some_assets, tmpdir):
     clean_assets_files()
     assert count_asset_files() == 0
     clean_assets_files()
 
 
 def test_assets_export_size_view(app, some_assets):
     resp = app.get(reverse('combo-manager-assets-export-size'))
     assert resp.text.split() == ['(9', 'bytes)']
+    clean_assets_files()
diff --git a/tests/test_import_export.py b/tests/test_import_export.py
index a9b9e4c3..233f6e8d 100644
--- a/tests/test_import_export.py
+++ b/tests/test_import_export.py
@@ -37,36 +37,47 @@ def some_data():
     cell.save()
 
 
 @pytest.fixture
 def some_map_layers():
     MapLayer(label='Foo', slug='foo', geojson_url='http://example.net/foo/').save()
     MapLayer(label='Bar', slug='bar', geojson_url='http://example.net/bar/').save()
 
+
 @pytest.fixture
 def some_assets():
     Asset(key='banner', asset=File(BytesIO(b'test'), 'test.png')).save()
     Asset(key='favicon', asset=File(BytesIO(b'test2'), 'test2.png')).save()
 
+
 def get_output_of_command(command, *args, **kwargs):
+    options = {
+        'exclude_pages': True,
+        'exclude_cartography': True,
+        'exclude_pwa': True,
+        'exclude_assets': True,
+        'exclude_payment': True,
+    }
+    options.update(kwargs)
     old_stdout = sys.stdout
     output = sys.stdout = StringIO()
-    call_command(command, format_json=True, *args, **kwargs)
+    call_command(command, format_json=True, *args, **options)
     sys.stdout = old_stdout
     return output.getvalue()
 
 
 def test_import_export(app, some_data):
-    output = get_output_of_command('export_site')
+    assert 'pages' not in json.loads(get_output_of_command('export_site'))
+    output = get_output_of_command('export_site', exclude_pages=False)
     assert len(json.loads(output)['pages']) == 3
     import_site(data={}, clean=True)
     assert Page.objects.all().count() == 0
     assert TextCell.objects.all().count() == 0
-    empty_output = get_output_of_command('export_site')
+    empty_output = get_output_of_command('export_site', exclude_pages=False)
     assert len(json.loads(empty_output)['pages']) == 0
 
     Page(title='test', slug='test').save()
     old_stdin = sys.stdin
     sys.stdin = StringIO(json.dumps({}))
     assert Page.objects.count() == 1
     try:
         call_command('import_site', '-', clean=True)
@@ -83,36 +94,37 @@ def test_import_export(app, some_data):
     assert TextCell.objects.all().count() == 1
 
     import_site(data={}, if_empty=True)
     assert Page.objects.count() == 3
     assert TextCell.objects.all().count() == 1
 
     import_site(data=[], clean=True)
     tempdir = tempfile.mkdtemp('chrono-test')
-    empty_output = get_output_of_command('export_site', output=os.path.join(tempdir, 't.json'))
+    empty_output = get_output_of_command(
+        'export_site', exclude_pages=False, output=os.path.join(tempdir, 't.json'))
     assert os.path.exists(os.path.join(tempdir, 't.json'))
     shutil.rmtree(tempdir)
 
 
 def test_import_export_with_parent(app, some_data):
-    output = get_output_of_command('export_site')
+    output = get_output_of_command('export_site', exclude_pages=False)
     payload = json.loads(output)
     payload['pages'][1]['fields']['parent'] = ['one']
 
     Page.objects.all().delete()
     import_site(data=payload)
 
     assert Page.objects.count() == 3
     two = Page.objects.get(slug='two')
     assert two.parent.slug == 'one'
 
 
 def test_import_export_with_unknown_parent(app, some_data):
-    output = get_output_of_command('export_site')
+    output = get_output_of_command('export_site', exclude_pages=False)
     payload = json.loads(output)
     payload['pages'][0]['fields']['exclude_from_navigation'] = False
     payload['pages'][0]['fields']['parent'] = ['unknown-parent']
 
     Page.objects.all().delete()
     import_site(data=payload)
 
     assert Page.objects.count() == 3
@@ -123,22 +135,24 @@ def test_import_export_with_unknown_parent(app, some_data):
 
 
 def test_backward_compatibility_import(app, some_data):
     old_export = Page.export_all_for_json()
     Page.objects.all().delete()
     import_site(data=old_export)
     assert Page.objects.count() == 3
 
+
 def test_import_export_map_layers(app, some_map_layers):
-    output = get_output_of_command('export_site')
+    assert 'map-layers' not in json.loads(get_output_of_command('export_site'))
+    output = get_output_of_command('export_site', exclude_cartography=False)
     assert len(json.loads(output)['map-layers']) == 2
     import_site(data={}, clean=True)
     assert MapLayer.objects.all().count() == 0
-    empty_output = get_output_of_command('export_site')
+    empty_output = get_output_of_command('export_site', exclude_cartography=False)
     assert len(json.loads(empty_output)['map-layers']) == 0
 
     MapLayer(label='Baz', slug='baz', geojson_url='http://example.net/baz/').save()
     old_stdin = sys.stdin
     sys.stdin = StringIO(json.dumps({}))
     assert MapLayer.objects.count() == 1
     try:
         call_command('import_site', '-', clean=True)
@@ -157,17 +171,17 @@ def test_import_export_map_layers(app, some_map_layers):
     assert MapLayer.objects.count() == 2
 
 
 def test_import_export_map_cells(app, some_data, some_map_layers):
     page = Page.objects.get(slug='one')
     cell = Map(page=page, order=0, placeholder='content')
     cell.save()
     MapLayerOptions.objects.create(map_cell=cell, map_layer=MapLayer.objects.get(slug='foo'))
-    site_export = get_output_of_command('export_site')
+    site_export = get_output_of_command('export_site', exclude_pages=False, exclude_cartography=False)
     import_site(data={}, clean=True)
     assert Map.objects.count() == 0
     assert MapLayer.objects.count() == 0
 
     site_data = json.loads(site_export)
     import_site(data=site_data, clean=True)
     assert Map.objects.count() == 1
     assert MapLayer.objects.filter(slug='foo').exists() is True
@@ -193,17 +207,17 @@ def test_group_restrictions_import_export(app, some_data):
     page = Page.objects.get(slug='one')
     page.groups.set([group])
     page.save()
 
     cell = TextCell.objects.get(order=0)
     cell.groups.set([group])
     cell.save()
 
-    output = get_output_of_command('export_site')
+    output = get_output_of_command('export_site', exclude_pages=False)
     assert len(json.loads(output)['pages']) == 3
     import_site(data={}, clean=True)
     assert Page.objects.all().count() == 0
     assert TextCell.objects.all().count() == 0
 
     Group.objects.all().delete()
 
     with pytest.raises(MissingGroups) as excinfo:
@@ -226,22 +240,24 @@ def test_group_restrictions_import_export(app, some_data):
     assert TextCell.objects.all().count() == 1
 
     page = Page.objects.get(slug='one')
     assert [x.name for x in page.groups.all()] == ['A Group']
 
     cell = TextCell.objects.get(order=0)
     assert [x.name for x in cell.groups.all()] == ['A Group']
 
+
 def test_import_export_assets(app, some_assets):
-    output = get_output_of_command('export_site')
+    assert 'assets' not in json.loads(get_output_of_command('export_site'))
+    output = get_output_of_command('export_site', exclude_assets=False)
     assert len(json.loads(output)['assets']) == 2
     import_site(data={}, clean=True)
     assert Asset.objects.all().count() == 0
-    empty_output = get_output_of_command('export_site')
+    empty_output = get_output_of_command('export_site', exclude_assets=False)
     assert len(json.loads(empty_output)['assets']) == 0
 
     Asset(key='footer', asset=File(StringIO('test3'), 'test3.png')).save()
     old_stdin = sys.stdin
     sys.stdin = StringIO(json.dumps({}))
     assert Asset.objects.count() == 1
     try:
         call_command('import_site', '-', clean=True)
@@ -254,29 +270,30 @@ def test_import_export_assets(app, some_assets):
         f.flush()
         call_command('import_site', f.name)
 
     assert Asset.objects.count() == 2
 
     import_site(data={}, if_empty=True)
     assert Asset.objects.count() == 2
 
+
 def test_import_export_pwa_settings(app):
-    output = get_output_of_command('export_site')
+    output = get_output_of_command('export_site', exclude_pwa=False)
     pwa_settings = PwaSettings.singleton()
     pwa_settings.offline_text = 'Hello world'
     pwa_settings.offline_retry_button = False
     pwa_settings.save()
 
-    # check exort with no application icon
-    output = get_output_of_command('export_site')
+    # check export with no application icon
+    output = get_output_of_command('export_site', exclude_pwa=False)
 
     pwa_settings.application_icon = File(BytesIO(b'te\30st'), 'test.png')
     pwa_settings.save()
-    output = get_output_of_command('export_site')
+    output = get_output_of_command('export_site', exclude_pwa=False)
     import_site(data={}, clean=True)
     assert PwaSettings.objects.all().count() == 0
 
     import_site(data=json.loads(output))
     assert PwaSettings.singleton().offline_retry_button is False
     assert PwaSettings.singleton().offline_text == 'Hello world'
     # check identical file was not touched
     assert os.path.basename(PwaSettings.objects.get().application_icon.file.name) == 'test.png'
@@ -291,23 +308,27 @@ def test_import_export_pwa_settings(app):
     # check with a change in icon file name
     data = json.loads(output)
     data['pwa']['settings']['icon:base64'] = force_text(base64.encodebytes(b'TEST2'))
     data['pwa']['settings']['application_icon'] = 'pwa/test2.png'
     import_site(data=data)
     assert os.path.basename(PwaSettings.objects.get().application_icon.file.name) == 'test2.png'
     assert PwaSettings.objects.get().application_icon.read() == b'TEST2'
 
+
 def test_import_export_pwa_navigation(app, some_data):
     page = Page.objects.get(slug='one')
     entry1 = PwaNavigationEntry(label='a', url='/', order=0)
     entry2 = PwaNavigationEntry(link_page=page, order=1, icon=File(BytesIO(b'te\30st'), 'test.png'))
     entry1.save()
     entry2.save()
-    output = get_output_of_command('export_site')
+    assert 'pwa' not in json.loads(get_output_of_command('export_site'))
+    output = get_output_of_command('export_site', exclude_pages=False, exclude_pwa=False)
+    assert len(json.loads(output)['pwa']['navigation']) == 2
+
     import_site(data={}, clean=True)
     assert PwaNavigationEntry.objects.all().count() == 0
 
     import_site(data=json.loads(output))
     assert PwaNavigationEntry.objects.all().count() == 2
     # check identical file was not touched
     assert os.path.basename(PwaNavigationEntry.objects.get(order=1).icon.file.name) == 'test.png'
     assert PwaNavigationEntry.objects.get(order=1).icon.read() == b'te\30st'
@@ -327,34 +348,36 @@ def test_import_export_pwa_navigation(app, some_data):
     data = json.loads(output)
     data['pwa']['navigation'][1]['fields']['icon'] = 'pwa/test2.png'
     data['pwa']['navigation'][1]['icon:base64'] = force_text(base64.encodebytes(b'TEST2'))
     import_site(data=data)
     assert PwaNavigationEntry.objects.all().count() == 2
     assert os.path.basename(PwaNavigationEntry.objects.get(order=1).icon.file.name) == 'test2.png'
     assert PwaNavigationEntry.objects.get(order=1).icon.read() == b'TEST2'
 
+
 def test_import_export_gallery_images(app, some_data):
     page = Page.objects.get(slug='one')
     gallery = GalleryCell(page=page, order=2, placeholder='images')
     gallery.save()
     image1 = Image(gallery=gallery, image='path/foo.jpg', title='foo', order=1)
     image2 = Image(gallery=gallery, image='path/bar.jpg', title='bar', order=2)
     image1.save()
     image2.save()
-    output = get_output_of_command('export_site')
+    output = get_output_of_command('export_site', exclude_pages=False)
     import_site(data={}, clean=True)
     assert Image.objects.all().count() == 0
 
     import_site(data=json.loads(output))
     assert Image.objects.all().count() == 2
     image1 = Image.objects.get(title='foo')
     assert image1.image == 'path/foo.jpg'
     assert image1.gallery.placeholder == 'images'
 
+
 def test_import_export_extra_fields(app, some_data):
     site_export = export_site()
     for page in site_export['pages']:
         if page['fields']['slug'] == 'one':
             page['fields']['extra_field_not_in_model'] = True
         elif page['fields']['slug'] == 'three':
             page['cells'][0]['fields']['extra_field_not_in_model'] = True
 
@@ -424,17 +447,18 @@ def test_import_export_tar(tmpdir, some_assets):
         call_command('import_site', filename)
 
 
 def test_import_export_payment(app):
     backend = PaymentBackend.objects.create(label='Test', slug='test', service_options={'test': True})
     Regie.objects.create(label='Test regie', slug='test-regie', payment_backend=backend)
     Regie.objects.create(label='Test regie 2', slug='test-regie-2', payment_backend=backend)
 
-    output = get_output_of_command('export_site')
+    assert 'payment' not in json.loads(get_output_of_command('export_site'))
+    output = get_output_of_command('export_site', exclude_payment=False)
     payload = json.loads(output)
     assert len(payload['payment']['backends']) == 1
     assert len(payload['payment']['regies']) == 2
 
     import_site(payload)
     assert PaymentBackend.objects.count() == 1
     assert Regie.objects.count() == 2
 
@@ -449,11 +473,11 @@ def test_import_export_payment(app):
     assert Regie.objects.count() == 2
     regie = Regie.objects.first()
     assert regie.payment_backend == backend
 
     import_site(data={}, clean=True)
     assert PaymentBackend.objects.count() == 0
     assert Regie.objects.count() == 0
 
-    empty_output = get_output_of_command('export_site')
+    empty_output = get_output_of_command('export_site', exclude_payment=False)
     assert len(json.loads(empty_output)['payment']['backends']) == 0
     assert len(json.loads(empty_output)['payment']['regies']) == 0
-- 
2.30.0

