0002-commands-add-tar-format-for-site-export-import-39426.patch
combo/data/management/commands/export_site.py | ||
---|---|---|
12 | 12 |
# GNU Affero General Public License for more details. |
13 | 13 |
# |
14 | 14 |
# You should have received a copy of the GNU Affero General Public License |
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 | 17 |
import json |
18 | 18 |
import sys |
19 | 19 | |
20 |
from django.core.management.base import BaseCommand |
|
20 |
from django.core.management.base import BaseCommand, CommandError |
|
21 |
from django.utils.translation import ugettext_lazy as _ |
|
21 | 22 | |
22 |
from combo.data.utils import export_site |
|
23 |
from combo.data.utils import export_site, export_site_tar
|
|
23 | 24 | |
24 | 25 |
class Command(BaseCommand): |
25 | 26 |
help = 'Export the site' |
26 | 27 | |
27 | 28 |
def add_arguments(self, parser): |
28 | 29 |
parser.add_argument( |
29 | 30 |
'--output', metavar='FILE', default=None, |
30 | 31 |
help='name of a file to write output to') |
32 |
parser.add_argument( |
|
33 |
'--format-json', action='store_true', default=False, |
|
34 |
help='use JSON format with no asset files') |
|
31 | 35 | |
32 | 36 |
def handle(self, *args, **options): |
33 |
if options['output'] and options['output'] != '-': |
|
34 |
output = open(options['output'], 'w') |
|
37 |
if options['format_json']: |
|
38 |
if options['output'] and options['output'] != '-': |
|
39 |
output = open(options['output'], 'w') |
|
40 |
else: |
|
41 |
output = sys.stdout |
|
42 |
json.dump(export_site(), output, indent=2) |
|
35 | 43 |
else: |
36 |
output = sys.stdout |
|
37 |
json.dump(export_site(), output, indent=2) |
|
44 |
if options['output'] and options['output'] != '-': |
|
45 |
try: |
|
46 |
output = open(options['output'], 'wb') |
|
47 |
except IOError as e: |
|
48 |
raise CommandError(e) |
|
49 |
export_site_tar(output) |
|
50 |
output.close() |
|
51 |
else: |
|
52 |
raise CommandError(_('TAR format require output filename parameter')) |
combo/data/management/commands/import_site.py | ||
---|---|---|
11 | 11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 | 12 |
# GNU Affero General Public License for more details. |
13 | 13 |
# |
14 | 14 |
# You should have received a copy of the GNU Affero General Public License |
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 | 17 |
import json |
18 | 18 |
import sys |
19 |
import tarfile |
|
19 | 20 | |
20 | 21 |
from django.core.management.base import BaseCommand, CommandError |
21 |
from django.utils.encoding import force_text |
|
22 | 22 | |
23 |
from combo.data.utils import import_site, MissingGroups
|
|
23 |
from combo.data.utils import import_site, import_site_tar, ImportSiteError
|
|
24 | 24 | |
25 | 25 |
class Command(BaseCommand): |
26 | 26 |
help = 'Import an exported site' |
27 | 27 | |
28 | 28 |
def add_arguments(self, parser): |
29 | 29 |
parser.add_argument('filename', metavar='FILENAME', type=str, |
30 | 30 |
help='name of file to import') |
31 | 31 |
parser.add_argument( |
32 | 32 |
'--clean', action='store_true', default=False, |
33 | 33 |
help='Clean site before importing') |
34 | 34 |
parser.add_argument( |
35 | 35 |
'--if-empty', action='store_true', default=False, |
36 | 36 |
help='Import only if site is empty') |
37 |
parser.add_argument( |
|
38 |
'--overwrite', action='store_true', default=False, |
|
39 |
help='Overwrite asset files') |
|
37 | 40 | |
38 | 41 |
def handle(self, filename, *args, **options): |
39 | 42 |
if filename == '-': |
43 |
format = 'json' |
|
40 | 44 |
fd = sys.stdin |
41 | 45 |
else: |
42 |
fd = open(filename) |
|
46 |
try: |
|
47 |
fd = open(filename, 'rb') |
|
48 |
except IOError as e: |
|
49 |
raise CommandError(e) |
|
50 |
try: |
|
51 |
tarfile.open(mode='r', fileobj=fd) |
|
52 |
except tarfile.TarError as e: |
|
53 |
format = 'json' |
|
54 |
fd = open(filename, 'r') |
|
55 |
else: |
|
56 |
format = 'tar' |
|
57 |
fd = open(filename, 'rb') |
|
43 | 58 |
try: |
44 |
import_site(json.load(fd), |
|
45 |
if_empty=options['if_empty'], |
|
46 |
clean=options['clean']) |
|
47 |
except MissingGroups as e: |
|
59 |
if format == 'json': |
|
60 |
import_site(json.load(fd), |
|
61 |
if_empty=options['if_empty'], |
|
62 |
clean=options['clean']) |
|
63 |
else: |
|
64 |
import_site_tar(fd, |
|
65 |
if_empty=options['if_empty'], |
|
66 |
clean=options['clean'], |
|
67 |
overwrite=options['overwrite']) |
|
68 |
except ImportSiteError as e: |
|
48 | 69 |
raise CommandError(e) |
combo/data/utils.py | ||
---|---|---|
9 | 9 |
# This program is distributed in the hope that it will be useful, |
10 | 10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
11 | 11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 | 12 |
# GNU Affero General Public License for more details. |
13 | 13 |
# |
14 | 14 |
# You should have received a copy of the GNU Affero General Public License |
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 |
import json |
|
18 |
import tarfile |
|
19 | ||
17 | 20 |
from django.contrib.auth.models import Group |
18 | 21 |
from django.db import transaction |
19 | 22 |
from django.utils import six |
20 | 23 |
from django.utils.encoding import python_2_unicode_compatible |
21 | 24 |
from django.utils.translation import ugettext_lazy as _ |
22 | 25 | |
23 | 26 |
from combo.apps.assets.models import Asset |
27 |
from combo.apps.assets.utils import (add_tar_content, clean_assets_files, |
|
28 |
untar_assets_files, tar_assets_files) |
|
24 | 29 |
from combo.apps.maps.models import MapLayer |
25 | 30 |
from combo.apps.pwa.models import PwaSettings, PwaNavigationEntry |
26 | 31 |
from .models import Page |
27 | 32 | |
28 | 33 | |
34 |
class ImportSiteError(Exception): |
|
35 |
pass |
|
36 | ||
29 | 37 |
@python_2_unicode_compatible |
30 |
class MissingGroups(Exception):
|
|
38 |
class MissingGroups(ImportSiteError):
|
|
31 | 39 |
def __init__(self, names): |
32 | 40 |
self.names = names |
33 | 41 | |
34 | 42 |
def __str__(self): |
35 | 43 |
return _('Missing groups: %s') % ', '.join(self.names) |
36 | 44 | |
37 | 45 | |
38 | 46 |
def export_site(): |
... | ... | |
80 | 88 | |
81 | 89 |
MapLayer.load_serialized_objects(data.get('map-layers') or []) |
82 | 90 |
Asset.load_serialized_objects(data.get('assets') or []) |
83 | 91 |
Page.load_serialized_pages(data.get('pages') or [], request=request) |
84 | 92 | |
85 | 93 |
if data.get('pwa'): |
86 | 94 |
PwaSettings.load_serialized_settings(data['pwa'].get('settings')) |
87 | 95 |
PwaNavigationEntry.load_serialized_objects(data['pwa'].get('navigation')) |
96 | ||
97 | ||
98 |
def export_site_tar(fd): |
|
99 |
tar = tarfile.open(mode='w', fileobj=fd) |
|
100 |
data = export_site() |
|
101 |
del data['assets'] |
|
102 |
add_tar_content(tar, '_site.json', json.dumps(data, indent=2)) |
|
103 |
tar_assets_files(tar) |
|
104 |
tar.close() |
|
105 | ||
106 | ||
107 |
def import_site_tar(fd, if_empty=False, clean=False, overwrite=False, request=None): |
|
108 |
tar = tarfile.open(mode='r', fileobj=fd) |
|
109 |
try: |
|
110 |
tarinfo = tar.getmember('_site.json') |
|
111 |
except KeyError: |
|
112 |
raise ImportSiteError(_('TAR file should provide _site.json file')) |
|
113 | ||
114 |
if if_empty and (Page.objects.count() or MapLayer.objects.count()): |
|
115 |
return |
|
116 | ||
117 |
if clean: |
|
118 |
clean_assets_files() |
|
119 | ||
120 |
json_site = tar.extractfile(tarinfo).read() |
|
121 |
data = json.loads(json_site.decode('utf-8')) |
|
122 |
data.update(untar_assets_files(tar, overwrite=overwrite)) |
|
123 |
import_site(data, if_empty=if_empty, clean=clean, request=request) |
|
124 |
tar.close() |
tests/test_import_export.py | ||
---|---|---|
1 | 1 |
import base64 |
2 | 2 |
import datetime |
3 | 3 |
import json |
4 | 4 |
import os |
5 | 5 |
import shutil |
6 | 6 |
import sys |
7 |
import tarfile |
|
7 | 8 |
import tempfile |
8 | 9 | |
9 | 10 |
import pytest |
10 | 11 |
from django.contrib.auth.models import Group |
11 | 12 |
from django.core.files import File |
13 |
from django.core.files.storage import default_storage |
|
12 | 14 |
from django.core.management import call_command |
13 | 15 |
from django.core.management.base import CommandError |
14 | 16 |
from django.utils.encoding import force_bytes, force_text |
15 | 17 |
from django.utils.six import BytesIO, StringIO |
16 | 18 | |
17 | 19 |
from combo.apps.assets.models import Asset |
20 |
from combo.apps.assets.utils import clean_assets_files |
|
18 | 21 |
from combo.apps.gallery.models import Image, GalleryCell |
19 | 22 |
from combo.apps.maps.models import MapLayer, Map, MapLayerOptions |
20 | 23 |
from combo.apps.pwa.models import PwaSettings, PwaNavigationEntry |
21 | 24 |
from combo.data.models import Page, TextCell |
22 | 25 |
from combo.data.utils import export_site, import_site, MissingGroups |
23 | 26 | |
24 | 27 |
pytestmark = pytest.mark.django_db |
25 | 28 | |
... | ... | |
41 | 44 |
@pytest.fixture |
42 | 45 |
def some_assets(): |
43 | 46 |
Asset(key='banner', asset=File(BytesIO(b'test'), 'test.png')).save() |
44 | 47 |
Asset(key='favicon', asset=File(BytesIO(b'test2'), 'test2.png')).save() |
45 | 48 | |
46 | 49 |
def get_output_of_command(command, *args, **kwargs): |
47 | 50 |
old_stdout = sys.stdout |
48 | 51 |
output = sys.stdout = StringIO() |
49 |
call_command(command, *args, **kwargs) |
|
52 |
call_command(command, format_json=True, *args, **kwargs)
|
|
50 | 53 |
sys.stdout = old_stdout |
51 | 54 |
return output.getvalue() |
52 | 55 | |
53 | 56 | |
54 | 57 |
def test_import_export(app, some_data): |
55 | 58 |
output = get_output_of_command('export_site') |
56 | 59 |
assert len(json.loads(output)['pages']) == 3 |
57 | 60 |
import_site(data={}, clean=True) |
... | ... | |
329 | 332 |
if page['fields']['slug'] == 'one': |
330 | 333 |
page['fields']['extra_field_not_in_model'] = True |
331 | 334 |
elif page['fields']['slug'] == 'three': |
332 | 335 |
page['cells'][0]['fields']['extra_field_not_in_model'] = True |
333 | 336 | |
334 | 337 |
import_site(site_export) |
335 | 338 |
assert Page.objects.count() == 3 |
336 | 339 |
assert TextCell.objects.count() == 1 |
340 | ||
341 | ||
342 |
def test_import_export_tar(tmpdir, some_assets): |
|
343 |
filename = os.path.join(str(tmpdir), 'file.tar') |
|
344 | ||
345 |
# build import having some_assets fixtures assets: banner and favicon |
|
346 |
call_command('export_site', '--output', filename) |
|
347 | ||
348 |
def populate_site(): |
|
349 |
Page.objects.all().delete() |
|
350 |
Asset.objects.all().delete() |
|
351 |
clean_assets_files() |
|
352 |
Page.objects.create(title='One', slug='one') |
|
353 |
Asset(key='banner', asset=File(BytesIO(b'original content'), 'test.png')).save() |
|
354 |
Asset(key='logo', asset=File(BytesIO(b'logo'), 'logo.png')).save() |
|
355 | ||
356 |
populate_site() |
|
357 |
call_command('import_site', filename) # default behaviour |
|
358 |
assert Page.objects.count() == 1 |
|
359 |
assert Asset.objects.count() == 3 |
|
360 |
Asset.objects.get(key='banner').asset.name == 'assets/test.png' |
|
361 |
assert open('%s/assets/test.png' % default_storage.path('')).read() == 'original content' |
|
362 | ||
363 |
populate_site() |
|
364 |
call_command('import_site', filename, '--overwrite') |
|
365 |
assert Page.objects.count() == 1 |
|
366 |
assert Asset.objects.count() == 3 |
|
367 |
Asset.objects.get(key='banner').asset.name == 'assets/test.png' |
|
368 |
assert open('%s/assets/test.png' % default_storage.path('')).read() == 'test' |
|
369 | ||
370 |
populate_site() |
|
371 |
call_command('import_site', filename, '--if-empty') |
|
372 |
assert Page.objects.count() == 1 |
|
373 |
assert Asset.objects.count() == 2 |
|
374 |
Asset.objects.get(key='banner').asset.name == 'assets/test3.png' |
|
375 |
assert open('%s/assets/test.png' % default_storage.path('')).read() == 'original content' |
|
376 |
Asset.objects.get(key='logo').asset.name == 'assets/logo.png' |
|
377 |
assert os.path.isfile('%s/assets/logo.png' % default_storage.path('')) |
|
378 | ||
379 |
populate_site() |
|
380 |
call_command('import_site', filename, '--clean') |
|
381 |
assert Page.objects.count() == 0 |
|
382 |
assert Asset.objects.count() == 2 |
|
383 |
Asset.objects.get(key='banner').asset.name == 'assets/test.png' |
|
384 |
assert open('%s/assets/test.png' % default_storage.path('')).read() == 'test' |
|
385 |
assert not Asset.objects.filter(key='logo') |
|
386 |
assert not os.path.isfile('%s/assets/logo.png' % default_storage.path('')) |
|
387 | ||
388 |
# error cases |
|
389 |
with pytest.raises(CommandError, match=r'No such file or directory'): |
|
390 |
call_command('export_site', '--output', '%s/noway/foo.tar' % tmpdir) |
|
391 | ||
392 |
with pytest.raises(CommandError, match='TAR format require output filename parameter'): |
|
393 |
call_command('export_site', '--output', '-') |
|
394 | ||
395 |
with pytest.raises(CommandError, match=r'No such file or directory'): |
|
396 |
call_command('import_site', '%s/noway/foo.tar' % tmpdir) |
|
397 | ||
398 |
tarfile.open(filename, 'w').close() # empty tar file |
|
399 |
with pytest.raises(CommandError, match=r'TAR file should provide _site.json file'): |
|
400 |
call_command('import_site', filename) |
tests/test_pages.py | ||
---|---|---|
336 | 336 |
cell = TextCell(page=page2, text='bar', order=0, placeholder='content') |
337 | 337 |
cell.save() |
338 | 338 | |
339 | 339 |
export_filename = os.path.join(settings.MEDIA_ROOT, 'site-export.json') |
340 | 340 |
if os.path.exists(export_filename): |
341 | 341 |
os.unlink(export_filename) |
342 | 342 | |
343 | 343 |
cmd = ExportSiteCommand() |
344 |
cmd.handle(output=export_filename) |
|
344 |
cmd.handle(output=export_filename, format_json=True)
|
|
345 | 345 |
assert os.path.exists(export_filename) |
346 | 346 | |
347 | 347 |
stdout = sys.stdout |
348 | 348 |
try: |
349 | 349 |
sys.stdout = StringIO() |
350 |
cmd.handle(output='-') |
|
350 |
cmd.handle(output='-', format_json=True)
|
|
351 | 351 |
assert sys.stdout.getvalue() == open(export_filename).read() |
352 | 352 |
finally: |
353 | 353 |
sys.stdout = stdout |
354 | 354 | |
355 | 355 |
Page.objects.all().delete() |
356 | 356 | |
357 | 357 |
cmd = ImportSiteCommand() |
358 | 358 |
cmd.handle(filename=export_filename, if_empty=False, clean=False) |
359 |
- |