0003-general-add-import-export-tar-format-for-site-39425.patch
combo/data/management/commands/export_site.py | ||
---|---|---|
12 | 12 |
# GNU Affero General Public License for more details. |
13 | 13 |
# |
14 | 14 |
# You should have received a copy of the GNU Affero General Public License |
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 | 17 |
import json |
18 | 18 |
import sys |
19 | 19 | |
20 |
from django.core.management.base import BaseCommand |
|
20 |
from django.core.management.base import BaseCommand, CommandError |
|
21 |
from django.utils.translation import ugettext_lazy as _ |
|
21 | 22 | |
22 |
from combo.data.utils import export_site |
|
23 |
from combo.data.utils import export_site, export_site_tar
|
|
23 | 24 | |
24 | 25 |
class Command(BaseCommand): |
25 | 26 |
help = 'Export the site' |
26 | 27 | |
27 | 28 |
def add_arguments(self, parser): |
28 | 29 |
parser.add_argument( |
29 | 30 |
'--output', metavar='FILE', default=None, |
30 | 31 |
help='name of a file to write output to') |
32 |
parser.add_argument( |
|
33 |
'--format-json', action='store_true', default=False, |
|
34 |
help='use JSON format with no asset files') |
|
31 | 35 | |
32 | 36 |
def handle(self, *args, **options): |
33 |
if options['output'] and options['output'] != '-': |
|
34 |
output = open(options['output'], 'w') |
|
37 |
if options['format_json']: |
|
38 |
if options['output'] and options['output'] != '-': |
|
39 |
output = open(options['output'], 'w') |
|
40 |
else: |
|
41 |
output = sys.stdout |
|
42 |
json.dump(export_site(), output, indent=2) |
|
35 | 43 |
else: |
36 |
output = sys.stdout |
|
37 |
json.dump(export_site(), output, indent=2) |
|
44 |
if options['output'] and options['output'] != '-': |
|
45 |
try: |
|
46 |
output = open(options['output'], 'wb') |
|
47 |
except IOError as e: |
|
48 |
raise CommandError(e) |
|
49 |
export_site_tar(output) |
|
50 |
else: |
|
51 |
raise CommandError(_('TAR format require output filename parameter')) |
combo/data/management/commands/import_site.py | ||
---|---|---|
11 | 11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 | 12 |
# GNU Affero General Public License for more details. |
13 | 13 |
# |
14 | 14 |
# You should have received a copy of the GNU Affero General Public License |
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 | 17 |
import json |
18 | 18 |
import sys |
19 |
import tarfile |
|
19 | 20 | |
20 | 21 |
from django.core.management.base import BaseCommand, CommandError |
21 |
from django.utils.encoding import force_text |
|
22 | 22 | |
23 |
from combo.data.utils import import_site, MissingGroups
|
|
23 |
from combo.data.utils import import_site, import_site_tar, ImportSiteError
|
|
24 | 24 | |
25 | 25 |
class Command(BaseCommand): |
26 | 26 |
help = 'Import an exported site' |
27 | 27 | |
28 | 28 |
def add_arguments(self, parser): |
29 | 29 |
parser.add_argument('filename', metavar='FILENAME', type=str, |
30 | 30 |
help='name of file to import') |
31 | 31 |
parser.add_argument( |
32 | 32 |
'--clean', action='store_true', default=False, |
33 | 33 |
help='Clean site before importing') |
34 | 34 |
parser.add_argument( |
35 | 35 |
'--if-empty', action='store_true', default=False, |
36 | 36 |
help='Import only if site is empty') |
37 | 37 | |
38 | 38 |
def handle(self, filename, *args, **options): |
39 | 39 |
if filename == '-': |
40 |
format = 'json' |
|
40 | 41 |
fd = sys.stdin |
41 | 42 |
else: |
42 |
fd = open(filename) |
|
43 |
try: |
|
44 |
fd = open(filename, 'rb') |
|
45 |
except IOError as e: |
|
46 |
raise CommandError(e) |
|
47 |
try: |
|
48 |
tarfile.open(mode='r', fileobj=fd) |
|
49 |
except tarfile.TarError as e: |
|
50 |
format = 'json' |
|
51 |
fd = open(filename, 'r') |
|
52 |
else: |
|
53 |
format = 'tar' |
|
54 |
fd = open(filename, 'rb') |
|
43 | 55 |
try: |
44 |
import_site(json.load(fd), |
|
45 |
if_empty=options['if_empty'], |
|
46 |
clean=options['clean']) |
|
47 |
except MissingGroups as e: |
|
56 |
if format == 'json': |
|
57 |
import_site(json.load(fd), |
|
58 |
if_empty=options['if_empty'], |
|
59 |
clean=options['clean']) |
|
60 |
else: |
|
61 |
import_site_tar(fd, |
|
62 |
if_empty=options['if_empty'], |
|
63 |
clean=options['clean']) |
|
64 |
except ImportSiteError as e: |
|
48 | 65 |
raise CommandError(e) |
combo/data/utils.py | ||
---|---|---|
9 | 9 |
# This program is distributed in the hope that it will be useful, |
10 | 10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
11 | 11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 | 12 |
# GNU Affero General Public License for more details. |
13 | 13 |
# |
14 | 14 |
# You should have received a copy of the GNU Affero General Public License |
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 |
import json |
|
18 |
import tarfile |
|
19 | ||
17 | 20 |
from django.contrib.auth.models import Group |
18 | 21 |
from django.db import transaction |
19 | 22 |
from django.utils import six |
20 | 23 |
from django.utils.encoding import python_2_unicode_compatible |
21 | 24 |
from django.utils.translation import ugettext_lazy as _ |
22 | 25 | |
23 | 26 |
from combo.apps.assets.models import Asset |
27 |
from combo.apps.assets.utils import (add_tar_content, clean_assets_files, |
|
28 |
untar_assets_files, tar_assets_files) |
|
24 | 29 |
from combo.apps.maps.models import MapLayer |
25 | 30 |
from combo.apps.pwa.models import PwaSettings, PwaNavigationEntry |
26 | 31 |
from .models import Page |
27 | 32 | |
28 | 33 | |
34 |
class ImportSiteError(Exception): |
|
35 |
pass |
|
36 | ||
29 | 37 |
@python_2_unicode_compatible |
30 |
class MissingGroups(Exception):
|
|
38 |
class MissingGroups(ImportSiteError):
|
|
31 | 39 |
def __init__(self, names): |
32 | 40 |
self.names = names |
33 | 41 | |
34 | 42 |
def __str__(self): |
35 | 43 |
return _('Missing groups: %s') % ', '.join(self.names) |
36 | 44 | |
37 | 45 | |
38 | 46 |
def export_site(): |
... | ... | |
80 | 88 | |
81 | 89 |
MapLayer.load_serialized_objects(data.get('map-layers') or []) |
82 | 90 |
Asset.load_serialized_objects(data.get('assets') or []) |
83 | 91 |
Page.load_serialized_pages(data.get('pages') or []) |
84 | 92 | |
85 | 93 |
if data.get('pwa'): |
86 | 94 |
PwaSettings.load_serialized_settings(data['pwa'].get('settings')) |
87 | 95 |
PwaNavigationEntry.load_serialized_objects(data['pwa'].get('navigation')) |
96 | ||
97 | ||
98 |
def export_site_tar(fd): |
|
99 |
tar = tarfile.open(mode='w', fileobj=fd) |
|
100 |
data = export_site() |
|
101 |
del data['assets'] |
|
102 |
add_tar_content(tar, '_site.json', json.dumps(data, indent=2)) |
|
103 |
tar_assets_files(tar) |
|
104 |
tar.close() |
|
105 | ||
106 | ||
107 |
def import_site_tar(fd, if_empty=False, clean=False): |
|
108 |
tar = tarfile.open(mode='r', fileobj=fd) |
|
109 |
try: |
|
110 |
tarinfo = tar.getmember('_site.json') |
|
111 |
except KeyError: |
|
112 |
raise ImportSiteError(_('TAR file should provide _site.json file')) |
|
113 |
json_site = tar.extractfile(tarinfo).read() |
|
114 |
data = json.loads(json_site.decode('utf-8')) |
|
115 |
data.update(untar_assets_files(tar, overwrite=not if_empty)) |
|
116 |
if clean: |
|
117 |
clean_assets_files() |
|
118 |
import_site(data, if_empty=if_empty, clean=clean) |
|
119 |
tar.close() |
combo/manager/views.py | ||
---|---|---|
11 | 11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 | 12 |
# GNU Affero General Public License for more details. |
13 | 13 |
# |
14 | 14 |
# You should have received a copy of the GNU Affero General Public License |
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 | 17 |
import hashlib |
18 | 18 |
import json |
19 |
import tarfile |
|
19 | 20 | |
20 | 21 |
from django.conf import settings |
21 | 22 |
from django.contrib import messages |
22 | 23 |
from django.core.exceptions import ObjectDoesNotExist |
23 | 24 |
from django.core.urlresolvers import reverse, reverse_lazy |
24 | 25 |
from django.http import HttpResponse, HttpResponseRedirect, Http404 |
25 | 26 |
from django.shortcuts import redirect |
26 | 27 |
from django.shortcuts import get_object_or_404 |
27 | 28 |
from django.utils.translation import ugettext_lazy as _ |
28 | 29 |
from django.utils.encoding import force_text, force_bytes |
29 | 30 |
from django.utils.formats import date_format |
31 |
from django.utils.six import BytesIO |
|
30 | 32 |
from django.utils.timezone import localtime |
31 | 33 |
from django.views.decorators.csrf import requires_csrf_token |
32 | 34 |
from django.views.generic import (RedirectView, DetailView, |
33 | 35 |
CreateView, UpdateView, ListView, DeleteView, FormView) |
34 | 36 | |
35 | 37 |
from combo.data.models import Page, CellBase, ParentContentCell, PageSnapshot, LinkListCell |
36 | 38 |
from combo.data.library import get_cell_class |
37 |
from combo.data.utils import export_site, import_site, MissingGroups |
|
39 |
from combo.data.utils import (export_site_tar, import_site, import_site_tar, ImportSiteError, |
|
40 |
MissingGroups) |
|
38 | 41 |
from combo import plugins |
39 | 42 | |
40 | 43 |
from .forms import (PageEditTitleForm, PageVisibilityForm, SiteImportForm, |
41 | 44 |
PageEditRedirectionForm, PageSelectTemplateForm, PageEditSlugForm, |
42 | 45 |
PageEditPictureForm, PageEditIncludeInNavigationForm, |
43 | 46 |
PageEditDescriptionForm, CellVisibilityForm) |
44 | 47 | |
45 | 48 | |
... | ... | |
55 | 58 | |
56 | 59 |
homepage = HomepageView.as_view() |
57 | 60 | |
58 | 61 | |
59 | 62 |
class SiteExportView(ListView): |
60 | 63 |
model = Page |
61 | 64 | |
62 | 65 |
def render_to_response(self, context, **response_kwargs): |
63 |
response = HttpResponse(content_type='application/json')
|
|
64 |
json.dump(export_site(), response, indent=2)
|
|
65 |
return response
|
|
66 |
fd = BytesIO()
|
|
67 |
export_site_tar(fd)
|
|
68 |
return HttpResponse(fd.getvalue(), content_type='application/x-tar')
|
|
66 | 69 | |
67 | 70 |
site_export = SiteExportView.as_view() |
68 | 71 | |
69 | 72 | |
70 | 73 |
class SiteImportView(FormView): |
71 | 74 |
form_class = SiteImportForm |
72 | 75 |
template_name = 'combo/site_import.html' |
73 | 76 |
success_url = reverse_lazy('combo-manager-homepage') |
74 | 77 | |
75 | 78 |
def form_valid(self, form): |
79 |
fd = self.request.FILES['site_json'].file |
|
76 | 80 |
try: |
77 |
json_site = json.loads(force_text(self.request.FILES['site_json'].read())) |
|
78 |
except ValueError: |
|
79 |
form.add_error('site_json', _('File is not in the expected JSON format.')) |
|
80 |
return self.form_invalid(form) |
|
81 | ||
81 |
tarfile.open(mode='r', fileobj=fd) |
|
82 |
except tarfile.TarError as e: |
|
83 |
try: |
|
84 |
fd.seek(0) |
|
85 |
json_site = json.loads(force_text(fd.read())) |
|
86 |
except ValueError: |
|
87 |
form.add_error('site_json', _('File is not in the expected TAR or JSON format.')) |
|
88 |
return self.form_invalid(form) |
|
89 |
else: |
|
90 |
format = 'json' |
|
91 |
else: |
|
92 |
format = 'tar' |
|
93 |
fd.seek(0) |
|
82 | 94 |
try: |
83 |
import_site(json_site) |
|
95 |
if format == 'json': |
|
96 |
import_site(json_site) |
|
97 |
else: |
|
98 |
import_site_tar(fd) |
|
84 | 99 |
except MissingGroups as e: |
85 | 100 |
form.add_error('site_json', force_text(e)) |
86 | 101 |
return self.form_invalid(form) |
87 | 102 | |
88 | 103 |
return super(SiteImportView, self).form_valid(form) |
89 | 104 | |
90 | 105 |
site_import = SiteImportView.as_view() |
91 | 106 |
tests/test_import_export.py | ||
---|---|---|
1 | 1 |
import base64 |
2 | 2 |
import json |
3 | 3 |
import os |
4 | 4 |
import shutil |
5 | 5 |
import sys |
6 |
import tarfile |
|
6 | 7 |
import tempfile |
7 | 8 | |
8 | 9 |
import pytest |
9 | 10 |
from django.contrib.auth.models import Group |
10 | 11 |
from django.core.files import File |
11 | 12 |
from django.core.management import call_command |
12 | 13 |
from django.core.management.base import CommandError |
13 | 14 |
from django.utils.encoding import force_bytes, force_text |
... | ... | |
42 | 43 |
@pytest.fixture |
43 | 44 |
def some_assets(): |
44 | 45 |
Asset(key='banner', asset=File(BytesIO(b'test'), 'test.png')).save() |
45 | 46 |
Asset(key='favicon', asset=File(BytesIO(b'test2'), 'test2.png')).save() |
46 | 47 | |
47 | 48 |
def get_output_of_command(command, *args, **kwargs): |
48 | 49 |
old_stdout = sys.stdout |
49 | 50 |
output = sys.stdout = StringIO() |
50 |
call_command(command, *args, **kwargs) |
|
51 |
call_command(command, format_json=True, *args, **kwargs)
|
|
51 | 52 |
sys.stdout = old_stdout |
52 | 53 |
return output.getvalue() |
53 | 54 | |
54 | 55 |
def test_import_export(app, some_data): |
55 | 56 |
output = get_output_of_command('export_site') |
56 | 57 |
assert len(json.loads(output)['pages']) == 3 |
57 | 58 |
import_site(data={}, clean=True) |
58 | 59 |
assert Page.objects.all().count() == 0 |
... | ... | |
270 | 271 |
output = get_output_of_command('export_site') |
271 | 272 |
import_site(data={}, clean=True) |
272 | 273 |
assert Image.objects.all().count() == 0 |
273 | 274 | |
274 | 275 |
import_site(data=json.loads(output)) |
275 | 276 |
assert Image.objects.all().count() == 2 |
276 | 277 |
assert image1.title == 'foo' |
277 | 278 |
assert image1.image == 'path/foo.jpg' |
279 | ||
280 |
def test_import_export_tar(tmpdir): |
|
281 |
filename = os.path.join(str(tmpdir), 'file.tar') |
|
282 |
call_command('export_site', '--output', filename) |
|
283 |
call_command('import_site', filename, '--clean') |
|
284 | ||
285 |
with pytest.raises(CommandError, match=r'No such file or directory'): |
|
286 |
call_command('export_site', '--output', '%s/noway/foo.tar' % tmpdir) |
|
287 | ||
288 |
with pytest.raises(CommandError, match='TAR format require output filename parameter'): |
|
289 |
call_command('export_site', '--output', '-') |
|
290 | ||
291 |
with pytest.raises(CommandError, match=r'No such file or directory'): |
|
292 |
call_command('import_site', '%s/noway/foo.tar' % tmpdir) |
|
293 | ||
294 |
tar = tarfile.open(filename, 'w') |
|
295 |
tar.close() |
|
296 |
with pytest.raises(CommandError, match=r'TAR file should provide _site.json file'): |
|
297 |
call_command('import_site', filename) |
tests/test_manager.py | ||
---|---|---|
449 | 449 |
page4 = Page(title='Four', slug='four', parent=page1, template_name='standard') |
450 | 450 |
random_list = [page3, page4, page1, page2] |
451 | 451 |
ordered_list = Page.get_as_reordered_flat_hierarchy(random_list) |
452 | 452 |
assert ordered_list[0] == page1 |
453 | 453 |
assert ordered_list[1] == page4 |
454 | 454 |
assert ordered_list[2] == page2 |
455 | 455 |
assert ordered_list[3] == page3 |
456 | 456 | |
457 |
def test_page_export_import(app, admin_user): |
|
458 |
Page.objects.all().delete() |
|
459 |
page1 = Page(title='One', slug='one', template_name='standard') |
|
460 |
page1.save() |
|
461 | ||
462 |
cell = TextCell(page=page1, placeholder='content', text='Foobar', order=0) |
|
463 |
cell.save() |
|
464 | ||
465 |
app = login(app) |
|
466 |
resp = app.get('/manage/pages/%s/' % page1.id) |
|
467 |
resp = resp.click('Export') |
|
468 |
assert resp.headers['content-type'] == 'application/json' |
|
469 |
site_export = resp.body |
|
470 | ||
471 |
Page.objects.all().delete() |
|
472 |
assert TextCell.objects.count() == 0 |
|
473 |
app = login(app) |
|
474 |
resp = app.get('/manage/') |
|
475 |
resp = resp.click('Import Site') |
|
476 |
resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json') |
|
477 |
resp = resp.form.submit() |
|
478 |
assert Page.objects.count() == 1 |
|
479 |
assert TextCell.objects.count() == 1 |
|
480 | ||
457 | 481 |
def test_site_export_import(app, admin_user): |
458 | 482 |
Page.objects.all().delete() |
459 | 483 |
page1 = Page(title='One', slug='one', template_name='standard') |
460 | 484 |
page1.save() |
461 | 485 |
page2 = Page(title='Two', slug='two', parent=page1, template_name='standard') |
462 | 486 |
page2.save() |
463 | 487 |
page3 = Page(title='Three', slug='three', parent=page2, template_name='standard') |
464 | 488 |
page3.save() |
... | ... | |
472 | 496 |
cell.save() |
473 | 497 | |
474 | 498 |
cell = LinkCell(page=page2, placeholder='content', link_page=page1, order=0) |
475 | 499 |
cell.save() |
476 | 500 | |
477 | 501 |
app = login(app) |
478 | 502 |
resp = app.get('/manage/') |
479 | 503 |
resp = resp.click('Export Site') |
480 |
assert resp.headers['content-type'] == 'application/json'
|
|
504 |
assert resp.headers['content-type'] == 'application/x-tar'
|
|
481 | 505 |
site_export = resp.body |
482 | 506 | |
483 | 507 |
Page.objects.all().delete() |
484 | 508 |
assert LinkCell.objects.count() == 0 |
485 | 509 |
app = login(app) |
486 | 510 |
resp = app.get('/manage/') |
487 | 511 |
resp = resp.click('Import Site') |
488 |
resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
|
|
512 |
resp.form['site_json'] = Upload('site-export.tar', site_export, 'application/x-tar')
|
|
489 | 513 |
resp = resp.form.submit() |
490 | 514 |
assert Page.objects.count() == 4 |
491 | 515 |
assert LinkCell.objects.count() == 2 |
492 | 516 |
assert LinkCell.objects.get(page__slug='one').link_page.slug == 'two' |
493 | 517 |
assert LinkCell.objects.get(page__slug='two').link_page.slug == 'one' |
494 | 518 | |
495 | 519 |
# check with invalid file |
496 | 520 |
resp = app.get('/manage/') |
497 | 521 |
resp = resp.click('Import Site') |
498 |
resp.form['site_json'] = Upload('site-export.json', b'invalid content', 'application/json')
|
|
522 |
resp.form['site_json'] = Upload('site-export.tar', b'invalid content', 'application/x-tar')
|
|
499 | 523 |
resp = resp.form.submit() |
500 |
assert 'File is not in the expected JSON format.' in resp.text |
|
524 |
assert 'File is not in the expected TAR or JSON format.' in resp.text
|
|
501 | 525 | |
502 | 526 |
def test_site_export_import_missing_group(app, admin_user): |
503 | 527 |
Page.objects.all().delete() |
504 | 528 |
group = Group.objects.create(name='foobar') |
505 | 529 |
page1 = Page(title='One', slug='one', template_name='standard') |
506 | 530 |
page1.save() |
507 | 531 |
page1.groups.set([group]) |
508 | 532 | |
509 | 533 |
app = login(app) |
510 | 534 |
resp = app.get('/manage/') |
511 | 535 |
resp = resp.click('Export Site') |
512 |
assert resp.headers['content-type'] == 'application/json'
|
|
536 |
assert resp.headers['content-type'] == 'application/x-tar'
|
|
513 | 537 |
site_export = resp.body |
514 | 538 | |
515 | 539 |
Page.objects.all().delete() |
516 | 540 |
group.delete() |
517 | 541 | |
518 | 542 |
app = login(app) |
519 | 543 |
resp = app.get('/manage/') |
520 | 544 |
resp = resp.click('Import Site') |
521 |
resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
|
|
545 |
resp.form['site_json'] = Upload('site-export.tar', site_export, 'application/x-tar')
|
|
522 | 546 |
resp = resp.form.submit() |
523 | 547 |
assert 'Missing groups: foobar' in resp.text |
524 | 548 | |
525 | 549 | |
526 | 550 |
def test_duplicate_page(app, admin_user): |
527 | 551 |
page = Page.objects.create(title='One', slug='one', template_name='standard', exclude_from_navigation=False) |
528 | 552 |
TextCell.objects.create(page=page, placeholder='content', text='Foobar', order=0) |
529 | 553 |
tests/test_pages.py | ||
---|---|---|
324 | 324 |
cell = TextCell(page=page2, text='bar', order=0, placeholder='content') |
325 | 325 |
cell.save() |
326 | 326 | |
327 | 327 |
export_filename = os.path.join(settings.MEDIA_ROOT, 'site-export.json') |
328 | 328 |
if os.path.exists(export_filename): |
329 | 329 |
os.unlink(export_filename) |
330 | 330 | |
331 | 331 |
cmd = ExportSiteCommand() |
332 |
cmd.handle(output=export_filename) |
|
332 |
cmd.handle(output=export_filename, format_json=True)
|
|
333 | 333 |
assert os.path.exists(export_filename) |
334 | 334 | |
335 | 335 |
stdout = sys.stdout |
336 | 336 |
try: |
337 | 337 |
sys.stdout = StringIO() |
338 |
cmd.handle(output='-') |
|
338 |
cmd.handle(output='-', format_json=True)
|
|
339 | 339 |
assert sys.stdout.getvalue() == open(export_filename).read() |
340 | 340 |
finally: |
341 | 341 |
sys.stdout = stdout |
342 | 342 | |
343 | 343 |
Page.objects.all().delete() |
344 | 344 | |
345 | 345 |
cmd = ImportSiteCommand() |
346 | 346 |
cmd.handle(filename=export_filename, if_empty=False, clean=False) |
347 |
- |