Projet

Général

Profil

0003-general-add-import-export-tar-format-for-site-39425.patch

Nicolas Roche, 17 février 2020 09:30

Télécharger (19,6 ko)

Voir les différences:

Subject: [PATCH 3/3] general: add import/export tar format for site (#39425)

 combo/data/management/commands/export_site.py | 26 ++++++++++----
 combo/data/management/commands/import_site.py | 31 ++++++++++++----
 combo/data/utils.py                           | 34 +++++++++++++++++-
 combo/manager/views.py                        | 35 ++++++++++++------
 tests/test_import_export.py                   | 22 +++++++++++-
 tests/test_manager.py                         | 36 +++++++++++++++----
 tests/test_pages.py                           |  4 +--
 7 files changed, 155 insertions(+), 33 deletions(-)
combo/data/management/commands/export_site.py
12 12
# GNU Affero General Public License for more details.
13 13
#
14 14
# You should have received a copy of the GNU Affero General Public License
15 15
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 16

  
17 17
import json
18 18
import sys
19 19

  
20
from django.core.management.base import BaseCommand
20
from django.core.management.base import BaseCommand, CommandError
21
from django.utils.translation import ugettext_lazy as _
21 22

  
22
from combo.data.utils import export_site
23
from combo.data.utils import export_site, export_site_tar
23 24

  
24 25
class Command(BaseCommand):
25 26
    help = 'Export the site'
26 27

  
27 28
    def add_arguments(self, parser):
28 29
        parser.add_argument(
29 30
                '--output', metavar='FILE', default=None,
30 31
                help='name of a file to write output to')
32
        parser.add_argument(
33
                '--format-json', action='store_true', default=False,
34
                help='use JSON format with no asset files')
31 35

  
32 36
    def handle(self, *args, **options):
33
        if options['output'] and options['output'] != '-':
34
            output = open(options['output'], 'w')
37
        if options['format_json']:
38
            if options['output'] and options['output'] != '-':
39
                output = open(options['output'], 'w')
40
            else:
41
                output = sys.stdout
42
            json.dump(export_site(), output, indent=2)
35 43
        else:
36
            output = sys.stdout
37
        json.dump(export_site(), output, indent=2)
44
            if options['output'] and options['output'] != '-':
45
                try:
46
                    output = open(options['output'], 'wb')
47
                except IOError as e:
48
                    raise CommandError(e)
49
                export_site_tar(output)
50
            else:
51
                raise CommandError(_('TAR format require output filename parameter'))
combo/data/management/commands/import_site.py
11 11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 12
# GNU Affero General Public License for more details.
13 13
#
14 14
# You should have received a copy of the GNU Affero General Public License
15 15
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 16

  
17 17
import json
18 18
import sys
19
import tarfile
19 20

  
20 21
from django.core.management.base import BaseCommand, CommandError
21
from django.utils.encoding import force_text
22 22

  
23
from combo.data.utils import import_site, MissingGroups
23
from combo.data.utils import import_site, import_site_tar, ImportSiteError
24 24

  
25 25
class Command(BaseCommand):
26 26
    help = 'Import an exported site'
27 27

  
28 28
    def add_arguments(self, parser):
29 29
        parser.add_argument('filename', metavar='FILENAME', type=str,
30 30
                help='name of file to import')
31 31
        parser.add_argument(
32 32
                '--clean', action='store_true', default=False,
33 33
                help='Clean site before importing')
34 34
        parser.add_argument(
35 35
                '--if-empty', action='store_true', default=False,
36 36
                help='Import only if site is empty')
37 37

  
38 38
    def handle(self, filename, *args, **options):
39 39
        if filename == '-':
40
            format = 'json'
40 41
            fd = sys.stdin
41 42
        else:
42
            fd = open(filename)
43
            try:
44
                fd = open(filename, 'rb')
45
            except IOError as e:
46
                raise CommandError(e)
47
            try:
48
                tarfile.open(mode='r', fileobj=fd)
49
            except tarfile.TarError as e:
50
                format = 'json'
51
                fd = open(filename, 'r')
52
            else:
53
                format = 'tar'
54
                fd = open(filename, 'rb')
43 55
        try:
44
            import_site(json.load(fd),
45
                        if_empty=options['if_empty'],
46
                        clean=options['clean'])
47
        except MissingGroups as e:
56
            if format == 'json':
57
                import_site(json.load(fd),
58
                            if_empty=options['if_empty'],
59
                            clean=options['clean'])
60
            else:
61
                import_site_tar(fd,
62
                                if_empty=options['if_empty'],
63
                                clean=options['clean'])
64
        except ImportSiteError as e:
48 65
            raise CommandError(e)
combo/data/utils.py
9 9
# This program is distributed in the hope that it will be useful,
10 10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 12
# GNU Affero General Public License for more details.
13 13
#
14 14
# You should have received a copy of the GNU Affero General Public License
15 15
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 16

  
17
import json
18
import tarfile
19

  
17 20
from django.contrib.auth.models import Group
18 21
from django.db import transaction
19 22
from django.utils import six
20 23
from django.utils.encoding import python_2_unicode_compatible
21 24
from django.utils.translation import ugettext_lazy as _
22 25

  
23 26
from combo.apps.assets.models import Asset
27
from combo.apps.assets.utils import (add_tar_content, clean_assets_files,
28
        untar_assets_files, tar_assets_files)
24 29
from combo.apps.maps.models import MapLayer
25 30
from combo.apps.pwa.models import PwaSettings, PwaNavigationEntry
26 31
from .models import Page
27 32

  
28 33

  
34
class ImportSiteError(Exception):
35
    pass
36

  
29 37
@python_2_unicode_compatible
30
class MissingGroups(Exception):
38
class MissingGroups(ImportSiteError):
31 39
    def __init__(self, names):
32 40
        self.names = names
33 41

  
34 42
    def __str__(self):
35 43
        return _('Missing groups: %s') % ', '.join(self.names)
36 44

  
37 45

  
38 46
def export_site():
......
80 88

  
81 89
        MapLayer.load_serialized_objects(data.get('map-layers') or [])
82 90
        Asset.load_serialized_objects(data.get('assets') or [])
83 91
        Page.load_serialized_pages(data.get('pages') or [])
84 92

  
85 93
        if data.get('pwa'):
86 94
            PwaSettings.load_serialized_settings(data['pwa'].get('settings'))
87 95
            PwaNavigationEntry.load_serialized_objects(data['pwa'].get('navigation'))
96

  
97

  
98
def export_site_tar(fd):
99
    tar = tarfile.open(mode='w', fileobj=fd)
100
    data = export_site()
101
    del data['assets']
102
    add_tar_content(tar, '_site.json', json.dumps(data, indent=2))
103
    tar_assets_files(tar)
104
    tar.close()
105

  
106

  
107
def import_site_tar(fd, if_empty=False, clean=False):
108
    tar = tarfile.open(mode='r', fileobj=fd)
109
    try:
110
        tarinfo = tar.getmember('_site.json')
111
    except KeyError:
112
        raise ImportSiteError(_('TAR file should provide _site.json file'))
113
    json_site = tar.extractfile(tarinfo).read()
114
    data = json.loads(json_site.decode('utf-8'))
115
    data.update(untar_assets_files(tar, overwrite=not if_empty))
116
    if clean:
117
        clean_assets_files()
118
    import_site(data, if_empty=if_empty, clean=clean)
119
    tar.close()
combo/manager/views.py
11 11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 12
# GNU Affero General Public License for more details.
13 13
#
14 14
# You should have received a copy of the GNU Affero General Public License
15 15
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 16

  
17 17
import hashlib
18 18
import json
19
import tarfile
19 20

  
20 21
from django.conf import settings
21 22
from django.contrib import messages
22 23
from django.core.exceptions import ObjectDoesNotExist
23 24
from django.core.urlresolvers import reverse, reverse_lazy
24 25
from django.http import HttpResponse, HttpResponseRedirect, Http404
25 26
from django.shortcuts import redirect
26 27
from django.shortcuts import get_object_or_404
27 28
from django.utils.translation import ugettext_lazy as _
28 29
from django.utils.encoding import force_text, force_bytes
29 30
from django.utils.formats import date_format
31
from django.utils.six import BytesIO
30 32
from django.utils.timezone import localtime
31 33
from django.views.decorators.csrf import requires_csrf_token
32 34
from django.views.generic import (RedirectView, DetailView,
33 35
        CreateView, UpdateView, ListView, DeleteView, FormView)
34 36

  
35 37
from combo.data.models import Page, CellBase, ParentContentCell, PageSnapshot, LinkListCell
36 38
from combo.data.library import get_cell_class
37
from combo.data.utils import export_site, import_site, MissingGroups
39
from combo.data.utils import (export_site_tar, import_site, import_site_tar, ImportSiteError,
40
        MissingGroups)
38 41
from combo import plugins
39 42

  
40 43
from .forms import (PageEditTitleForm, PageVisibilityForm, SiteImportForm,
41 44
        PageEditRedirectionForm, PageSelectTemplateForm, PageEditSlugForm,
42 45
        PageEditPictureForm, PageEditIncludeInNavigationForm,
43 46
        PageEditDescriptionForm, CellVisibilityForm)
44 47

  
45 48

  
......
55 58

  
56 59
homepage = HomepageView.as_view()
57 60

  
58 61

  
59 62
class SiteExportView(ListView):
60 63
    model = Page
61 64

  
62 65
    def render_to_response(self, context, **response_kwargs):
63
        response = HttpResponse(content_type='application/json')
64
        json.dump(export_site(), response, indent=2)
65
        return response
66
        fd = BytesIO()
67
        export_site_tar(fd)
68
        return HttpResponse(fd.getvalue(), content_type='application/x-tar')
66 69

  
67 70
site_export = SiteExportView.as_view()
68 71

  
69 72

  
70 73
class SiteImportView(FormView):
71 74
    form_class = SiteImportForm
72 75
    template_name = 'combo/site_import.html'
73 76
    success_url = reverse_lazy('combo-manager-homepage')
74 77

  
75 78
    def form_valid(self, form):
79
        fd = self.request.FILES['site_json'].file
76 80
        try:
77
            json_site = json.loads(force_text(self.request.FILES['site_json'].read()))
78
        except ValueError:
79
            form.add_error('site_json', _('File is not in the expected JSON format.'))
80
            return self.form_invalid(form)
81

  
81
            tarfile.open(mode='r', fileobj=fd)
82
        except tarfile.TarError as e:
83
            try:
84
                fd.seek(0)
85
                json_site = json.loads(force_text(fd.read()))
86
            except ValueError:
87
                form.add_error('site_json', _('File is not in the expected TAR or JSON format.'))
88
                return self.form_invalid(form)
89
            else:
90
                format = 'json'
91
        else:
92
            format = 'tar'
93
            fd.seek(0)
82 94
        try:
83
            import_site(json_site)
95
            if format == 'json':
96
                import_site(json_site)
97
            else:
98
                import_site_tar(fd)
84 99
        except MissingGroups as e:
85 100
            form.add_error('site_json', force_text(e))
86 101
            return self.form_invalid(form)
87 102

  
88 103
        return super(SiteImportView, self).form_valid(form)
89 104

  
90 105
site_import = SiteImportView.as_view()
91 106

  
tests/test_import_export.py
1 1
import base64
2 2
import json
3 3
import os
4 4
import shutil
5 5
import sys
6
import tarfile
6 7
import tempfile
7 8

  
8 9
import pytest
9 10
from django.contrib.auth.models import Group
10 11
from django.core.files import File
11 12
from django.core.management import call_command
12 13
from django.core.management.base import CommandError
13 14
from django.utils.encoding import force_bytes, force_text
......
42 43
@pytest.fixture
43 44
def some_assets():
44 45
    Asset(key='banner', asset=File(BytesIO(b'test'), 'test.png')).save()
45 46
    Asset(key='favicon', asset=File(BytesIO(b'test2'), 'test2.png')).save()
46 47

  
47 48
def get_output_of_command(command, *args, **kwargs):
48 49
    old_stdout = sys.stdout
49 50
    output = sys.stdout = StringIO()
50
    call_command(command, *args, **kwargs)
51
    call_command(command, format_json=True, *args, **kwargs)
51 52
    sys.stdout = old_stdout
52 53
    return output.getvalue()
53 54

  
54 55
def test_import_export(app, some_data):
55 56
    output = get_output_of_command('export_site')
56 57
    assert len(json.loads(output)['pages']) == 3
57 58
    import_site(data={}, clean=True)
58 59
    assert Page.objects.all().count() == 0
......
270 271
    output = get_output_of_command('export_site')
271 272
    import_site(data={}, clean=True)
272 273
    assert Image.objects.all().count() == 0
273 274

  
274 275
    import_site(data=json.loads(output))
275 276
    assert Image.objects.all().count() == 2
276 277
    assert image1.title == 'foo'
277 278
    assert image1.image == 'path/foo.jpg'
279

  
280
def test_import_export_tar(tmpdir):
281
    filename = os.path.join(str(tmpdir), 'file.tar')
282
    call_command('export_site', '--output', filename)
283
    call_command('import_site', filename, '--clean')
284

  
285
    with pytest.raises(CommandError, match=r'No such file or directory'):
286
        call_command('export_site', '--output', '%s/noway/foo.tar' % tmpdir)
287

  
288
    with pytest.raises(CommandError, match='TAR format require output filename parameter'):
289
        call_command('export_site', '--output', '-')
290

  
291
    with pytest.raises(CommandError, match=r'No such file or directory'):
292
        call_command('import_site', '%s/noway/foo.tar' % tmpdir)
293

  
294
    tar = tarfile.open(filename, 'w')
295
    tar.close()
296
    with pytest.raises(CommandError, match=r'TAR file should provide _site.json file'):
297
        call_command('import_site', filename)
tests/test_manager.py
449 449
    page4 = Page(title='Four', slug='four', parent=page1, template_name='standard')
450 450
    random_list = [page3, page4, page1, page2]
451 451
    ordered_list = Page.get_as_reordered_flat_hierarchy(random_list)
452 452
    assert ordered_list[0] == page1
453 453
    assert ordered_list[1] == page4
454 454
    assert ordered_list[2] == page2
455 455
    assert ordered_list[3] == page3
456 456

  
457
def test_page_export_import(app, admin_user):
458
    Page.objects.all().delete()
459
    page1 = Page(title='One', slug='one', template_name='standard')
460
    page1.save()
461

  
462
    cell = TextCell(page=page1, placeholder='content', text='Foobar', order=0)
463
    cell.save()
464

  
465
    app = login(app)
466
    resp = app.get('/manage/pages/%s/' % page1.id)
467
    resp = resp.click('Export')
468
    assert resp.headers['content-type'] == 'application/json'
469
    site_export = resp.body
470

  
471
    Page.objects.all().delete()
472
    assert TextCell.objects.count() == 0
473
    app = login(app)
474
    resp = app.get('/manage/')
475
    resp = resp.click('Import Site')
476
    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
477
    resp = resp.form.submit()
478
    assert Page.objects.count() == 1
479
    assert TextCell.objects.count() == 1
480

  
457 481
def test_site_export_import(app, admin_user):
458 482
    Page.objects.all().delete()
459 483
    page1 = Page(title='One', slug='one', template_name='standard')
460 484
    page1.save()
461 485
    page2 = Page(title='Two', slug='two', parent=page1, template_name='standard')
462 486
    page2.save()
463 487
    page3 = Page(title='Three', slug='three', parent=page2, template_name='standard')
464 488
    page3.save()
......
472 496
    cell.save()
473 497

  
474 498
    cell = LinkCell(page=page2, placeholder='content', link_page=page1, order=0)
475 499
    cell.save()
476 500

  
477 501
    app = login(app)
478 502
    resp = app.get('/manage/')
479 503
    resp = resp.click('Export Site')
480
    assert resp.headers['content-type'] == 'application/json'
504
    assert resp.headers['content-type'] == 'application/x-tar'
481 505
    site_export = resp.body
482 506

  
483 507
    Page.objects.all().delete()
484 508
    assert LinkCell.objects.count() == 0
485 509
    app = login(app)
486 510
    resp = app.get('/manage/')
487 511
    resp = resp.click('Import Site')
488
    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
512
    resp.form['site_json'] = Upload('site-export.tar', site_export, 'application/x-tar')
489 513
    resp = resp.form.submit()
490 514
    assert Page.objects.count() == 4
491 515
    assert LinkCell.objects.count() == 2
492 516
    assert LinkCell.objects.get(page__slug='one').link_page.slug == 'two'
493 517
    assert LinkCell.objects.get(page__slug='two').link_page.slug == 'one'
494 518

  
495 519
    # check with invalid file
496 520
    resp = app.get('/manage/')
497 521
    resp = resp.click('Import Site')
498
    resp.form['site_json'] = Upload('site-export.json', b'invalid content', 'application/json')
522
    resp.form['site_json'] = Upload('site-export.tar', b'invalid content', 'application/x-tar')
499 523
    resp = resp.form.submit()
500
    assert 'File is not in the expected JSON format.' in resp.text
524
    assert 'File is not in the expected TAR or JSON format.' in resp.text
501 525

  
502 526
def test_site_export_import_missing_group(app, admin_user):
503 527
    Page.objects.all().delete()
504 528
    group = Group.objects.create(name='foobar')
505 529
    page1 = Page(title='One', slug='one', template_name='standard')
506 530
    page1.save()
507 531
    page1.groups.set([group])
508 532

  
509 533
    app = login(app)
510 534
    resp = app.get('/manage/')
511 535
    resp = resp.click('Export Site')
512
    assert resp.headers['content-type'] == 'application/json'
536
    assert resp.headers['content-type'] == 'application/x-tar'
513 537
    site_export = resp.body
514 538

  
515 539
    Page.objects.all().delete()
516 540
    group.delete()
517 541

  
518 542
    app = login(app)
519 543
    resp = app.get('/manage/')
520 544
    resp = resp.click('Import Site')
521
    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
545
    resp.form['site_json'] = Upload('site-export.tar', site_export, 'application/x-tar')
522 546
    resp = resp.form.submit()
523 547
    assert 'Missing groups: foobar' in resp.text
524 548

  
525 549

  
526 550
def test_duplicate_page(app, admin_user):
527 551
    page = Page.objects.create(title='One', slug='one', template_name='standard', exclude_from_navigation=False)
528 552
    TextCell.objects.create(page=page, placeholder='content', text='Foobar', order=0)
529 553

  
tests/test_pages.py
324 324
    cell = TextCell(page=page2, text='bar', order=0, placeholder='content')
325 325
    cell.save()
326 326

  
327 327
    export_filename = os.path.join(settings.MEDIA_ROOT, 'site-export.json')
328 328
    if os.path.exists(export_filename):
329 329
        os.unlink(export_filename)
330 330

  
331 331
    cmd = ExportSiteCommand()
332
    cmd.handle(output=export_filename)
332
    cmd.handle(output=export_filename, format_json=True)
333 333
    assert os.path.exists(export_filename)
334 334

  
335 335
    stdout = sys.stdout
336 336
    try:
337 337
        sys.stdout = StringIO()
338
        cmd.handle(output='-')
338
        cmd.handle(output='-', format_json=True)
339 339
        assert sys.stdout.getvalue() == open(export_filename).read()
340 340
    finally:
341 341
        sys.stdout = stdout
342 342

  
343 343
    Page.objects.all().delete()
344 344

  
345 345
    cmd = ImportSiteCommand()
346 346
    cmd.handle(filename=export_filename, if_empty=False, clean=False)
347
-