Projet

Général

Profil

0001-manager-add-tar-format-for-site-export-import-39425.patch

Nicolas Roche, 15 juillet 2020 12:33

Télécharger (10,8 ko)

Voir les différences:

Subject: [PATCH] manager: add tar format for site export/import (#39425)

 combo/manager/views.py | 36 +++++++++++++++++++++---------
 tests/test_manager.py  | 50 +++++++++++++++++++++++++++++++++++-------
 2 files changed, 68 insertions(+), 18 deletions(-)
combo/manager/views.py
12 12
# GNU Affero General Public License for more details.
13 13
#
14 14
# You should have received a copy of the GNU Affero General Public License
15 15
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 16

  
17 17
import hashlib
18 18
import json
19 19
import os
20
import tarfile
21

  
20 22
from operator import attrgetter
21 23

  
22 24
from django.conf import settings
23 25
from django.contrib import messages
24 26
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
25 27
from django.urls import reverse, reverse_lazy
26 28
from django.http import HttpResponse, HttpResponseRedirect, Http404
27 29
from django.shortcuts import redirect
28 30
from django.shortcuts import render
29 31
from django.shortcuts import get_object_or_404
30 32
from django.utils.translation import ugettext_lazy as _
31 33
from django.utils.encoding import force_text, force_bytes
32 34
from django.utils.formats import date_format
35
from django.utils.six import BytesIO
33 36
from django.utils.timezone import localtime
34 37
from django.views.decorators.csrf import requires_csrf_token
35 38
from django.views.generic import (RedirectView, DetailView,
36 39
        CreateView, UpdateView, ListView, DeleteView, FormView)
37 40

  
38 41
from combo.data.models import Page, CellBase, ParentContentCell, PageSnapshot, LinkListCell
39 42
from combo.data.library import get_cell_class
40
from combo.data.utils import export_site, import_site, MissingGroups
43
from combo.data.utils import (export_site_tar, import_site, import_site_tar, ImportSiteError,
44
        MissingGroups)
41 45
from combo import plugins
42 46

  
43 47
from .forms import (PageEditTitleForm, PageVisibilityForm, SiteImportForm,
44 48
        PageEditRedirectionForm, PageSelectTemplateForm, PageEditSlugForm,
45 49
        PageEditPictureForm, PageEditIncludeInNavigationForm,
46 50
        PageEditDescriptionForm, CellVisibilityForm, PageDuplicateForm)
47 51

  
48 52

  
......
60 64

  
61 65
homepage = HomepageView.as_view()
62 66

  
63 67

  
64 68
class SiteExportView(ListView):
65 69
    model = Page
66 70

  
67 71
    def render_to_response(self, context, **response_kwargs):
68
        response = HttpResponse(content_type='application/json')
69
        json.dump(export_site(), response, indent=2)
70
        return response
72
        fd = BytesIO()
73
        export_site_tar(fd)
74
        return HttpResponse(fd.getvalue(), content_type='application/x-tar')
71 75

  
72 76
site_export = SiteExportView.as_view()
73 77

  
74 78

  
75 79
class SiteImportView(FormView):
76 80
    form_class = SiteImportForm
77 81
    template_name = 'combo/site_import.html'
78 82
    success_url = reverse_lazy('combo-manager-homepage')
79 83

  
80 84
    def form_valid(self, form):
85
        fd = self.request.FILES['site_json'].file
81 86
        try:
82
            json_site = json.loads(force_text(self.request.FILES['site_json'].read()))
83
        except ValueError:
84
            form.add_error('site_json', _('File is not in the expected JSON format.'))
85
            return self.form_invalid(form)
86

  
87
            tarfile.open(mode='r', fileobj=fd)
88
        except tarfile.TarError as e:
89
            try:
90
                fd.seek(0)
91
                json_site = json.loads(force_text(fd.read()))
92
            except ValueError:
93
                form.add_error('site_json', _('File is not in the expected TAR or JSON format.'))
94
                return self.form_invalid(form)
95
            else:
96
                format = 'json'
97
        else:
98
            format = 'tar'
99
            fd.seek(0)
87 100
        try:
88
            import_site(json_site, request=self.request)
101
            if format == 'json':
102
                import_site(json_site, request=self.request)
103
            else:
104
                import_site_tar(fd, request=self.request)
89 105
        except MissingGroups as e:
90 106
            form.add_error('site_json', force_text(e))
91 107
            return self.form_invalid(form)
92 108

  
93 109
        return super(SiteImportView, self).form_valid(form)
94 110

  
95 111

  
96 112
site_import = SiteImportView.as_view()
tests/test_manager.py
1 1
import base64
2 2
import datetime
3 3
import json
4 4
import mock
5 5
import os
6 6
import re
7 7
import shutil
8
import tarfile
8 9

  
9 10
from django.core.files.storage import default_storage
10 11
from django.urls import reverse
11 12
from django.conf import settings
12 13
from django.contrib.auth.models import Group
13 14
from django.db import connection
14 15
from django.template import TemplateSyntaxError
15 16
from django.test import override_settings
......
24 25
import pytest
25 26
from webtest import Upload
26 27

  
27 28
from combo.data.forms import LinkCellForm
28 29
from combo.data.models import (
29 30
    Page, CellBase, TextCell, LinkCell, ConfigJsonCell, JsonCell, PageSnapshot,
30 31
    LinkListCell, ParentContentCell, MenuCell, ValidityInfo)
31 32
from combo.apps.assets.models import Asset
33
from combo.apps.assets.utils import add_tar_content
32 34
from combo.apps.family.models import FamilyInfosCell
33 35
from combo.apps.search.models import SearchCell
34 36

  
35 37
pytestmark = pytest.mark.django_db
36 38

  
37 39
TESTS_DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
38 40

  
39 41

  
......
546 548
    random_list = [page3, page4, page1, page2]
547 549
    ordered_list = Page.get_as_reordered_flat_hierarchy(random_list)
548 550
    assert ordered_list[0] == page1
549 551
    assert ordered_list[1] == page4
550 552
    assert ordered_list[2] == page2
551 553
    assert ordered_list[3] == page3
552 554

  
553 555

  
556
def test_page_export_import(app, admin_user):
557
    Page.objects.all().delete()
558
    page1 = Page(title='One', slug='one', template_name='standard')
559
    page1.save()
560

  
561
    cell = TextCell(page=page1, placeholder='content', text='Foobar', order=0)
562
    cell.save()
563

  
564
    app = login(app)
565
    resp = app.get('/manage/pages/%s/' % page1.id)
566
    resp = resp.click('Export')
567
    assert resp.headers['content-type'] == 'application/json'
568
    site_export = resp.body
569

  
570
    Page.objects.all().delete()
571
    assert TextCell.objects.count() == 0
572
    app = login(app)
573
    resp = app.get('/manage/')
574
    resp = resp.click('Import Site')
575
    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
576
    resp = resp.form.submit()
577
    assert Page.objects.count() == 1
578
    assert TextCell.objects.count() == 1
579

  
580

  
554 581
def test_site_export_import(app, admin_user):
555 582
    Page.objects.all().delete()
556 583
    page1 = Page(title='One', slug='one', template_name='standard')
557 584
    page1.save()
558 585
    page2 = Page(title='Two', slug='two', parent=page1, template_name='standard')
559 586
    page2.save()
560 587
    page3 = Page(title='Three', slug='three', parent=page2, template_name='standard')
561 588
    page3.save()
......
569 596
    cell.save()
570 597

  
571 598
    cell = LinkCell(page=page2, placeholder='content', link_page=page1, order=0)
572 599
    cell.save()
573 600

  
574 601
    app = login(app)
575 602
    resp = app.get('/manage/')
576 603
    resp = resp.click('Export Site')
577
    assert resp.headers['content-type'] == 'application/json'
604
    assert resp.headers['content-type'] == 'application/x-tar'
578 605
    site_export = resp.body
579 606

  
580 607
    Page.objects.all().delete()
581 608
    assert LinkCell.objects.count() == 0
582 609
    app = login(app)
583 610
    resp = app.get('/manage/')
584 611
    resp = resp.click('Import Site')
585
    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
612
    resp.form['site_json'] = Upload('site-export.tar', site_export, 'application/x-tar')
586 613
    resp = resp.form.submit()
587 614
    assert Page.objects.count() == 4
588 615
    assert LinkCell.objects.count() == 2
589 616
    assert LinkCell.objects.get(page__slug='one').link_page.slug == 'two'
590 617
    assert LinkCell.objects.get(page__slug='two').link_page.slug == 'one'
591 618

  
592 619
    # check with invalid file
593 620
    resp = app.get('/manage/')
594 621
    resp = resp.click('Import Site')
595
    resp.form['site_json'] = Upload('site-export.json', b'invalid content', 'application/json')
622
    resp.form['site_json'] = Upload('site-export.tar', b'invalid content', 'application/x-tar')
596 623
    resp = resp.form.submit()
597
    assert 'File is not in the expected JSON format.' in resp.text
624
    assert 'File is not in the expected TAR or JSON format.' in resp.text
598 625

  
599 626

  
600 627
def test_site_export_import_missing_group(app, admin_user):
601 628
    Page.objects.all().delete()
602 629
    group = Group.objects.create(name='foobar')
603 630
    page1 = Page(title='One', slug='one', template_name='standard')
604 631
    page1.save()
605 632
    page1.groups.set([group])
606 633

  
607 634
    app = login(app)
608 635
    resp = app.get('/manage/')
609 636
    resp = resp.click('Export Site')
610
    assert resp.headers['content-type'] == 'application/json'
637
    assert resp.headers['content-type'] == 'application/x-tar'
611 638
    site_export = resp.body
612 639

  
613 640
    Page.objects.all().delete()
614 641
    group.delete()
615 642

  
616 643
    app = login(app)
617 644
    resp = app.get('/manage/')
618 645
    resp = resp.click('Import Site')
619
    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
646
    resp.form['site_json'] = Upload('site-export.tar', site_export, 'application/x-tar')
620 647
    resp = resp.form.submit()
621 648
    assert 'Missing groups: foobar' in resp.text
622 649

  
623 650

  
624 651
def test_site_export_import_unknown_parent(app, admin_user):
625 652
    Page.objects.create(title='One', slug='one', template_name='standard')
626 653
    Page.objects.create(title='Two', slug='two', template_name='standard')
627 654

  
628 655
    app = login(app)
629 656
    resp = app.get('/manage/')
630 657
    resp = resp.click('Export Site')
631
    payload = json.loads(force_str(resp.body))
658
    tar = tarfile.open(mode='r', fileobj=BytesIO(resp.body))
659
    site_json = tar.extractfile(tar.getmember('_site.json')).read()
660
    payload = json.loads(force_str(site_json))
632 661
    payload['pages'][0]['fields']['exclude_from_navigation'] = False
633 662
    payload['pages'][0]['fields']['parent'] = ['unknown-parent']
634 663

  
635 664
    resp = app.get('/manage/')
636 665
    resp = resp.click('Import Site')
637
    resp.form['site_json'] = Upload('site-export.json', force_bytes(json.dumps(payload)), 'application/json')
666
    fd = BytesIO()
667
    tar = tarfile.open(mode='w', fileobj=fd)
668
    add_tar_content(tar, '_site.json', json.dumps(payload))
669
    tar_bytes = fd.getvalue()
670
    tar.close()
671
    resp.form['site_json'] = Upload('site-export.json', tar_bytes, 'application/x-tar')
638 672
    resp = resp.form.submit().follow()
639 673
    assert 'Unknown parent for page &quot;One&quot;; parent has been reset and page was excluded from navigation.' in resp.text
640 674

  
641 675

  
642 676
def test_invalid_cell_report(app, admin_user):
643 677
    app = login(app)
644 678
    resp = app.get('/manage/cells/invalid-report/')
645 679
    assert resp.context['object_list'] == []
646
-