Projet

Général

Profil

0001-manager-add-tar-format-for-site-export-import-45128.patch

Nicolas Roche, 26 juillet 2020 15:44

Télécharger (14,6 ko)

Voir les différences:

Subject: [PATCH] manager: add tar format for site export/import (#45128)

 combo/manager/forms.py                        |  6 +-
 .../manager/templates/combo/manager_home.html |  2 +-
 .../manager/templates/combo/site_export.html  | 17 ++++++
 combo/manager/views.py                        | 55 ++++++++++++-----
 tests/test_manager.py                         | 59 +++++++++++++++++--
 5 files changed, 116 insertions(+), 23 deletions(-)
 create mode 100644 combo/manager/templates/combo/site_export.html
combo/manager/forms.py
187 187
            self.instance.public = False
188 188
            self.instance.restricted_to_unlogged = True
189 189
            self.instance.groups.set(self.cleaned_data['groups'])
190 190
        self.instance.save()
191 191
        return self.instance
192 192

  
193 193

  
194 194
class SiteImportForm(forms.Form):
195
    site_json = forms.FileField(label=_('Site Export File'))
195
    site_file = forms.FileField(label=_('Site Export File'))
196

  
197

  
198
class SiteExportForm(forms.Form):
199
    include_asset = forms.BooleanField(label=_('Include assets into the export'), required=False)
combo/manager/templates/combo/manager_home.html
2 2
{% load i18n %}
3 3

  
4 4
{% block appbar %}
5 5
<h2>{% trans 'Pages' %}</h2>
6 6
<span class="actions">
7 7
<a class="extra-actions-menu-opener"></a>
8 8
<a rel="popup" href="{% url 'combo-manager-page-add' %}">{% trans 'New' %}</a>
9 9
<ul class="extra-actions-menu">
10
 <li><a download href="{% url 'combo-manager-site-export' %}">{% trans 'Export Site' %}</a></li>
10
 <li><a href="{% url 'combo-manager-site-export' %}" rel="popup" data-autoclose-dialog="true">{% trans 'Export Site' %}</a></li>
11 11
 <li><a href="{% url 'combo-manager-site-import' %}">{% trans 'Import Site' %}</a></li>
12 12
 <li><a href="{% url 'combo-manager-invalid-cell-report' %}">{% trans 'Anomaly report' %}</a></li>
13 13
 {% for extra_action in extra_actions %}
14 14
 <li><a href="{{ extra_action.href }}">{{ extra_action.text }}</a></li>
15 15
 {% endfor %}
16 16
</ul>
17 17
</span>
18 18
{% endblock %}
combo/manager/templates/combo/site_export.html
1
{% extends "combo/manager_base.html" %}
2
{% load i18n %}
3

  
4
{% block appbar %}
5
<h2>{% trans "Site Export" %}</h2>
6
{% endblock %}
7

  
8
{% block content %}
9
<form method="post">
10
  {% csrf_token %}
11
  {{ form.as_p }}
12
  <div class="buttons">
13
    <button>{% trans 'Export' %}</button>
14
    <a class="cancel" href="{% url 'combo-manager-homepage' %}">{% trans 'Cancel' %}</a>
15
  </div>
16
</form>
17
{% endblock %}
combo/manager/views.py
12 12
# GNU Affero General Public License for more details.
13 13
#
14 14
# You should have received a copy of the GNU Affero General Public License
15 15
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 16

  
17 17
import hashlib
18 18
import json
19 19
import os
20
import tarfile
21

  
20 22
from operator import attrgetter
21 23

  
22 24
from django.conf import settings
23 25
from django.contrib import messages
24 26
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
25 27
from django.urls import reverse, reverse_lazy
26 28
from django.http import HttpResponse, HttpResponseRedirect, Http404
27 29
from django.shortcuts import redirect
28 30
from django.shortcuts import render
29 31
from django.shortcuts import get_object_or_404
30 32
from django.utils.translation import ugettext_lazy as _
31 33
from django.utils.encoding import force_text, force_bytes
32 34
from django.utils.formats import date_format
35
from django.utils.six import BytesIO
33 36
from django.utils.timezone import localtime
34 37
from django.views.decorators.csrf import requires_csrf_token
35 38
from django.views.generic import (RedirectView, DetailView,
36
        CreateView, UpdateView, ListView, DeleteView, FormView)
39
        CreateView, UpdateView, ListView, DeleteView, FormView, TemplateView)
37 40

  
38 41
from combo.data.models import Page, CellBase, ParentContentCell, PageSnapshot, LinkListCell
39 42
from combo.data.library import get_cell_class
40
from combo.data.utils import export_site, import_site, MissingGroups
43
from combo.data.utils import (export_site, export_site_tar, import_site, import_site_tar,
44
        ImportSiteError, MissingGroups)
41 45
from combo import plugins
42 46

  
43
from .forms import (PageEditTitleForm, PageVisibilityForm, SiteImportForm,
47
from .forms import (PageEditTitleForm, PageVisibilityForm, SiteImportForm, SiteExportForm,
44 48
        PageEditRedirectionForm, PageSelectTemplateForm, PageEditSlugForm,
45 49
        PageEditPictureForm, PageEditIncludeInNavigationForm,
46 50
        PageEditDescriptionForm, CellVisibilityForm, PageDuplicateForm)
47 51

  
48 52

  
49 53
class HomepageView(ListView):
50 54
    model = Page
51 55
    template_name = 'combo/manager_home.html'
......
56 60
        context['extra_actions'] = plugins.get_extra_manager_actions()
57 61
        context['collapse_pages'] = settings.COMBO_MANAGE_HOME_COLLAPSE_PAGES
58 62
        return context
59 63

  
60 64

  
61 65
homepage = HomepageView.as_view()
62 66

  
63 67

  
64
class SiteExportView(ListView):
65
    model = Page
68
class SiteExportView(FormView):
69
    form_class = SiteExportForm
70
    template_name = 'combo/site_export.html'
66 71

  
67
    def render_to_response(self, context, **response_kwargs):
68
        response = HttpResponse(content_type='application/json')
69
        json.dump(export_site(), response, indent=2)
72
    def post(self, request, *args, **kwargs):
73
        if request.POST.get('include_asset'):
74
            fd = BytesIO()
75
            export_site_tar(fd)
76
            response = HttpResponse(content=fd.getvalue(), content_type='application/x-tar')
77
            response['Content-Disposition'] = 'attachment; filename="site-export.tar"'
78
        else:
79
            response = HttpResponse(content_type='application/json')
80
            response['Content-Disposition'] = 'attachment; filename="site-export.json"'
81
            json.dump(export_site(), response, indent=2)
70 82
        return response
71 83

  
84

  
72 85
site_export = SiteExportView.as_view()
73 86

  
74 87

  
75 88
class SiteImportView(FormView):
76 89
    form_class = SiteImportForm
77 90
    template_name = 'combo/site_import.html'
78 91
    success_url = reverse_lazy('combo-manager-homepage')
79 92

  
80 93
    def form_valid(self, form):
94
        fd = self.request.FILES['site_file'].file
81 95
        try:
82
            json_site = json.loads(force_text(self.request.FILES['site_json'].read()))
83
        except ValueError:
84
            form.add_error('site_json', _('File is not in the expected JSON format.'))
85
            return self.form_invalid(form)
86

  
96
            tarfile.open(mode='r', fileobj=fd)
97
        except tarfile.TarError as e:
98
            try:
99
                fd.seek(0)
100
                json_site = json.loads(force_text(fd.read()))
101
            except ValueError:
102
                form.add_error('site_file', _('File is not in the expected TAR or JSON format.'))
103
                return self.form_invalid(form)
104
            else:
105
                format = 'json'
106
        else:
107
            format = 'tar'
108
            fd.seek(0)
87 109
        try:
88
            import_site(json_site, request=self.request)
110
            if format == 'json':
111
                import_site(json_site, request=self.request)
112
            else:
113
                import_site_tar(fd, request=self.request)
89 114
        except MissingGroups as e:
90
            form.add_error('site_json', force_text(e))
115
            form.add_error('site_file', force_text(e))
91 116
            return self.form_invalid(form)
92 117

  
93 118
        return super(SiteImportView, self).form_valid(form)
94 119

  
95 120

  
96 121
site_import = SiteImportView.as_view()
97 122

  
98 123

  
tests/test_manager.py
1 1
import base64
2 2
import datetime
3 3
import json
4 4
import mock
5 5
import os
6 6
import re
7 7
import shutil
8 8

  
9
from django.core.files import File
9 10
from django.core.files.storage import default_storage
10 11
from django.urls import reverse
11 12
from django.conf import settings
12 13
from django.contrib.auth.models import Group
13 14
from django.db import connection
14 15
from django.template import TemplateSyntaxError
15 16
from django.test import override_settings
16 17
from django.test.client import RequestFactory
......
551 552
    random_list = [page3, page4, page1, page2]
552 553
    ordered_list = Page.get_as_reordered_flat_hierarchy(random_list)
553 554
    assert ordered_list[0] == page1
554 555
    assert ordered_list[1] == page4
555 556
    assert ordered_list[2] == page2
556 557
    assert ordered_list[3] == page3
557 558

  
558 559

  
559
def test_site_export_import(app, admin_user):
560
def test_site_export_import_json(app, admin_user):
560 561
    Page.objects.all().delete()
561 562
    page1 = Page(title='One', slug='one', template_name='standard')
562 563
    page1.save()
563 564
    page2 = Page(title='Two', slug='two', parent=page1, template_name='standard')
564 565
    page2.save()
565 566
    page3 = Page(title='Three', slug='three', parent=page2, template_name='standard')
566 567
    page3.save()
567 568
    page4 = Page(title='Four', slug='four', parent=page1, template_name='standard')
......
574 575
    cell.save()
575 576

  
576 577
    cell = LinkCell(page=page2, placeholder='content', link_page=page1, order=0)
577 578
    cell.save()
578 579

  
579 580
    app = login(app)
580 581
    resp = app.get('/manage/')
581 582
    resp = resp.click('Export Site')
583
    resp = resp.form.submit()
582 584
    assert resp.headers['content-type'] == 'application/json'
583 585
    site_export = resp.body
584 586

  
585 587
    Page.objects.all().delete()
586 588
    assert LinkCell.objects.count() == 0
587 589
    app = login(app)
588 590
    resp = app.get('/manage/')
589 591
    resp = resp.click('Import Site')
590
    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
592
    resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json')
591 593
    resp = resp.form.submit()
592 594
    assert Page.objects.count() == 4
593 595
    assert LinkCell.objects.count() == 2
594 596
    assert LinkCell.objects.get(page__slug='one').link_page.slug == 'two'
595 597
    assert LinkCell.objects.get(page__slug='two').link_page.slug == 'one'
596 598

  
597 599
    # check with invalid file
598 600
    resp = app.get('/manage/')
599 601
    resp = resp.click('Import Site')
600
    resp.form['site_json'] = Upload('site-export.json', b'invalid content', 'application/json')
602
    resp.form['site_file'] = Upload('site-export.json', b'invalid content', 'application/json')
603
    resp = resp.form.submit()
604
    assert 'File is not in the expected TAR or JSON format.' in resp.text
605

  
606

  
607
def test_site_export_import_tar(app, admin_user):
608
    Page.objects.all().delete()
609
    page1 = Page(title='One', slug='one', template_name='standard')
610
    page1.save()
611
    cell = TextCell(page=page1, placeholder='content', text='Foobar', order=0)
612
    cell.save()
613
    Asset(key='collectivity:banner', asset=File(BytesIO(b'test'), 'test.png')).save()
614
    path = default_storage.path('')
615
    assert open('%s/assets/test.png' % path, 'r').read() == 'test'
616

  
617
    app = login(app)
618
    resp = app.get('/manage/')
619
    resp = resp.click('Export Site')
620
    resp.form['include_asset'] = True
621
    resp = resp.form.submit()
622
    assert resp.headers['content-type'] == 'application/x-tar'
623
    site_export = resp.body
624

  
625
    Page.objects.all().delete()
626
    Asset.objects.filter(key='collectivity:banner').delete()
627
    assert Page.objects.count() == 0
628
    assert TextCell.objects.count() == 0
629
    assert Asset.objects.filter(key='collectivity:banner').count() == 0
630
    open('%s/assets/test.png' % path, 'w').write('foo')
631
    app = login(app)
632
    resp = app.get('/manage/')
633
    resp = resp.click('Import Site')
634
    resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json')
635
    resp = resp.form.submit()
636
    assert Page.objects.count() == 1
637
    assert TextCell.objects.count() == 1
638
    assert Asset.objects.filter(key='collectivity:banner').count() == 1
639
    assert open('%s/assets/test.png' % path, 'r').read() == 'foo'
640

  
641
    os.remove('%s/assets/test.png' % path)
642
    app = login(app)
643
    resp = app.get('/manage/')
644
    resp = resp.click('Import Site')
645
    resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json')
601 646
    resp = resp.form.submit()
602
    assert 'File is not in the expected JSON format.' in resp.text
647
    assert open('%s/assets/test.png' % path, 'r').read() == 'test'
603 648

  
604 649

  
605 650
def test_site_export_import_missing_group(app, admin_user):
606 651
    Page.objects.all().delete()
607 652
    group = Group.objects.create(name='foobar')
608 653
    page1 = Page(title='One', slug='one', template_name='standard')
609 654
    page1.save()
610 655
    page1.groups.set([group])
611 656

  
612 657
    app = login(app)
613 658
    resp = app.get('/manage/')
614 659
    resp = resp.click('Export Site')
660
    resp = resp.form.submit()
615 661
    assert resp.headers['content-type'] == 'application/json'
616 662
    site_export = resp.body
617 663

  
618 664
    Page.objects.all().delete()
619 665
    group.delete()
620 666

  
621 667
    app = login(app)
622 668
    resp = app.get('/manage/')
623 669
    resp = resp.click('Import Site')
624
    resp.form['site_json'] = Upload('site-export.json', site_export, 'application/json')
670
    resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json')
625 671
    resp = resp.form.submit()
626 672
    assert 'Missing groups: foobar' in resp.text
627 673

  
628 674

  
629 675
def test_site_export_import_unknown_parent(app, admin_user):
630 676
    Page.objects.create(title='One', slug='one', template_name='standard')
631 677
    Page.objects.create(title='Two', slug='two', template_name='standard')
632 678

  
633 679
    app = login(app)
634 680
    resp = app.get('/manage/')
635 681
    resp = resp.click('Export Site')
682
    resp = resp.form.submit()
636 683
    payload = json.loads(force_str(resp.body))
637 684
    payload['pages'][0]['fields']['exclude_from_navigation'] = False
638 685
    payload['pages'][0]['fields']['parent'] = ['unknown-parent']
639 686

  
640 687
    resp = app.get('/manage/')
641 688
    resp = resp.click('Import Site')
642
    resp.form['site_json'] = Upload('site-export.json', force_bytes(json.dumps(payload)), 'application/json')
689
    resp.form['site_file'] = Upload('site-export.json', force_bytes(json.dumps(payload)), 'application/json')
643 690
    resp = resp.form.submit().follow()
644 691
    assert 'Unknown parent for page &quot;One&quot;; parent has been reset and page was excluded from navigation.' in resp.text
645 692

  
646 693

  
647 694
def test_invalid_cell_report(app, admin_user):
648 695
    app = login(app)
649 696
    resp = app.get('/manage/cells/invalid-report/')
650 697
    assert resp.context['object_list'] == []
651
-