Projet

Général

Profil

0001-assets-add-content-exclusion-options-to-export-comma.patch

Nicolas Roche, 10 février 2021 13:24

Télécharger (18 ko)

Voir les différences:

Subject: [PATCH] assets: add content exclusion options to export command
 (#50399)

 combo/data/management/commands/export_site.py | 39 ++++++++---
 combo/data/utils.py                           |  6 +-
 tests/test_assets.py                          |  1 +
 tests/test_import_export.py                   | 64 +++++++++++++------
 4 files changed, 79 insertions(+), 31 deletions(-)
combo/data/management/commands/export_site.py
27 27

  
28 28
    def add_arguments(self, parser):
29 29
        parser.add_argument(
30 30
                '--output', metavar='FILE', default=None,
31 31
                help='name of a file to write output to')
32 32
        parser.add_argument(
33 33
                '--format-json', action='store_true', default=False,
34 34
                help='use JSON format with no asset files')
35
        parser.add_argument(
36
                '--exclude-pages', action='store_true', default=False,
37
                help='exclude pages from the export')
38
        parser.add_argument(
39
                '--exclude-cartography', action='store_true', default=False,
40
                help='exclude cartography from the export')
41
        parser.add_argument(
42
                '--exclude-pwa', action='store_true', default=False,
43
                help='exclude pwa from the export')
44
        parser.add_argument(
45
                '--exclude-assets', action='store_true', default=False,
46
                help='exclude assets from the export')
47
        parser.add_argument(
48
                '--exclude-payment', action='store_true', default=False,
49
                help='exclude payment from the export')
35 50

  
36 51
    def handle(self, *args, **options):
37
        if options['format_json']:
38
            if options['output'] and options['output'] != '-':
39
                output = open(options['output'], 'w')
52
        params = {}
53
        for content in 'pages', 'cartography', 'pwa', 'assets', 'payment':
54
            if options.get('exclude_%s' % content):
55
                params[content] = False
56
        output = options.get('output') or '-'
57

  
58
        if options.get('format_json'):
59
            if output != '-':
60
                file_fd = open(output, 'w')
40 61
            else:
41
                output = sys.stdout
42
            json.dump(export_site(), output, indent=2)
62
                file_fd = sys.stdout
63
            json.dump(export_site(**params), file_fd, indent=2)
43 64
        else:
44
            if options['output'] and options['output'] != '-':
65
            if output != '-':
45 66
                try:
46
                    output = open(options['output'], 'wb')
67
                    file_fd = open(output, 'wb')
47 68
                except IOError as e:
48 69
                    raise CommandError(e)
49
                export_site_tar(output)
50
                output.close()
70
                export_site_tar(file_fd, **params)
71
                file_fd.close()
51 72
            else:
52 73
                raise CommandError(_('TAR format require output filename parameter'))
combo/data/utils.py
135 135
            except IndexError:
136 136
                raise ImportSiteError(message)
137 137
            raise ImportSiteError(_('Unknown page "%s".') % page_slug)
138 138

  
139 139

  
140 140
def export_site_tar(fd, export_kwargs=None):
141 141
    tar = tarfile.open(mode='w', fileobj=fd)
142 142
    data = export_site(**(export_kwargs or {}))
143
    del data['assets']
143
    if data.get('assets'):
144
        del data['assets']
144 145
    add_tar_content(tar, '_site.json', json.dumps(data, indent=2))
145
    tar_assets_files(tar)
146
    if not (export_kwargs and export_kwargs.get('exclude_assets')):
147
        tar_assets_files(tar)
146 148
    tar.close()
147 149

  
148 150

  
149 151
def import_site_tar(fd, if_empty=False, clean=False, overwrite=False, request=None):
150 152
    tar = tarfile.open(mode='r', fileobj=fd)
151 153
    try:
152 154
        tarinfo = tar.getmember('_site.json')
153 155
    except KeyError:
tests/test_assets.py
138 138
    clean_assets_files()
139 139
    assert count_asset_files() == 0
140 140
    clean_assets_files()
141 141

  
142 142

  
143 143
def test_assets_export_size_view(app, some_assets):
144 144
    resp = app.get(reverse('combo-manager-assets-export-size'))
145 145
    assert resp.text.split() == ['(9', 'bytes)']
146
    clean_assets_files()
tests/test_import_export.py
37 37
    cell.save()
38 38

  
39 39

  
40 40
@pytest.fixture
41 41
def some_map_layers():
42 42
    MapLayer(label='Foo', slug='foo', geojson_url='http://example.net/foo/').save()
43 43
    MapLayer(label='Bar', slug='bar', geojson_url='http://example.net/bar/').save()
44 44

  
45

  
45 46
@pytest.fixture
46 47
def some_assets():
47 48
    Asset(key='banner', asset=File(BytesIO(b'test'), 'test.png')).save()
48 49
    Asset(key='favicon', asset=File(BytesIO(b'test2'), 'test2.png')).save()
49 50

  
51

  
50 52
def get_output_of_command(command, *args, **kwargs):
53
    options = {
54
        'exclude_pages': True,
55
        'exclude_cartography': True,
56
        'exclude_pwa': True,
57
        'exclude_assets': True,
58
        'exclude_payment': True,
59
    }
60
    options.update(kwargs)
51 61
    old_stdout = sys.stdout
52 62
    output = sys.stdout = StringIO()
53
    call_command(command, format_json=True, *args, **kwargs)
63
    call_command(command, format_json=True, *args, **options)
54 64
    sys.stdout = old_stdout
55 65
    return output.getvalue()
56 66

  
57 67

  
58 68
def test_import_export(app, some_data):
59
    output = get_output_of_command('export_site')
69
    assert 'pages' not in json.loads(get_output_of_command('export_site'))
70
    output = get_output_of_command('export_site', exclude_pages=False)
60 71
    assert len(json.loads(output)['pages']) == 3
61 72
    import_site(data={}, clean=True)
62 73
    assert Page.objects.all().count() == 0
63 74
    assert TextCell.objects.all().count() == 0
64
    empty_output = get_output_of_command('export_site')
75
    empty_output = get_output_of_command('export_site', exclude_pages=False)
65 76
    assert len(json.loads(empty_output)['pages']) == 0
66 77

  
67 78
    Page(title='test', slug='test').save()
68 79
    old_stdin = sys.stdin
69 80
    sys.stdin = StringIO(json.dumps({}))
70 81
    assert Page.objects.count() == 1
71 82
    try:
72 83
        call_command('import_site', '-', clean=True)
......
83 94
    assert TextCell.objects.all().count() == 1
84 95

  
85 96
    import_site(data={}, if_empty=True)
86 97
    assert Page.objects.count() == 3
87 98
    assert TextCell.objects.all().count() == 1
88 99

  
89 100
    import_site(data=[], clean=True)
90 101
    tempdir = tempfile.mkdtemp('chrono-test')
91
    empty_output = get_output_of_command('export_site', output=os.path.join(tempdir, 't.json'))
102
    empty_output = get_output_of_command(
103
        'export_site', exclude_pages=False, output=os.path.join(tempdir, 't.json'))
92 104
    assert os.path.exists(os.path.join(tempdir, 't.json'))
93 105
    shutil.rmtree(tempdir)
94 106

  
95 107

  
96 108
def test_import_export_with_parent(app, some_data):
97
    output = get_output_of_command('export_site')
109
    output = get_output_of_command('export_site', exclude_pages=False)
98 110
    payload = json.loads(output)
99 111
    payload['pages'][1]['fields']['parent'] = ['one']
100 112

  
101 113
    Page.objects.all().delete()
102 114
    import_site(data=payload)
103 115

  
104 116
    assert Page.objects.count() == 3
105 117
    two = Page.objects.get(slug='two')
106 118
    assert two.parent.slug == 'one'
107 119

  
108 120

  
109 121
def test_import_export_with_unknown_parent(app, some_data):
110
    output = get_output_of_command('export_site')
122
    output = get_output_of_command('export_site', exclude_pages=False)
111 123
    payload = json.loads(output)
112 124
    payload['pages'][0]['fields']['exclude_from_navigation'] = False
113 125
    payload['pages'][0]['fields']['parent'] = ['unknown-parent']
114 126

  
115 127
    Page.objects.all().delete()
116 128
    import_site(data=payload)
117 129

  
118 130
    assert Page.objects.count() == 3
......
123 135

  
124 136

  
125 137
def test_backward_compatibility_import(app, some_data):
126 138
    old_export = Page.export_all_for_json()
127 139
    Page.objects.all().delete()
128 140
    import_site(data=old_export)
129 141
    assert Page.objects.count() == 3
130 142

  
143

  
131 144
def test_import_export_map_layers(app, some_map_layers):
132
    output = get_output_of_command('export_site')
145
    assert 'map-layers' not in json.loads(get_output_of_command('export_site'))
146
    output = get_output_of_command('export_site', exclude_cartography=False)
133 147
    assert len(json.loads(output)['map-layers']) == 2
134 148
    import_site(data={}, clean=True)
135 149
    assert MapLayer.objects.all().count() == 0
136
    empty_output = get_output_of_command('export_site')
150
    empty_output = get_output_of_command('export_site', exclude_cartography=False)
137 151
    assert len(json.loads(empty_output)['map-layers']) == 0
138 152

  
139 153
    MapLayer(label='Baz', slug='baz', geojson_url='http://example.net/baz/').save()
140 154
    old_stdin = sys.stdin
141 155
    sys.stdin = StringIO(json.dumps({}))
142 156
    assert MapLayer.objects.count() == 1
143 157
    try:
144 158
        call_command('import_site', '-', clean=True)
......
157 171
    assert MapLayer.objects.count() == 2
158 172

  
159 173

  
160 174
def test_import_export_map_cells(app, some_data, some_map_layers):
161 175
    page = Page.objects.get(slug='one')
162 176
    cell = Map(page=page, order=0, placeholder='content')
163 177
    cell.save()
164 178
    MapLayerOptions.objects.create(map_cell=cell, map_layer=MapLayer.objects.get(slug='foo'))
165
    site_export = get_output_of_command('export_site')
179
    site_export = get_output_of_command('export_site', exclude_pages=False, exclude_cartography=False)
166 180
    import_site(data={}, clean=True)
167 181
    assert Map.objects.count() == 0
168 182
    assert MapLayer.objects.count() == 0
169 183

  
170 184
    site_data = json.loads(site_export)
171 185
    import_site(data=site_data, clean=True)
172 186
    assert Map.objects.count() == 1
173 187
    assert MapLayer.objects.filter(slug='foo').exists() is True
......
193 207
    page = Page.objects.get(slug='one')
194 208
    page.groups.set([group])
195 209
    page.save()
196 210

  
197 211
    cell = TextCell.objects.get(order=0)
198 212
    cell.groups.set([group])
199 213
    cell.save()
200 214

  
201
    output = get_output_of_command('export_site')
215
    output = get_output_of_command('export_site', exclude_pages=False)
202 216
    assert len(json.loads(output)['pages']) == 3
203 217
    import_site(data={}, clean=True)
204 218
    assert Page.objects.all().count() == 0
205 219
    assert TextCell.objects.all().count() == 0
206 220

  
207 221
    Group.objects.all().delete()
208 222

  
209 223
    with pytest.raises(MissingGroups) as excinfo:
......
226 240
    assert TextCell.objects.all().count() == 1
227 241

  
228 242
    page = Page.objects.get(slug='one')
229 243
    assert [x.name for x in page.groups.all()] == ['A Group']
230 244

  
231 245
    cell = TextCell.objects.get(order=0)
232 246
    assert [x.name for x in cell.groups.all()] == ['A Group']
233 247

  
248

  
234 249
def test_import_export_assets(app, some_assets):
235
    output = get_output_of_command('export_site')
250
    assert 'assets' not in json.loads(get_output_of_command('export_site'))
251
    output = get_output_of_command('export_site', exclude_assets=False)
236 252
    assert len(json.loads(output)['assets']) == 2
237 253
    import_site(data={}, clean=True)
238 254
    assert Asset.objects.all().count() == 0
239
    empty_output = get_output_of_command('export_site')
255
    empty_output = get_output_of_command('export_site', exclude_assets=False)
240 256
    assert len(json.loads(empty_output)['assets']) == 0
241 257

  
242 258
    Asset(key='footer', asset=File(StringIO('test3'), 'test3.png')).save()
243 259
    old_stdin = sys.stdin
244 260
    sys.stdin = StringIO(json.dumps({}))
245 261
    assert Asset.objects.count() == 1
246 262
    try:
247 263
        call_command('import_site', '-', clean=True)
......
254 270
        f.flush()
255 271
        call_command('import_site', f.name)
256 272

  
257 273
    assert Asset.objects.count() == 2
258 274

  
259 275
    import_site(data={}, if_empty=True)
260 276
    assert Asset.objects.count() == 2
261 277

  
278

  
262 279
def test_import_export_pwa_settings(app):
263
    output = get_output_of_command('export_site')
280
    output = get_output_of_command('export_site', exclude_pwa=False)
264 281
    pwa_settings = PwaSettings.singleton()
265 282
    pwa_settings.offline_text = 'Hello world'
266 283
    pwa_settings.offline_retry_button = False
267 284
    pwa_settings.save()
268 285

  
269
    # check exort with no application icon
270
    output = get_output_of_command('export_site')
286
    # check export with no application icon
287
    output = get_output_of_command('export_site', exclude_pwa=False)
271 288

  
272 289
    pwa_settings.application_icon = File(BytesIO(b'te\30st'), 'test.png')
273 290
    pwa_settings.save()
274
    output = get_output_of_command('export_site')
291
    output = get_output_of_command('export_site', exclude_pwa=False)
275 292
    import_site(data={}, clean=True)
276 293
    assert PwaSettings.objects.all().count() == 0
277 294

  
278 295
    import_site(data=json.loads(output))
279 296
    assert PwaSettings.singleton().offline_retry_button is False
280 297
    assert PwaSettings.singleton().offline_text == 'Hello world'
281 298
    # check identical file was not touched
282 299
    assert os.path.basename(PwaSettings.objects.get().application_icon.file.name) == 'test.png'
......
291 308
    # check with a change in icon file name
292 309
    data = json.loads(output)
293 310
    data['pwa']['settings']['icon:base64'] = force_text(base64.encodebytes(b'TEST2'))
294 311
    data['pwa']['settings']['application_icon'] = 'pwa/test2.png'
295 312
    import_site(data=data)
296 313
    assert os.path.basename(PwaSettings.objects.get().application_icon.file.name) == 'test2.png'
297 314
    assert PwaSettings.objects.get().application_icon.read() == b'TEST2'
298 315

  
316

  
299 317
def test_import_export_pwa_navigation(app, some_data):
300 318
    page = Page.objects.get(slug='one')
301 319
    entry1 = PwaNavigationEntry(label='a', url='/', order=0)
302 320
    entry2 = PwaNavigationEntry(link_page=page, order=1, icon=File(BytesIO(b'te\30st'), 'test.png'))
303 321
    entry1.save()
304 322
    entry2.save()
305
    output = get_output_of_command('export_site')
323
    assert 'pwa' not in json.loads(get_output_of_command('export_site'))
324
    output = get_output_of_command('export_site', exclude_pages=False, exclude_pwa=False)
325
    assert len(json.loads(output)['pwa']['navigation']) == 2
326

  
306 327
    import_site(data={}, clean=True)
307 328
    assert PwaNavigationEntry.objects.all().count() == 0
308 329

  
309 330
    import_site(data=json.loads(output))
310 331
    assert PwaNavigationEntry.objects.all().count() == 2
311 332
    # check identical file was not touched
312 333
    assert os.path.basename(PwaNavigationEntry.objects.get(order=1).icon.file.name) == 'test.png'
313 334
    assert PwaNavigationEntry.objects.get(order=1).icon.read() == b'te\30st'
......
327 348
    data = json.loads(output)
328 349
    data['pwa']['navigation'][1]['fields']['icon'] = 'pwa/test2.png'
329 350
    data['pwa']['navigation'][1]['icon:base64'] = force_text(base64.encodebytes(b'TEST2'))
330 351
    import_site(data=data)
331 352
    assert PwaNavigationEntry.objects.all().count() == 2
332 353
    assert os.path.basename(PwaNavigationEntry.objects.get(order=1).icon.file.name) == 'test2.png'
333 354
    assert PwaNavigationEntry.objects.get(order=1).icon.read() == b'TEST2'
334 355

  
356

  
335 357
def test_import_export_gallery_images(app, some_data):
336 358
    page = Page.objects.get(slug='one')
337 359
    gallery = GalleryCell(page=page, order=2, placeholder='images')
338 360
    gallery.save()
339 361
    image1 = Image(gallery=gallery, image='path/foo.jpg', title='foo', order=1)
340 362
    image2 = Image(gallery=gallery, image='path/bar.jpg', title='bar', order=2)
341 363
    image1.save()
342 364
    image2.save()
343
    output = get_output_of_command('export_site')
365
    output = get_output_of_command('export_site', exclude_pages=False)
344 366
    import_site(data={}, clean=True)
345 367
    assert Image.objects.all().count() == 0
346 368

  
347 369
    import_site(data=json.loads(output))
348 370
    assert Image.objects.all().count() == 2
349 371
    image1 = Image.objects.get(title='foo')
350 372
    assert image1.image == 'path/foo.jpg'
351 373
    assert image1.gallery.placeholder == 'images'
352 374

  
375

  
353 376
def test_import_export_extra_fields(app, some_data):
354 377
    site_export = export_site()
355 378
    for page in site_export['pages']:
356 379
        if page['fields']['slug'] == 'one':
357 380
            page['fields']['extra_field_not_in_model'] = True
358 381
        elif page['fields']['slug'] == 'three':
359 382
            page['cells'][0]['fields']['extra_field_not_in_model'] = True
360 383

  
......
424 447
        call_command('import_site', filename)
425 448

  
426 449

  
427 450
def test_import_export_payment(app):
428 451
    backend = PaymentBackend.objects.create(label='Test', slug='test', service_options={'test': True})
429 452
    Regie.objects.create(label='Test regie', slug='test-regie', payment_backend=backend)
430 453
    Regie.objects.create(label='Test regie 2', slug='test-regie-2', payment_backend=backend)
431 454

  
432
    output = get_output_of_command('export_site')
455
    assert 'payment' not in json.loads(get_output_of_command('export_site'))
456
    output = get_output_of_command('export_site', exclude_payment=False)
433 457
    payload = json.loads(output)
434 458
    assert len(payload['payment']['backends']) == 1
435 459
    assert len(payload['payment']['regies']) == 2
436 460

  
437 461
    import_site(payload)
438 462
    assert PaymentBackend.objects.count() == 1
439 463
    assert Regie.objects.count() == 2
440 464

  
......
449 473
    assert Regie.objects.count() == 2
450 474
    regie = Regie.objects.first()
451 475
    assert regie.payment_backend == backend
452 476

  
453 477
    import_site(data={}, clean=True)
454 478
    assert PaymentBackend.objects.count() == 0
455 479
    assert Regie.objects.count() == 0
456 480

  
457
    empty_output = get_output_of_command('export_site')
481
    empty_output = get_output_of_command('export_site', exclude_payment=False)
458 482
    assert len(json.loads(empty_output)['payment']['backends']) == 0
459 483
    assert len(json.loads(empty_output)['payment']['regies']) == 0
460
-