Projet

Général

Profil

0001-csvdatasource-store-data-in-temporary-file-when-load.patch

Benjamin Dauvergne, 23 avril 2020 21:12

Télécharger (3,56 ko)

Voir les différences: 

Subject: [PATCH] csvdatasource: store data in temporary file when loading
 excel sheets (#40704)


 passerelle/apps/csvdatasource/models.py | 10 ++++++-
 tests/test_csv_datasource.py            | 36 +++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)
passerelle/apps/csvdatasource/models.py
18 18 
import re
19 19 
import csv
20 20 
from collections import OrderedDict
21 
import tempfile
21 22 

  		




  22
  23
  
    
import six

  		




  23
  24
  
    
import pytz

  		




  ......




  243
  244
  
    
                content = get_data_ods(self.csv_file)

  		




  244
  245
  
    

  		




  245
  246
  
    
            elif file_type == 'xls' or file_type == 'xlsx':

  		




  246
  
  
    
                content = get_data_xls(self.csv_file.path)

  		




  
  247
  
    
                # Suffix is necessary as pyexcel is too stupid to detect the

  		




  
  248
  
    
                # filetype from content

  		




  
  249
  
    
                with tempfile.NamedTemporaryFile(mode='wb', suffix='.' + file_type) as fd:

  		




  
  250
  
    
                    self.csv_file.seek(0)

  		




  
  251
  
    
                    for buf in iter(lambda: self.csv_file.read(32768), b''):

  		




  
  252
  
    
                        fd.write(buf)

  		




  
  253
  
    
                    fd.flush()

  		




  
  254
  
    
                    content = get_data_xls(fd.name)

  		




  247
  255
  
    

  		




  248
  256
  
    
            if len(content.keys()) == 1:

  		




  249
  257
  
    
                # if there's a single sheet, ignore specified sheet name and

  		












  

    
      tests/test_csv_datasource.py
    
  





  119
  119
  
    
def filetype(request):

  		




  120
  120
  
    
    return request.param

  		




  121
  121
  
    

  		




  
  122
  
    
@pytest.fixture

  		




  
  123
  
    
def sheet_name(filetype):

  		




  
  124
  
    
    return 'Feuille2' if filetype != 'data.csv' else ''

  		




  
  125
  
    

  		




  
  126
  
    

  		




  
  127
  
    
@pytest.fixture

  		




  
  128
  
    
def file_content(filetype):

  		




  
  129
  
    
    with get_file_content(filetype) as fd:

  		




  
  130
  
    
        yield fd.read()

  		




  
  131
  
    

  		




  
  132
  
    

  		




  122
  133
  
    
def test_default_column_keynames(setup, filetype):

  		




  123
  134
  
    
    csvdata = CsvDataSource.objects.create(csv_file=File(get_file_content(filetype), filetype),

  		




  124
  135
  
    
                                           sheet_name='Feuille2',

  		




  ......




  855
  866
  
    
        app = login(app)

  		




  856
  867
  
    
        response = app.get(csvdata.get_absolute_url())

  		




  857
  868
  
    
        assert 'Oct. 27, 2019, 2:20 a.m.' in response

  		




  
  869
  
    

  		




  
  870
  
    

  		




  
  871
  
    
def test_view_manage_create(app, admin_user, filetype, file_content, sheet_name):

  		




  
  872
  
    
    app = login(app)

  		




  
  873
  
    
    response = app.get(reverse('create-connector', kwargs={'connector': 'csvdatasource'}))

  		




  
  874
  
    
    response.form.set('title', 'test title')

  		




  
  875
  
    
    response.form.set('slug', 'test-slug')

  		




  
  876
  
    
    response.form.set('description', 'test description')

  		




  
  877
  
    
    response.form.set('csv_file', webtest.Upload(filetype, file_content, 'application/octet-stream'))

  		




  
  878
  
    
    response.form.set('columns_keynames', 'a,b,c,d,e')

  		




  
  879
  
    
    if sheet_name:

  		




  
  880
  
    
        response.form.set('sheet_name', sheet_name)

  		




  
  881
  
    
    response = response.form.submit()

  		




  
  882
  
    
    assert response.location

  		




  
  883
  
    
    response = response.follow()

  		




  
  884
  
    
    assert 'test title' in response

  		




  
  885
  
    
    assert 'test description' in response

  		




  
  886
  
    
    assert CsvDataSource.objects.count() == 1

  		




  
  887
  
    
    resource = CsvDataSource.objects.get()

  		




  
  888
  
    
    assert resource.title == 'test title'

  		




  
  889
  
    
    assert resource.slug == 'test-slug'

  		




  
  890
  
    
    assert resource.description == 'test description'

  		




  
  891
  
    
    assert resource.csv_file.read() == file_content

  		




  
  892
  
    
    assert resource.columns_keynames == 'a,b,c,d,e'

  		




  
  893
  
    
    assert resource.sheet_name == sheet_name

  		




  858
  
  
    
-