From 14a4cce0ba826269754a50c9f9836d4bf21b6771 Mon Sep 17 00:00:00 2001
From: Benjamin Dauvergne <bdauvergne@entrouvert.com>
Date: Thu, 23 Apr 2020 21:11:02 +0200
Subject: [PATCH] csvdatasource: store data in temporary file when loading
 excel sheets (#40704)

---
 passerelle/apps/csvdatasource/models.py | 10 ++++++-
 tests/test_csv_datasource.py            | 36 +++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git passerelle/apps/csvdatasource/models.py passerelle/apps/csvdatasource/models.py
index fe38e846..79f40827 100644
--- passerelle/apps/csvdatasource/models.py
+++ passerelle/apps/csvdatasource/models.py
@@ -18,6 +18,7 @@ import os
 import re
 import csv
 from collections import OrderedDict
+import tempfile
 
 import six
 import pytz
@@ -243,7 +244,14 @@ class CsvDataSource(BaseResource):
                 content = get_data_ods(self.csv_file)
 
             elif file_type == 'xls' or file_type == 'xlsx':
-                content = get_data_xls(self.csv_file.path)
+                # Suffix is necessary as pyexcel is too stupid to detect the
+                # filetype from content
+                with tempfile.NamedTemporaryFile(mode='wb', suffix='.' + file_type) as fd:
+                    self.csv_file.seek(0)
+                    for buf in iter(lambda: self.csv_file.read(32768), b''):
+                        fd.write(buf)
+                    fd.flush()
+                    content = get_data_xls(fd.name)
 
             if len(content.keys()) == 1:
                 # if there's a single sheet, ignore specified sheet name and
diff --git tests/test_csv_datasource.py tests/test_csv_datasource.py
index 96bd28e6..bc5535aa 100644
--- tests/test_csv_datasource.py
+++ tests/test_csv_datasource.py
@@ -119,6 +119,17 @@ def client():
 def filetype(request):
     return request.param
 
+@pytest.fixture
+def sheet_name(filetype):
+    return 'Feuille2' if filetype != 'data.csv' else ''
+
+
+@pytest.fixture
+def file_content(filetype):
+    with get_file_content(filetype) as fd:
+        yield fd.read()
+
+
 def test_default_column_keynames(setup, filetype):
     csvdata = CsvDataSource.objects.create(csv_file=File(get_file_content(filetype), filetype),
                                            sheet_name='Feuille2',
@@ -855,3 +866,28 @@ def test_csv_dst(app, setup, admin_user):
         app = login(app)
         response = app.get(csvdata.get_absolute_url())
         assert 'Oct. 27, 2019, 2:20 a.m.' in response
+
+
+def test_view_manage_create(app, admin_user, filetype, file_content, sheet_name):
+    app = login(app)
+    response = app.get(reverse('create-connector', kwargs={'connector': 'csvdatasource'}))
+    response.form.set('title', 'test title')
+    response.form.set('slug', 'test-slug')
+    response.form.set('description', 'test description')
+    response.form.set('csv_file', webtest.Upload(filetype, file_content, 'application/octet-stream'))
+    response.form.set('columns_keynames', 'a,b,c,d,e')
+    if sheet_name:
+        response.form.set('sheet_name', sheet_name)
+    response = response.form.submit()
+    assert response.location
+    response = response.follow()
+    assert 'test title' in response
+    assert 'test description' in response
+    assert CsvDataSource.objects.count() == 1
+    resource = CsvDataSource.objects.get()
+    assert resource.title == 'test title'
+    assert resource.slug == 'test-slug'
+    assert resource.description == 'test description'
+    assert resource.csv_file.read() == file_content
+    assert resource.columns_keynames == 'a,b,c,d,e'
+    assert resource.sheet_name == sheet_name
-- 
2.26.0