0001-csvdatasource-add-upload-csv-file-endpoint-29713.patch
passerelle/apps/csvdatasource/models.py | ||
---|---|---|
9 | 9 |
# This program is distributed in the hope that it will be useful, |
10 | 10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
11 | 11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 | 12 |
# GNU Affero General Public License for more details. |
13 | 13 |
# |
14 | 14 |
# You should have received a copy of the GNU Affero General Public License |
15 | 15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 |
import base64 |
|
18 |
import binascii |
|
17 | 19 |
import csv |
18 | 20 |
import datetime |
19 | 21 |
import os |
20 | 22 |
import re |
21 | 23 |
import tempfile |
22 | 24 |
from collections import OrderedDict |
23 | 25 | |
24 | 26 |
import pytz |
25 | 27 |
import six |
26 | 28 |
from django.conf import settings |
27 | 29 |
from django.contrib.postgres.fields import JSONField |
28 | 30 |
from django.core.exceptions import ValidationError |
31 |
from django.core.files.base import ContentFile |
|
29 | 32 |
from django.db import models, transaction |
33 |
from django.db.utils import DataError |
|
30 | 34 |
from django.urls import reverse |
31 | 35 |
from django.utils.encoding import force_str, force_text, smart_text |
32 | 36 |
from django.utils.timezone import make_aware, now |
33 | 37 |
from django.utils.translation import ugettext_lazy as _ |
34 | 38 |
from pyexcel_ods import get_data as get_data_ods |
35 | 39 |
from pyexcel_xls import get_data as get_data_xls |
36 | 40 | |
37 | 41 |
from passerelle.base.models import BaseResource |
38 | 42 |
from passerelle.utils import batch |
39 | 43 |
from passerelle.utils.api import endpoint |
40 | 44 |
from passerelle.utils.conversion import normalize |
41 | 45 |
from passerelle.utils.jsonresponse import APIError |
42 | 46 | |
47 |
SPECIAL_CHARS = '!#$%&+-^_`~;[]{}+=~' |
|
48 |
FILE_NAME_PATTERN = r'[\w%s\.]+$' % re.escape(SPECIAL_CHARS) |
|
49 |
UPLOAD_SCHEMA = { |
|
50 |
'type': 'object', |
|
51 |
'properties': { |
|
52 |
'filename': { |
|
53 |
'type': 'string', |
|
54 |
'pattern': FILE_NAME_PATTERN, |
|
55 |
}, |
|
56 |
'content': {'type': 'string'}, |
|
57 |
'sheet_name': { |
|
58 |
'type': 'string', |
|
59 |
'maxLength': 150, |
|
60 |
}, |
|
61 |
}, |
|
62 |
'required': ['filename', 'content'], |
|
63 |
} |
|
64 | ||
43 | 65 |
identifier_re = re.compile(r"^[^\d\W]\w*\Z", re.UNICODE) |
44 | 66 | |
45 | 67 | |
46 | 68 |
code_cache = OrderedDict() |
47 | 69 | |
48 | 70 | |
49 | 71 |
def get_code(expr): |
50 | 72 |
# limit size of code cache to 1024 |
... | ... | |
288 | 310 |
self.cache_data() |
289 | 311 |
for data in self.get_cached_rows(initial=False): |
290 | 312 |
yield data |
291 | 313 | |
292 | 314 |
@property |
293 | 315 |
def titles(self): |
294 | 316 |
return [smart_text(t.strip()) for t in self.columns_keynames.split(',')] |
295 | 317 | |
318 |
@endpoint( |
|
319 |
perm='can_access', |
|
320 |
display_order=-1, |
|
321 |
post={ |
|
322 |
'description': _('Modify CSV file'), |
|
323 |
'request_body': {'schema': {'application/json': UPLOAD_SCHEMA}}, |
|
324 |
}, |
|
325 |
) |
|
326 |
def update(self, request, post_data): |
|
327 |
try: |
|
328 |
file_byte_content = base64.b64decode(post_data['content']) |
|
329 |
except (TypeError, binascii.Error): |
|
330 |
raise APIError('file content must be a valid base64 string', http_status=400) |
|
331 |
self.csv_file = ContentFile(content=file_byte_content, name=post_data['filename']) |
|
332 |
self.sheet_name = post_data.get('sheet_name') or '' |
|
333 |
try: |
|
334 |
self.clean() |
|
335 |
except ValidationError as e: |
|
336 |
raise APIError(e, http_status=400) |
|
337 |
self.save() |
|
338 |
return {'created': self.csv_file.name} |
|
339 | ||
296 | 340 |
@endpoint(perm='can_access', methods=['get'], name='data') |
297 | 341 |
def data(self, request, **kwargs): |
298 | 342 |
params = request.GET |
299 | 343 |
filters = [] |
300 | 344 |
for column_title in [t.strip() for t in self.columns_keynames.split(',') if t]: |
301 | 345 |
if column_title in params.keys(): |
302 | 346 |
query_value = request.GET.get(column_title, '') |
303 | 347 |
if 'case-insensitive' in params: |
passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html | ||
---|---|---|
9 | 9 |
{% if object|can_edit:request.user %}<a href="{% url 'csv-download' connector_slug=object.slug %}">{{object.csv_file}}</a> |
10 | 10 |
— {% trans "added on" %} {{object.csv_file_datetime}} |
11 | 11 |
{% else %}{{object.csv_file}}{% endif %} |
12 | 12 |
</p> |
13 | 13 |
{% endblock %} |
14 | 14 | |
15 | 15 |
{% block endpoints %} |
16 | 16 |
<ul class="endpoints"> |
17 |
{% include "passerelle/manage/endpoint.html" with endpoint=object.get_endpoints_infos.0 %} |
|
17 | 18 |
{% url 'generic-endpoint' connector='csvdatasource' slug=object.slug endpoint='data' as csvdatasource_data_url %} |
18 | 19 |
<li class="get-method"> |
19 | 20 |
<div class="description"><span class="description--label">{% trans "Returning all file lines:" %}</span> |
20 | 21 |
<a class="example-url" href="{{ csvdatasource_data_url }}">{{ csvdatasource_data_url }}</a></div> |
21 | 22 |
</li> |
22 | 23 |
<li class="get-method"> |
23 | 24 |
<div class="description"><span class="description--label">{% trans "Returning lines containing 'abc' in 'text' column (case insensitive):" %}</span> |
24 | 25 |
<a class="example-url" href="{{ csvdatasource_data_url }}?q=abc">{{ csvdatasource_data_url }}?q=abc</a></div> |
tests/test_csv_datasource.py | ||
---|---|---|
11 | 11 |
# This program is distributed in the hope that it will be useful, |
12 | 12 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
13 | 13 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
14 | 14 |
# GNU Affero General Public License for more details. |
15 | 15 |
# |
16 | 16 |
# You should have received a copy of the GNU Affero General Public License |
17 | 17 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
18 | 18 | |
19 |
import base64 |
|
19 | 20 |
import datetime |
20 | 21 |
import os |
21 | 22 |
import time |
22 | 23 |
import uuid |
23 | 24 |
from posix import stat_result |
24 | 25 |
from stat import ST_MTIME |
25 | 26 | |
26 | 27 |
import mock |
27 | 28 |
import pytest |
28 | 29 |
import six |
29 | 30 |
import webtest |
30 | 31 |
from django.contrib.contenttypes.models import ContentType |
31 | 32 |
from django.core.files import File |
33 |
from django.core.files.storage import default_storage |
|
32 | 34 |
from django.core.management import call_command |
33 | 35 |
from django.test import Client, override_settings |
34 | 36 |
from django.urls import reverse |
35 | 37 |
from django.utils.encoding import force_bytes, force_str, force_text |
36 | 38 |
from django.utils.six import StringIO |
37 | 39 |
from django.utils.six.moves.urllib.parse import urlencode |
38 | 40 |
from django.utils.timezone import now |
39 | 41 |
from test_manager import login |
40 | 42 | |
41 |
from passerelle.apps.csvdatasource.models import CsvDataSource, Query, TableRow |
|
43 |
from passerelle.apps.csvdatasource.models import CsvDataSource, Query, TableRow, upload_to
|
|
42 | 44 |
from passerelle.base.models import AccessRight, ApiUser |
43 | 45 |
from passerelle.compat import json_loads |
44 | 46 | |
45 | 47 |
data = """121;69981;DELANOUE;Eliot;H |
46 | 48 |
525;6;DANIEL WILLIAMS;Shanone;F |
47 | 49 |
253;67742;MARTIN;Sandra;F |
48 | 50 |
511;38142;MARCHETTI;Lucie;F |
49 | 51 |
235;22;MARTIN;Sandra;F |
... | ... | |
862 | 864 |
csv.refresh_from_db() |
863 | 865 |
assert list(csv.get_rows()) == [] |
864 | 866 | |
865 | 867 |
call_command('change-csv', 'test', os.path.join(TEST_BASE_DIR, 'data.ods'), sheet_name='Feuille2') |
866 | 868 |
csv.refresh_from_db() |
867 | 869 |
assert list(csv.get_rows()) != [] |
868 | 870 | |
869 | 871 | |
872 |
def test_update(admin_user, app, setup): |
|
873 |
csv, url = setup(data=StringIO(data)) |
|
874 |
upload_dir = default_storage.path(upload_to(csv, '')) |
|
875 |
url = reverse( |
|
876 |
'generic-endpoint', |
|
877 |
kwargs={ |
|
878 |
'connector': 'csvdatasource', |
|
879 |
'slug': csv.slug, |
|
880 |
'endpoint': 'update', |
|
881 |
}, |
|
882 |
) |
|
883 | ||
884 |
assert CsvDataSource.objects.get().get_rows()[0]['fam'] == '121' |
|
885 |
assert len([files for root, dirs, files in os.walk(upload_dir)][0]) == 1 |
|
886 |
assert 'data.csv' in [files for root, dirs, files in os.walk(upload_dir)][0] |
|
887 | ||
888 |
params = { |
|
889 |
'filename': 'data.csv', |
|
890 |
'content': force_str(base64.b64encode(force_bytes('212' + data[3:]))), |
|
891 |
} |
|
892 |
app = login(app) |
|
893 |
resp = app.post_json(url, params=params) |
|
894 |
assert not resp.json['err'] |
|
895 |
assert CsvDataSource.objects.get().get_rows()[0]['fam'] == '212' |
|
896 |
assert len([files for root, dirs, files in os.walk(upload_dir)][0]) == 2 |
|
897 | ||
898 |
params['content'] = 'no good' |
|
899 |
resp = app.post_json(url, params=params, status=400) |
|
900 |
assert resp.json['err'] |
|
901 |
assert 'file content must be a valid base64 string' in resp.json['err_desc'] |
|
902 | ||
903 |
params['content'] = force_str(base64.b64encode(b'\n')) |
|
904 |
resp = app.post_json(url, params=params, status=400) |
|
905 |
assert resp.json['err'] |
|
906 |
assert 'Could not detect CSV dialect' in resp.json['err_desc'] |
|
907 | ||
908 |
params['content'] = force_str(base64.b64encode(b'a,b,c\n1,2\0,3\n4,5,6')) |
|
909 |
resp = app.post_json(url, params=params, status=400) |
|
910 |
assert resp.json['err'] |
|
911 |
assert 'Invalid CSV file: line contains NUL' in resp.json['err_desc'] |
|
912 | ||
913 |
params['filename'] = 'data.ods' |
|
914 |
resp = app.post_json(url, params=params, status=400) |
|
915 |
assert resp.json['err'] |
|
916 |
assert 'You must specify a sheet name' in resp.json['err_desc'] |
|
917 | ||
918 |
params['sheet_name'] = 'Feuille2' |
|
919 |
resp = app.post_json(url, params=params, status=400) |
|
920 |
assert resp.json['err'] |
|
921 |
assert 'Invalid CSV file: File is not a zip file' in resp.json['err_desc'] |
|
922 | ||
923 | ||
870 | 924 |
@pytest.mark.parametrize( |
871 | 925 |
'payload,expected', |
872 | 926 |
[ |
873 | 927 |
({}, 20), |
874 | 928 |
({'limit': 10}, 10), |
875 | 929 |
({'limit': 10, 'offset': 0}, 10), |
876 | 930 |
({'limit': 10, 'offset': 15}, 5), |
877 | 931 |
({'limit': 10, 'offset': 42}, 0), |
878 |
- |