From f8753a45380f0cc474ab9153b9e975dfc9caaa09 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Tue, 22 Jun 2021 16:17:55 +0200
Subject: [PATCH] csvdatasource: add upload-csv-file endpoint (#29713)

---
 passerelle/apps/csvdatasource/models.py       | 44 +++++++++++++++
 .../csvdatasource/csvdatasource_detail.html   |  1 +
 tests/test_csv_datasource.py                  | 56 ++++++++++++++++++-
 3 files changed, 100 insertions(+), 1 deletion(-)

diff --git a/passerelle/apps/csvdatasource/models.py b/passerelle/apps/csvdatasource/models.py
index 096a12cb..2e0b40e9 100644
--- a/passerelle/apps/csvdatasource/models.py
+++ b/passerelle/apps/csvdatasource/models.py
@@ -9,42 +9,64 @@
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import base64
+import binascii
 import csv
 import datetime
 import os
 import re
 import tempfile
 from collections import OrderedDict
 
 import pytz
 import six
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
 from django.core.exceptions import ValidationError
+from django.core.files.base import ContentFile
 from django.db import models, transaction
+from django.db.utils import DataError
 from django.urls import reverse
 from django.utils.encoding import force_str, force_text, smart_text
 from django.utils.timezone import make_aware, now
 from django.utils.translation import ugettext_lazy as _
 from pyexcel_ods import get_data as get_data_ods
 from pyexcel_xls import get_data as get_data_xls
 
 from passerelle.base.models import BaseResource
 from passerelle.utils import batch
 from passerelle.utils.api import endpoint
 from passerelle.utils.conversion import normalize
 from passerelle.utils.jsonresponse import APIError
 
+SPECIAL_CHARS = '!#$%&+-^_`~;[]{}+=~'
+FILE_NAME_PATTERN = r'[\w%s\.]+$' % re.escape(SPECIAL_CHARS)
+UPLOAD_SCHEMA = {
+    'type': 'object',
+    'properties': {
+        'filename': {
+            'type': 'string',
+            'pattern': FILE_NAME_PATTERN,
+        },
+        'content': {'type': 'string'},
+        'sheet_name': {
+            'type': 'string',
+            'maxLength': 150,
+        },
+    },
+    'required': ['filename', 'content'],
+}
+
 identifier_re = re.compile(r"^[^\d\W]\w*\Z", re.UNICODE)
 
 
 code_cache = OrderedDict()
 
 
 def get_code(expr):
     # limit size of code cache to 1024
@@ -288,16 +310,38 @@ class CsvDataSource(BaseResource):
             self.cache_data()
             for data in self.get_cached_rows(initial=False):
                 yield data
 
     @property
     def titles(self):
         return [smart_text(t.strip()) for t in self.columns_keynames.split(',')]
 
+    @endpoint(
+        perm='can_access',
+        display_order=-1,
+        post={
+            'description': _('Modify CSV file'),
+            'request_body': {'schema': {'application/json': UPLOAD_SCHEMA}},
+        },
+    )
+    def update(self, request, post_data):
+        try:
+            file_byte_content = base64.b64decode(post_data['content'])
+        except (TypeError, binascii.Error):
+            raise APIError('file content must be a valid base64 string', http_status=400)
+        self.csv_file = ContentFile(content=file_byte_content, name=post_data['filename'])
+        self.sheet_name = post_data.get('sheet_name') or ''
+        try:
+            self.clean()
+        except ValidationError as e:
+            raise APIError(e, http_status=400)
+        self.save()
+        return {'created': self.csv_file.name}
+
     @endpoint(perm='can_access', methods=['get'], name='data')
     def data(self, request, **kwargs):
         params = request.GET
         filters = []
         for column_title in [t.strip() for t in self.columns_keynames.split(',') if t]:
             if column_title in params.keys():
                 query_value = request.GET.get(column_title, '')
                 if 'case-insensitive' in params:
diff --git a/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html b/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html
index a755dde1..e76dfc06 100644
--- a/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html
+++ b/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html
@@ -9,16 +9,17 @@
   {% if object|can_edit:request.user %}<a href="{% url 'csv-download' connector_slug=object.slug %}">{{object.csv_file}}</a>
   — {% trans "added on" %} {{object.csv_file_datetime}}
   {% else %}{{object.csv_file}}{% endif %}
 </p>
 {% endblock %}
 
 {% block endpoints %}
 <ul class="endpoints">
+{% include "passerelle/manage/endpoint.html" with endpoint=object.get_endpoints_infos.0 %}
 {% url 'generic-endpoint' connector='csvdatasource' slug=object.slug endpoint='data' as csvdatasource_data_url %}
 <li class="get-method">
   <div class="description"><span class="description--label">{% trans "Returning all file lines:" %}</span>
   <a class="example-url" href="{{ csvdatasource_data_url }}">{{ csvdatasource_data_url }}</a></div>
 </li>
 <li class="get-method">
   <div class="description"><span class="description--label">{% trans "Returning lines containing 'abc' in 'text' column (case insensitive):" %}</span>
   <a class="example-url" href="{{ csvdatasource_data_url }}?q=abc">{{ csvdatasource_data_url }}?q=abc</a></div>
diff --git a/tests/test_csv_datasource.py b/tests/test_csv_datasource.py
index 580644d7..7e5119a3 100644
--- a/tests/test_csv_datasource.py
+++ b/tests/test_csv_datasource.py
@@ -11,39 +11,41 @@
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import base64
 import datetime
 import os
 import time
 import uuid
 from posix import stat_result
 from stat import ST_MTIME
 
 import mock
 import pytest
 import six
 import webtest
 from django.contrib.contenttypes.models import ContentType
 from django.core.files import File
+from django.core.files.storage import default_storage
 from django.core.management import call_command
 from django.test import Client, override_settings
 from django.urls import reverse
 from django.utils.encoding import force_bytes, force_str, force_text
 from django.utils.six import StringIO
 from django.utils.six.moves.urllib.parse import urlencode
 from django.utils.timezone import now
 from test_manager import login
 
-from passerelle.apps.csvdatasource.models import CsvDataSource, Query, TableRow
+from passerelle.apps.csvdatasource.models import CsvDataSource, Query, TableRow, upload_to
 from passerelle.base.models import AccessRight, ApiUser
 from passerelle.compat import json_loads
 
 data = """121;69981;DELANOUE;Eliot;H
 525;6;DANIEL WILLIAMS;Shanone;F
 253;67742;MARTIN;Sandra;F
 511;38142;MARCHETTI;Lucie;F
 235;22;MARTIN;Sandra;F
@@ -862,16 +864,68 @@ def test_change_csv_command(setup):
     csv.refresh_from_db()
     assert list(csv.get_rows()) == []
 
     call_command('change-csv', 'test', os.path.join(TEST_BASE_DIR, 'data.ods'), sheet_name='Feuille2')
     csv.refresh_from_db()
     assert list(csv.get_rows()) != []
 
 
+def test_update(admin_user, app, setup):
+    csv, url = setup(data=StringIO(data))
+    upload_dir = default_storage.path(upload_to(csv, ''))
+    url = reverse(
+        'generic-endpoint',
+        kwargs={
+            'connector': 'csvdatasource',
+            'slug': csv.slug,
+            'endpoint': 'update',
+        },
+    )
+
+    assert CsvDataSource.objects.get().get_rows()[0]['fam'] == '121'
+    assert len([files for root, dirs, files in os.walk(upload_dir)][0]) == 1
+    assert 'data.csv' in [files for root, dirs, files in os.walk(upload_dir)][0]
+
+    params = {
+        'filename': 'data.csv',
+        'content': force_str(base64.b64encode(force_bytes('212' + data[3:]))),
+    }
+    app = login(app)
+    resp = app.post_json(url, params=params)
+    assert not resp.json['err']
+    assert CsvDataSource.objects.get().get_rows()[0]['fam'] == '212'
+    assert len([files for root, dirs, files in os.walk(upload_dir)][0]) == 2
+
+    params['content'] = 'no good'
+    resp = app.post_json(url, params=params, status=400)
+    assert resp.json['err']
+    assert 'file content must be a valid base64 string' in resp.json['err_desc']
+
+    params['content'] = force_str(base64.b64encode(b'\n'))
+    resp = app.post_json(url, params=params, status=400)
+    assert resp.json['err']
+    assert 'Could not detect CSV dialect' in resp.json['err_desc']
+
+    params['content'] = force_str(base64.b64encode(b'a,b,c\n1,2\0,3\n4,5,6'))
+    resp = app.post_json(url, params=params, status=400)
+    assert resp.json['err']
+    assert 'Invalid CSV file: line contains NUL' in resp.json['err_desc']
+
+    params['filename'] = 'data.ods'
+    resp = app.post_json(url, params=params, status=400)
+    assert resp.json['err']
+    assert 'You must specify a sheet name' in resp.json['err_desc']
+
+    params['sheet_name'] = 'Feuille2'
+    resp = app.post_json(url, params=params, status=400)
+    assert resp.json['err']
+    assert 'Invalid CSV file: File is not a zip file' in resp.json['err_desc']
+
+
 @pytest.mark.parametrize(
     'payload,expected',
     [
         ({}, 20),
         ({'limit': 10}, 10),
         ({'limit': 10, 'offset': 0}, 10),
         ({'limit': 10, 'offset': 15}, 5),
         ({'limit': 10, 'offset': 42}, 0),
-- 
2.30.2

