From d9efea722f752fab157d54ae863200462f97e391 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Tue, 22 Jun 2021 16:17:55 +0200
Subject: [PATCH 2/2] csvdatasource: add upload-csv-file endpoint (#29713)

---
 passerelle/apps/csvdatasource/models.py       | 29 +++++++++
 .../csvdatasource/csvdatasource_detail.html   |  1 +
 tests/test_csv_datasource.py                  | 60 ++++++++++++++++++-
 tests/test_manager.py                         |  4 +-
 4 files changed, 91 insertions(+), 3 deletions(-)

diff --git a/passerelle/apps/csvdatasource/models.py b/passerelle/apps/csvdatasource/models.py
index 096a12cb..7ea5a071 100644
--- a/passerelle/apps/csvdatasource/models.py
+++ b/passerelle/apps/csvdatasource/models.py
@@ -11,26 +11,28 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import csv
 import datetime
+import mimetypes
 import os
 import re
 import tempfile
 from collections import OrderedDict
 
 import pytz
 import six
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
 from django.core.exceptions import ValidationError
+from django.core.files.base import ContentFile
 from django.db import models, transaction
 from django.urls import reverse
 from django.utils.encoding import force_str, force_text, smart_text
 from django.utils.timezone import make_aware, now
 from django.utils.translation import ugettext_lazy as _
 from pyexcel_ods import get_data as get_data_ods
 from pyexcel_xls import get_data as get_data_xls
 
@@ -139,16 +141,18 @@ class CsvDataSource(BaseResource):
     _dialect_options = JSONField(editable=False, null=True)
     sheet_name = models.CharField(_('Sheet name'), blank=True, max_length=150)
 
     category = _('Data Sources')
     documentation_url = (
         'https://doc-publik.entrouvert.com/admin-fonctionnel/parametrage-avance/source-de-donnees-csv/'
     )
 
+    _can_update_file_description = _('Uploading CSV file is limited to the following API users:')
+
     class Meta:
         verbose_name = _('Spreadsheet File')
 
     def clean(self, *args, **kwargs):
         file_type = self.csv_file.name.split('.')[-1]
         if file_type in ('ods', 'xls', 'xlsx') and not self.sheet_name:
             raise ValidationError(_('You must specify a sheet name'))
         if file_type not in ('ods', 'xls', 'xlsx'):
@@ -288,16 +292,41 @@ class CsvDataSource(BaseResource):
             self.cache_data()
             for data in self.get_cached_rows(initial=False):
                 yield data
 
     @property
     def titles(self):
         return [smart_text(t.strip()) for t in self.columns_keynames.split(',')]
 
+    @endpoint(
+        perm='can_update_file',
+        display_order=-1,
+        methods=['put'],
+        description=_('Modify CSV file'),
+    )
+    def update(self, request):
+        '''accept curl -T queries'''
+        content_type = request.environ['CONTENT_TYPE'] or ''
+        mimetype = content_type.split(';')[0].strip().lower()
+        ext = mimetypes.guess_extension(mimetype)
+        if not ext:
+            raise APIError(
+                "can't guess filename extension for '%s' content type" % content_type, http_status=400
+            )
+        storage = getattr(self.csv_file, 'storage')
+        name = storage.get_available_name('api-uploaded-file' + ext)
+        self.csv_file = ContentFile(content=request.body, name=name)
+        try:
+            self.clean()
+        except ValidationError as e:
+            raise APIError(e, http_status=400)
+        self.save()
+        return {'created': self.csv_file.name}
+
     @endpoint(perm='can_access', methods=['get'], name='data')
     def data(self, request, **kwargs):
         params = request.GET
         filters = []
         for column_title in [t.strip() for t in self.columns_keynames.split(',') if t]:
             if column_title in params.keys():
                 query_value = request.GET.get(column_title, '')
                 if 'case-insensitive' in params:
diff --git a/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html b/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html
index a755dde1..e76dfc06 100644
--- a/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html
+++ b/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html
@@ -9,16 +9,17 @@
   {% if object|can_edit:request.user %}<a href="{% url 'csv-download' connector_slug=object.slug %}">{{object.csv_file}}</a>
   — {% trans "added on" %} {{object.csv_file_datetime}}
   {% else %}{{object.csv_file}}{% endif %}
 </p>
 {% endblock %}
 
 {% block endpoints %}
 <ul class="endpoints">
+{% include "passerelle/manage/endpoint.html" with endpoint=object.get_endpoints_infos.0 %}
 {% url 'generic-endpoint' connector='csvdatasource' slug=object.slug endpoint='data' as csvdatasource_data_url %}
 <li class="get-method">
   <div class="description"><span class="description--label">{% trans "Returning all file lines:" %}</span>
   <a class="example-url" href="{{ csvdatasource_data_url }}">{{ csvdatasource_data_url }}</a></div>
 </li>
 <li class="get-method">
   <div class="description"><span class="description--label">{% trans "Returning lines containing 'abc' in 'text' column (case insensitive):" %}</span>
   <a class="example-url" href="{{ csvdatasource_data_url }}?q=abc">{{ csvdatasource_data_url }}?q=abc</a></div>
diff --git a/tests/test_csv_datasource.py b/tests/test_csv_datasource.py
index 580644d7..a5bfd2d4 100644
--- a/tests/test_csv_datasource.py
+++ b/tests/test_csv_datasource.py
@@ -24,26 +24,27 @@ from posix import stat_result
 from stat import ST_MTIME
 
 import mock
 import pytest
 import six
 import webtest
 from django.contrib.contenttypes.models import ContentType
 from django.core.files import File
+from django.core.files.storage import default_storage
 from django.core.management import call_command
 from django.test import Client, override_settings
 from django.urls import reverse
 from django.utils.encoding import force_bytes, force_str, force_text
 from django.utils.six import StringIO
 from django.utils.six.moves.urllib.parse import urlencode
 from django.utils.timezone import now
 from test_manager import login
 
-from passerelle.apps.csvdatasource.models import CsvDataSource, Query, TableRow
+from passerelle.apps.csvdatasource.models import CsvDataSource, Query, TableRow, upload_to
 from passerelle.base.models import AccessRight, ApiUser
 from passerelle.compat import json_loads
 
 data = """121;69981;DELANOUE;Eliot;H
 525;6;DANIEL WILLIAMS;Shanone;F
 253;67742;MARTIN;Sandra;F
 511;38142;MARCHETTI;Lucie;F
 235;22;MARTIN;Sandra;F
@@ -862,16 +863,73 @@ def test_change_csv_command(setup):
     csv.refresh_from_db()
     assert list(csv.get_rows()) == []
 
     call_command('change-csv', 'test', os.path.join(TEST_BASE_DIR, 'data.ods'), sheet_name='Feuille2')
     csv.refresh_from_db()
     assert list(csv.get_rows()) != []
 
 
+def test_update(admin_user, app, setup):
+    csv, url = setup(data=StringIO(data), filename='api-uploaded-file.csv')
+    upload_dir = default_storage.path(upload_to(csv, ''))
+    url = reverse(
+        'generic-endpoint',
+        kwargs={
+            'connector': 'csvdatasource',
+            'slug': csv.slug,
+            'endpoint': 'update',
+        },
+    )
+
+    assert CsvDataSource.objects.get().get_rows()[0]['fam'] == '121'
+    assert [files for root, dirs, files in os.walk(upload_dir)][0] == ['api-uploaded-file.csv']
+
+    # curl -H "Content-Type: text/csv" -T data.csv
+    headers = {'CONTENT-TYPE': 'text/csv'}
+    body = '212' + data[3:]
+
+    # restricted access
+    resp = app.put(url, params=body, headers=headers, status=403)
+    assert resp.json['err']
+    assert 'PermissionDenied' in resp.json['err_class']
+    assert resp.json['err_class'] == "django.core.exceptions.PermissionDenied"
+
+    # add can_update_file access
+    api = ApiUser.objects.get()
+    obj_type = ContentType.objects.get_for_model(csv)
+    AccessRight.objects.create(
+        codename='can_update_file', apiuser=api, resource_type=obj_type, resource_pk=csv.pk
+    )
+
+    resp = app.put(url, params=body, headers=headers)
+    assert not resp.json['err']
+    assert CsvDataSource.objects.get().get_rows()[0]['fam'] == '212'
+    assert len([files for root, dirs, files in os.walk(upload_dir)][0]) == 2
+
+    resp = app.put(url, params=body, headers={}, status=400)
+    assert resp.json['err']
+    assert "can't guess filename extension" in resp.json['err_desc']
+
+    resp = app.put(url, params='\n', headers=headers, status=400)
+    assert resp.json['err']
+    assert 'Could not detect CSV dialect' in resp.json['err_desc']
+
+    headers = {'CONTENT-TYPE': 'application/vnd.oasis.opendocument.spreadsheet'}
+    resp = app.put(url, params=body, headers=headers, status=400)
+    assert resp.json['err']
+    assert 'Invalid CSV file: File is not a zip file' in resp.json['err_desc']
+
+    csv.sheet_name = ''
+    csv.save()
+    resp = app.put(url, params=body, headers=headers, status=400)
+    assert resp.json['err']
+    assert 'You must specify a sheet name' in resp.json['err_desc']
+
+
 @pytest.mark.parametrize(
     'payload,expected',
     [
         ({}, 20),
         ({'limit': 10}, 10),
         ({'limit': 10, 'offset': 0}, 10),
         ({'limit': 10, 'offset': 15}, 5),
         ({'limit': 10, 'offset': 42}, 0),
diff --git a/tests/test_manager.py b/tests/test_manager.py
index 3f0c7bc4..5524465c 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -627,23 +627,23 @@ def test_manager_add_open_access_warning(app, admin_user):
     private = ApiUser.objects.create(username='private', fullname='private', keytype='', key='xxx')
     public = ApiUser.objects.create(username='public', fullname='private', keytype='', key='')
     assert AccessRight.objects.count() == 0
 
     # adding private api user works
     app = login(app)
     resp = app.get(csv.get_absolute_url())
     assert 'Access is limited' in resp.html.find('div', {'id': 'security'}).div.text.strip()
-    resp = resp.click('Add')
+    resp = resp.click('Add', href='/can_access/')
     resp.form['apiuser'] = private.pk
     resp = resp.form.submit().follow()
     assert AccessRight.objects.count() == 1
 
     # adding public user displays a warning
-    resp = resp.click('Add')
+    resp = resp.click('Add', href='/can_access/')
     resp.form['apiuser'] = public.pk
     resp = resp.form.submit()
     assert AccessRight.objects.count() == 1
     assert 'user has no security' in resp.text
 
     resp = resp.form.submit()
     assert AccessRight.objects.count() == 1
     assert 'user has no security' in resp.text
-- 
2.30.2

