From 00c8254c63eec7e97f4b813e646a59d2c231fb77 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Mon, 7 Oct 2019 18:19:22 +0200
Subject: [PATCH] field: display attachment utf-8 filenames (#34783)

---
 tests/test_admin_pages.py                     |  4 +-
 tests/test_fields.py                          |  5 +--
 tests/test_form_pages.py                      | 42 +++++++++++++------
 tests/test_formdata.py                        |  3 +-
 tests/test_formdef_import.py                  |  2 +-
 tests/test_sessions.py                        | 17 ++++++++
 tests/test_workflow_import.py                 |  2 +-
 tests/test_workflows.py                       |  8 ++--
 wcs/compat.py                                 |  3 +-
 wcs/formdata.py                               |  2 +-
 wcs/formdef.py                                |  2 +-
 wcs/qommon/form.py                            |  2 +-
 wcs/qommon/http_request.py                    | 25 +++++++++++
 .../templates/qommon/forms/widgets/file.html  |  2 +-
 wcs/root.py                                   |  2 +-
 15 files changed, 88 insertions(+), 33 deletions(-)

diff --git a/tests/test_admin_pages.py b/tests/test_admin_pages.py
index 6cbcfbed..ee0dc5fa 100644
--- a/tests/test_admin_pages.py
+++ b/tests/test_admin_pages.py
@@ -24,12 +24,12 @@ from webtest import Upload
 import mock
 
 from quixote import cleanup, get_publisher
-from quixote.http_request import Upload as QuixoteUpload
 
 from wcs.qommon import errors, sessions
 from wcs.qommon.form import UploadedFile
 from wcs.qommon.ident.password_accounts import PasswordAccount
 from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.http_request import Upload as QommonUpload
 from wcs.qommon.template import get_current_theme
 from wcs.qommon.bounces import Bounce
 from wcs.admin.settings import UserFieldsFormDef
@@ -4154,7 +4154,7 @@ def test_settings_export_import(pub, studio):
     st1 = wf.add_status('Status1', 'st1')
     export_to = ExportToModel()
     export_to.label = 'test'
-    upload = QuixoteUpload('/foo/bar', content_type='application/vnd.oasis.opendocument.text')
+    upload = QommonUpload('/foo/bar', content_type='application/vnd.oasis.opendocument.text')
     file_content = '''PK\x03\x04\x14\x00\x00\x08\x00\x00\'l\x8eG^\xc62\x0c\'\x00'''
     upload.fp = StringIO.StringIO()
     upload.fp.write(file_content)
diff --git a/tests/test_fields.py b/tests/test_fields.py
index e33bd847..ccb69b1d 100644
--- a/tests/test_fields.py
+++ b/tests/test_fields.py
@@ -7,8 +7,7 @@ from bs4 import BeautifulSoup
 import pytest
 
 from quixote import cleanup
-from quixote.http_request import Upload
-from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.http_request import HTTPRequest, Upload
 from wcs.qommon import sessions
 from wcs import fields
 from wcs.qommon.form import Form
@@ -134,7 +133,7 @@ def test_password():
 
 def test_file():
     upload = Upload('/foo/bar', content_type='text/plain')
-    assert fields.FileField().get_csv_value(upload) == ['/foo/bar']
+    assert fields.FileField().get_csv_value(upload) == ['bar']
 
 def test_page():
     formdef = FormDef()
diff --git a/tests/test_form_pages.py b/tests/test_form_pages.py
index 1f9a122d..3e9b543f 100644
--- a/tests/test_form_pages.py
+++ b/tests/test_form_pages.py
@@ -19,10 +19,10 @@ try:
 except ImportError:
     Image = None
 
-from quixote.http_request import Upload as QuixoteUpload
 from wcs.qommon.emails import docutils
 from wcs.qommon.form import UploadedFile
 from wcs.qommon.ident.password_accounts import PasswordAccount
+from wcs.qommon.http_request import Upload as QommonUpload
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.workflows import (Workflow, EditableWorkflowStatusItem,
@@ -2680,6 +2680,22 @@ def test_form_file_field_without_fargo(pub):
     assert 'fargo.js' not in resp.body
     assert 'use-file-from-fargo' not in resp.body
 
+def test_form_file_field_upload(pub):
+    user = create_user(pub)
+    formdef = create_formdef()
+    formdef.fields = [fields.FileField(id='0', type='file')]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    upload = Upload(u'C:\\ding\\dong\\téêèst👀.txt', 'foobar', 'text/plain')
+
+    resp = get_app(pub).get('/test/')
+    resp.forms[0]['f0$file'] = upload
+    resp = resp.forms[0].submit('submit')
+    assert 'Check values then click submit.' in resp.body
+    assert 'dong' not in resp.text
+    assert u'téêèst👀.txt' in resp.text
+
 def test_form_file_field_submit(pub):
     formdef = create_formdef()
     formdef.fields = [fields.FileField(id='0', label='file')]
@@ -3149,7 +3165,7 @@ def test_formdata_generated_document_download(pub):
     export_to = ExportToModel()
     export_to.convert_to_pdf = False
     export_to.label = 'create doc'
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    upload = QommonUpload('/foo/test.rtf', content_type='application/rtf')
     upload.fp = StringIO.StringIO()
     upload.fp.write('HELLO WORLD')
     upload.fp.seek(0)
@@ -3197,7 +3213,7 @@ def test_formdata_generated_document_download(pub):
 
     # change export model to now be a RTF file, do the action again on the same form and
     # check that both the old .odt file and the new .rtf file are there and valid.
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    upload = QommonUpload('/foo/test.rtf', content_type='application/rtf')
     upload.fp = StringIO.StringIO()
     upload.fp.write('HELLO NEW WORLD')
     upload.fp.seek(0)
@@ -3213,7 +3229,7 @@ def test_formdata_generated_document_download(pub):
     assert resp.click('test.rtf', index=1).follow().body == 'HELLO NEW WORLD'
 
     # use substitution variables on rtf: only ezt format is accepted
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    upload = QommonUpload('/foo/test.rtf', content_type='application/rtf')
     upload.fp = StringIO.StringIO()
     upload.fp.write('HELLO {{DJANGO}} WORLD [form_name]')
     upload.fp.seek(0)
@@ -3241,7 +3257,7 @@ def test_formdata_generated_document_odt_download(pub, odt_template):
     export_to.label = 'create doc'
     template_filename = os.path.join(os.path.dirname(__file__), odt_template)
     template = open(template_filename).read()
-    upload = QuixoteUpload('/foo/' + odt_template, content_type='application/octet-stream')
+    upload = QommonUpload('/foo/' + odt_template, content_type='application/octet-stream')
     upload.fp = StringIO.StringIO()
     upload.fp.write(template)
     upload.fp.seek(0)
@@ -3303,7 +3319,7 @@ def test_formdata_generated_document_odt_download(pub, odt_template):
         assert_equal_zip(StringIO.StringIO(resp.body), f)
 
     # change file content, same name
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    upload = QommonUpload('/foo/test.rtf', content_type='application/rtf')
     upload.fp = StringIO.StringIO()
     upload.fp.write('HELLO NEW WORLD')
     upload.fp.seek(0)
@@ -3330,7 +3346,7 @@ def test_formdata_generated_document_odt_download_with_substitution_variable(pub
     export_to.varname = 'created_doc'
     template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
     template = open(template_filename).read()
-    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload = QommonUpload('/foo/template.odt', content_type='application/octet-stream')
     upload.fp = StringIO.StringIO()
     upload.fp.write(template)
     upload.fp.seek(0)
@@ -3380,7 +3396,7 @@ def test_formdata_generated_document_odt_download_with_substitution_variable(pub
         assert_equal_zip(StringIO.StringIO(resp.body), f)
 
     # change file content, same name
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    upload = QommonUpload('/foo/test.rtf', content_type='application/rtf')
     upload.fp = StringIO.StringIO()
     upload.fp.write('HELLO NEW WORLD')
     upload.fp.seek(0)
@@ -3448,7 +3464,7 @@ def test_formdata_generated_document_odt_to_pdf_download(pub):
     export_to.varname = 'created_doc'
     template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
     template = open(template_filename).read()
-    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload = QommonUpload('/foo/template.odt', content_type='application/octet-stream')
     upload.fp = StringIO.StringIO()
     upload.fp.write(template)
     upload.fp.seek(0)
@@ -3515,7 +3531,7 @@ def test_formdata_generated_document_odt_to_pdf_download_push_to_portfolio(pub,
     export_to.varname = 'created_doc'
     template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
     template = open(template_filename).read()
-    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload = QommonUpload('/foo/template.odt', content_type='application/octet-stream')
     upload.fp = StringIO.StringIO()
     upload.fp.write(template)
     upload.fp.seek(0)
@@ -3603,7 +3619,7 @@ def test_formdata_generated_document_non_interactive(pub):
     export_to.method = 'non-interactive'
     template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
     template = open(template_filename).read()
-    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload = QommonUpload('/foo/template.odt', content_type='application/octet-stream')
     upload.fp = StringIO.StringIO()
     upload.fp.write(template)
     upload.fp.seek(0)
@@ -3663,7 +3679,7 @@ def test_formdata_generated_document_to_backoffice_field(pub):
     export_to.method = 'non-interactive'
     template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
     template = open(template_filename).read()
-    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload = QommonUpload('/foo/template.odt', content_type='application/octet-stream')
     upload.fp = StringIO.StringIO()
     upload.fp.write(template)
     upload.fp.seek(0)
@@ -3729,7 +3745,7 @@ def test_formdata_generated_document_in_private_history(pub):
     st1.visibility = ['_receiver']
     export_to = ExportToModel()
     export_to.label = 'create doc'
-    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
+    upload = QommonUpload('/foo/test.rtf', content_type='application/rtf')
     upload.fp = StringIO.StringIO()
     upload.fp.write('HELLO WORLD')
     upload.fp.seek(0)
diff --git a/tests/test_formdata.py b/tests/test_formdata.py
index 2e9bedae..027004ed 100644
--- a/tests/test_formdata.py
+++ b/tests/test_formdata.py
@@ -6,10 +6,9 @@ import shutil
 import time
 
 from quixote import cleanup
-from quixote.http_request import Upload
 from wcs.qommon.template import Template
 from wcs.qommon.form import PicklableUpload
-from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.http_request import HTTPRequest, Upload
 from wcs import fields, formdef
 from wcs.categories import Category
 from wcs.conditions import Condition
diff --git a/tests/test_formdef_import.py b/tests/test_formdef_import.py
index 6d78ddf5..daad5d0f 100644
--- a/tests/test_formdef_import.py
+++ b/tests/test_formdef_import.py
@@ -195,8 +195,8 @@ def test_workflow_options_with_no_values():
     assert fd2.workflow_options == formdef.workflow_options
 
 def test_workflow_options_with_file():
-    from quixote.http_request import Upload
     from wcs.qommon.form import UploadedFile
+    from wcs.qommon.http_request import Upload
 
     upload = Upload('/foo/bar', content_type='application/vnd.oasis.opendocument.text')
     file_content = '''PK\x03\x04\x14\x00\x00\x08\x00\x00\'l\x8eG^\xc62\x0c\'\x00'''
diff --git a/tests/test_sessions.py b/tests/test_sessions.py
index 97d8b428..ec2e60e1 100644
--- a/tests/test_sessions.py
+++ b/tests/test_sessions.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import os
 import shutil
 import time
@@ -199,3 +201,18 @@ def test_session_substitution_variables_1st_page_condition(pub, user, app):
     assert pub.session_manager.session_class.count() == 1
     session_id = pub.session_manager.session_class.select()[0].id
     assert 'COM1' in resp.body
+
+def test_tmp_upload(pub, user, app):
+    user_agent = {'User-agent': 'Mozilla/5.0'}
+    files = [('field0', u'C:\\ding\\dong\\téêèst👀.txt', r'file content')]
+
+    # submit a file using the endpoint
+    resp = app.post('/tmp-upload', headers=user_agent, upload_files=files)
+    assert resp.json[0]['name'] == u'téêèst👀.txt'
+    token = resp.json[0]['token']
+
+    # retrieve the file from session
+    session = pub.session_manager.session_class.select()[0]
+    tempfile = session.get_tempfile_path(token)
+    with open(tempfile) as handler:
+        assert handler.read() == r'file content'
diff --git a/tests/test_workflow_import.py b/tests/test_workflow_import.py
index f5229b94..b706fb5c 100644
--- a/tests/test_workflow_import.py
+++ b/tests/test_workflow_import.py
@@ -256,8 +256,8 @@ def test_export_to_model_action(pub):
     wf.store()
     st1 = wf.add_status('Status1', 'st1')
 
-    from quixote.http_request import Upload
     from wcs.qommon.form import UploadedFile
+    from wcs.qommon.http_request import Upload
     from wcs.workflows import ExportToModel
 
     export_to = ExportToModel()
diff --git a/tests/test_workflows.py b/tests/test_workflows.py
index ccac4141..08e71d64 100644
--- a/tests/test_workflows.py
+++ b/tests/test_workflows.py
@@ -14,8 +14,8 @@ from django.utils.six import StringIO
 
 from quixote import cleanup, get_response
 from wcs.qommon.errors import ConnectionError
-from quixote.http_request import Upload as QuixoteUpload
 from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.http_request import Upload as QommonUpload
 from wcs.qommon.form import *
 
 from wcs.formdef import FormDef
@@ -2887,7 +2887,7 @@ def test_export_to_model_image(pub):
     item.method = 'non-interactive'
     template_filename = os.path.join(os.path.dirname(__file__), 'template-with-image.odt')
     template = open(template_filename).read()
-    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload = QommonUpload('/foo/template.odt', content_type='application/octet-stream')
     upload.fp = StringIO()
     upload.fp.write(template)
     upload.fp.seek(0)
@@ -2948,7 +2948,7 @@ def test_export_to_model_backoffice_field(pub):
     item.convert_to_pdf = False
     template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
     template = open(template_filename).read()
-    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload = QommonUpload('/foo/template.odt', content_type='application/octet-stream')
     upload.fp = StringIO()
     upload.fp.write(template)
     upload.fp.seek(0)
@@ -3005,7 +3005,7 @@ def test_export_to_model_django_template(pub):
     item.attach_to_history = True
     template_filename = os.path.join(os.path.dirname(__file__), 'template-django.odt')
     template = open(template_filename).read()
-    upload = QuixoteUpload('/foo/template-django.odt', content_type='application/octet-stream')
+    upload = QommonUpload('/foo/template-django.odt', content_type='application/octet-stream')
     upload.fp = StringIO()
     upload.fp.write(template)
     upload.fp.seek(0)
diff --git a/wcs/compat.py b/wcs/compat.py
index 2c724c79..81b2de81 100644
--- a/wcs/compat.py
+++ b/wcs/compat.py
@@ -23,7 +23,6 @@ from django.utils.six.moves import configparser as ConfigParser
 
 from quixote import get_publisher, get_request
 from quixote.errors import PublishError
-from quixote.http_request import Upload
 
 from django.http import HttpResponse
 from django.conf import settings
@@ -34,7 +33,7 @@ from django.views.generic.base import TemplateView
 from .qommon import template
 from .qommon.publisher import get_cfg, set_publisher_class
 from .publisher import WcsPublisher
-from .qommon.http_request import HTTPRequest
+from .qommon.http_request import HTTPRequest, Upload
 from .qommon.http_response import HTTPResponse
 
 
diff --git a/wcs/formdata.py b/wcs/formdata.py
index 4e9d55f9..9b2622c7 100644
--- a/wcs/formdata.py
+++ b/wcs/formdata.py
@@ -23,12 +23,12 @@ import sys
 import time
 
 from quixote import get_request, get_publisher, get_session
-from quixote.http_request import Upload
 
 from .qommon import _
 from .qommon.storage import StorableObject, Intersects, Contains
 from .qommon import misc
 from .qommon.evalutils import make_datetime
+from .qommon.http_request import Upload
 from .qommon.publisher import get_cfg
 from .qommon.substitution import Substitutions, invalidate_substitution_cache
 from .qommon.template import Template
diff --git a/wcs/formdef.py b/wcs/formdef.py
index 473cb7a4..f87c19e8 100644
--- a/wcs/formdef.py
+++ b/wcs/formdef.py
@@ -25,12 +25,12 @@ import xml.etree.ElementTree as ET
 import datetime
 
 from quixote import get_request, get_publisher
-from quixote.http_request import Upload
 
 from .qommon import _
 from .qommon.storage import StorableObject, fix_key
 from .qommon.cron import CronJob
 from .qommon.form import *
+from .qommon.http_request import Upload
 from .qommon.misc import simplify, get_as_datetime
 from .qommon import get_cfg
 from .qommon.substitution import Substitutions
diff --git a/wcs/qommon/form.py b/wcs/qommon/form.py
index de074526..57a1db80 100644
--- a/wcs/qommon/form.py
+++ b/wcs/qommon/form.py
@@ -57,7 +57,6 @@ import quixote
 import quixote.form.widget
 
 from quixote import get_publisher, get_request, get_response, get_session
-from quixote.http_request import Upload
 from quixote.form import *
 from quixote.html import htmltext, htmltag, htmlescape, TemplateIO
 from quixote.util import randbytes
@@ -75,6 +74,7 @@ from wcs.conditions import Condition, ValidationError
 from . import _, ngettext
 from . import misc
 from .humantime import humanduration2seconds, seconds2humanduration, timewords
+from .http_request import Upload
 from .misc import strftime, C_, HAS_PDFTOPPM
 from .publisher import get_cfg
 from .template_utils import render_block_to_string
diff --git a/wcs/qommon/http_request.py b/wcs/qommon/http_request.py
index ff1d9a4b..7b0cefa3 100644
--- a/wcs/qommon/http_request.py
+++ b/wcs/qommon/http_request.py
@@ -181,3 +181,28 @@ class HTTPRequest(quixote.http_request.HTTPRequest):
 
     def trace(self, msg):
         print('%.4f' % (time.time() - self.t0), msg)
+
+
+class Upload(quixote.http_request.Upload):
+    def __init__(self, orig_filename, content_type=None, charset=None):
+        if orig_filename:
+            bspos = orig_filename.rfind("\\")
+            cpos = orig_filename.rfind(":")
+            spos = orig_filename.rfind("/")
+            if bspos != -1:                 # eg. "\foo\bar" or "D:\ding\dong"
+                filename = orig_filename[bspos+1:]
+            elif cpos != -1:                # eg. "C:foo" or ":ding:dong:foo"
+                filename = orig_filename[cpos+1:]
+            elif spos != -1:                # eg. "foo/bar/baz" or "/tmp/blah"
+                filename = orig_filename[spos+1:]
+            else:
+                filename = orig_filename
+
+            self.orig_filename = filename
+            self.base_filename = quixote.http_request.make_safe_filename(filename)
+        else:
+            self.orig_filename = None
+            self.base_filename = None
+        self.content_type = content_type
+        self.charset = charset
+        self.fp = None
diff --git a/wcs/qommon/templates/qommon/forms/widgets/file.html b/wcs/qommon/templates/qommon/forms/widgets/file.html
index bb24204a..30b649cf 100644
--- a/wcs/qommon/templates/qommon/forms/widgets/file.html
+++ b/wcs/qommon/templates/qommon/forms/widgets/file.html
@@ -21,7 +21,7 @@
        data-error="{% trans "Error during upload." %}"></div>
 </div>
 <div class="fileinfo {% if widget.readonly and widget.has_tempfile_image %}thumbnail{% endif %}">
-  <span class="filename">{{ widget.tempfile.base_filename }}</span>
+  <span class="filename">{{ widget.tempfile.orig_filename }}</span>
 {% if not widget.readonly %}
   <a href="#" class="remove" title="{% trans 'Remove this file' %}">{% trans "remove" %}</a>
 {% elif widget.has_tempfile_image %}
diff --git a/wcs/root.py b/wcs/root.py
index c76b5748..34ab6290 100644
--- a/wcs/root.py
+++ b/wcs/root.py
@@ -296,7 +296,7 @@ class RootDirectory(Directory):
         for k, v in get_request().form.items():
             if hasattr(v, 'fp'):
                 tempfile = get_session().add_tempfile(v)
-                results.append({'name': tempfile.get('base_filename'),
+                results.append({'name': tempfile.get('orig_filename'),
                                 'type': tempfile.get('content_type'),
                                 'size': tempfile.get('size'),
                                 'token': tempfile.get('token')})
-- 
2.23.0

