0001-cards-add-import-data-from-CSV-39473.patch
tests/test_backoffice_pages.py | ||
---|---|---|
23 | 23 |
from django.utils.six.moves.urllib import parse as urllib |
24 | 24 |
from django.utils.six.moves.urllib import parse as urlparse |
25 | 25 | |
26 |
from webtest import Upload |
|
27 | ||
26 | 28 |
from quixote import cleanup, get_publisher |
27 | 29 |
from wcs.qommon import ods |
28 | 30 |
from wcs.api_utils import sign_url |
... | ... | |
5534 | 5536 |
resp.click('card plop') |
5535 | 5537 | |
5536 | 5538 | |
5539 |
def test_backoffice_cards_import_data_from_csv(pub, studio): |
|
5540 |
user = create_user(pub) |
|
5541 |
CardDef.wipe() |
|
5542 |
carddef = CardDef() |
|
5543 |
carddef.name = 'test' |
|
5544 |
carddef.fields = [ |
|
5545 |
fields.StringField(id='1', label='Test', varname='string'), |
|
5546 |
fields.ItemField(id='2', label='List', varname='list', |
|
5547 |
items=['item1', 'item2']), |
|
5548 |
fields.DateField(id='3', label='Date', varname='date'), |
|
5549 |
fields.BoolField(id='4', label='Boolean', varname='boolean') |
|
5550 |
] |
|
5551 |
carddef.backoffice_submission_roles = user.roles |
|
5552 |
carddef.workflow_roles = {'_editor': user.roles[0]} |
|
5553 |
carddef.store() |
|
5554 |
carddef.data_class().wipe() |
|
5555 | ||
5556 |
app = login(get_app(pub)) |
|
5557 | ||
5558 |
resp = app.get(carddef.get_url()) |
|
5559 |
assert 'import-csv' in resp |
|
5560 |
resp = resp.click('Import data from a CSV file') |
|
5561 | ||
5562 |
assert 'Download sample file' in resp |
|
5563 |
sample_resp = resp.click('Download sample file') |
|
5564 |
today = datetime.date.today() |
|
5565 |
assert sample_resp.text == 'Test,List,Date,Boolean\r\nstring-value,list-value,%s,Yes\r\n' % today |
|
5566 | ||
5567 |
resp.forms[0]['file'] = Upload('test.csv', b'\0', 'text/csv') |
|
5568 |
resp = resp.forms[0].submit() |
|
5569 |
assert 'Invalid file format.' in resp |
|
5570 | ||
5571 |
resp.forms[0]['file'] = Upload('test.csv', b'', 'text/csv') |
|
5572 |
resp = resp.forms[0].submit() |
|
5573 |
assert 'Invalid CSV file.' in resp |
|
5574 | ||
5575 |
resp.forms[0]['file'] = Upload('test.csv', |
|
5576 |
b'Test,List,Date\ndata1,item1,invalid', |
|
5577 |
'text/csv') |
|
5578 |
resp = resp.forms[0].submit() |
|
5579 |
assert 'CSV file contains less columns than card fields.' in resp.text |
|
5580 | ||
5581 |
resp.forms[0]['file'] = Upload('test.csv', |
|
5582 |
b'Test,List,Date,Boolean\ndata1,item1,invalid,Yes', |
|
5583 |
'text/csv') |
|
5584 |
resp = resp.forms[0].submit() |
|
5585 |
assert 'time data \'invalid\' does not match format \'%Y-%m-%d\'' in resp.text |
|
5586 | ||
5587 |
resp.forms[0]['file'] = Upload('test.csv', |
|
5588 |
b'Test,List,Date,Boolean\ndata1,item1,2020-01-01,Yes', |
|
5589 |
'text/csv') |
|
5590 |
resp = resp.forms[0].submit().follow() |
|
5591 |
assert 'Data imported successfully.' in resp |
|
5592 | ||
5593 | ||
5537 | 5594 |
def test_backoffice_cards_wscall_failure_display(http_requests, pub, studio): |
5538 | 5595 |
LoggedError.wipe() |
5539 | 5596 |
user = create_user(pub) |
wcs/backoffice/data_management.py | ||
---|---|---|
14 | 14 |
# You should have received a copy of the GNU General Public License |
15 | 15 |
# along with this program; if not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 |
import csv |
|
18 |
import datetime |
|
19 | ||
17 | 20 |
from quixote import get_request, get_response, get_session, redirect |
18 | 21 |
from quixote.html import TemplateIO, htmltext, htmlescape |
19 | 22 | |
23 |
from django.utils.encoding import force_text |
|
24 |
from django.utils.six import StringIO |
|
25 | ||
26 | ||
20 | 27 |
from ..qommon import _ |
21 | 28 |
from ..qommon import errors |
22 | 29 |
from ..qommon import template |
30 |
from ..qommon.form import Form, FileWidget |
|
23 | 31 |
from ..qommon.backoffice.menu import html_top |
32 |
from ..qommon import template |
|
24 | 33 | |
25 | 34 |
from wcs.carddef import CardDef |
35 |
from wcs import fields |
|
36 | ||
26 | 37 |
from .management import ManagementDirectory, FormPage, FormFillPage, FormBackOfficeStatusPage |
27 | 38 | |
28 | 39 | |
... | ... | |
67 | 78 | |
68 | 79 | |
69 | 80 |
class CardPage(FormPage): |
70 |
_q_exports = ['', 'csv', 'xls', 'ods', 'json', 'export', 'map', 'geojson', 'add'] |
|
81 |
_q_exports = ['', 'csv', 'xls', 'ods', 'json', 'export', 'map', 'geojson', 'add', |
|
82 |
('import-csv', 'import_csv'), |
|
83 |
('data-sample-csv', 'data_sample_csv')] |
|
71 | 84 | |
72 | 85 |
def __init__(self, component): |
73 | 86 |
try: |
... | ... | |
99 | 112 |
def get_filter_from_query(self, default='waiting'): |
100 | 113 |
return 'all' |
101 | 114 | |
115 |
def data_sample_csv(self): |
|
116 |
carddef_fields = self.formdef.get_all_fields() |
|
117 |
output = StringIO() |
|
118 |
csv_output = csv.writer(output) |
|
119 |
csv_output.writerow([f.label for f in carddef_fields]) |
|
120 |
sample_line = [] |
|
121 |
for f in carddef_fields: |
|
122 |
if isinstance(f, fields.DateField): |
|
123 |
value = datetime.date.today() |
|
124 |
elif isinstance(f, fields.BoolField): |
|
125 |
value = _('Yes') |
|
126 |
elif isinstance(f, fields.EmailField): |
|
127 |
value = 'foo@example.com' |
|
128 |
else: |
|
129 |
value = '%s-value' % f.varname |
|
130 |
sample_line.append(value) |
|
131 |
csv_output.writerow(sample_line) |
|
132 |
response = get_response() |
|
133 |
response.set_content_type('text/plain') |
|
134 |
response.set_header('content-disposition', 'attachment; filename=%s-sample.csv' % self.formdef.url_name) |
|
135 |
return output.getvalue() |
|
136 | ||
137 |
def import_csv(self): |
|
138 |
form = Form(enctype='multipart/form-data', use_tokens=False) |
|
139 |
form.add(FileWidget, 'file', title=_('File'), required=False) |
|
140 |
form.add_submit('submit', _('Submit')) |
|
141 |
form.add_submit('cancel', _('Cancel')) |
|
142 |
if form.get_widget('cancel').parse(): |
|
143 |
return redirect('.') |
|
144 | ||
145 |
if form.is_submitted() and not form.has_errors(): |
|
146 |
try: |
|
147 |
return self.import_csv_submit(form) |
|
148 |
except ValueError as e: |
|
149 |
form.set_error('file', e) |
|
150 | ||
151 |
get_response().breadcrumb.append(('import_csv', _('Import CSV'))) |
|
152 |
html_top('data_management', _('Import CSV')) |
|
153 |
return template.QommonTemplateResponse( |
|
154 |
templates=['wcs/backoffice/card-data-import-form.html'], |
|
155 |
context={'form': form}) |
|
156 | ||
157 |
def import_csv_submit(self, form): |
|
158 |
if form.get_widget('file').parse(): |
|
159 |
content = form.get_widget('file').parse().fp.read() |
|
160 |
if b'\0' in content: |
|
161 |
raise ValueError(_('Invalid file format.')) |
|
162 |
else: |
|
163 |
raise ValueError(_('You have to enter a file.')) |
|
164 | ||
165 |
for charset in ('utf-8', 'iso-8859-15'): |
|
166 |
try: |
|
167 |
content = content.decode(charset) |
|
168 |
break |
|
169 |
except UnicodeDecodeError: |
|
170 |
continue |
|
171 | ||
172 |
try: |
|
173 |
dialect = csv.Sniffer().sniff(content) |
|
174 |
except csv.Error: |
|
175 |
dialect = None |
|
176 | ||
177 |
carddef_fields = self.formdef.get_all_fields() |
|
178 |
reader = csv.reader(content.splitlines(), dialect=dialect) |
|
179 |
try: |
|
180 |
caption = next(reader) |
|
181 |
except StopIteration: |
|
182 |
raise ValueError(_('Invalid CSV file.')) |
|
183 | ||
184 |
if len(caption) < len(carddef_fields): |
|
185 |
raise ValueError(_('CSV file contains less columns than card fields.')) |
|
186 | ||
187 |
data_lines = [] |
|
188 |
for csv_line in reader: |
|
189 |
data_line = {} |
|
190 |
for i, field in enumerate(carddef_fields): |
|
191 |
value = csv_line[i] |
|
192 |
if not value: |
|
193 |
continue |
|
194 |
field_id = str(field.id) |
|
195 |
if field.convert_value_from_str: |
|
196 |
value = field.convert_value_from_str(value) |
|
197 |
data_line[field_id] = value |
|
198 |
if field.store_display_value: |
|
199 |
display_value = field.store_display_value(data_line, field_id) |
|
200 |
data_line['%s_display' % field_id] = display_value |
|
201 |
if value and field.store_structured_value: |
|
202 |
structured_value = field.store_structured_value(data_line, field_id) |
|
203 |
if structured_value: |
|
204 |
if isinstance(structured_value, dict) and structured_value.get('id'): |
|
205 |
formdata.data[field_id] = str(structured_value.get('id')) |
|
206 |
data_line['%s_structured' % field_id] = structured_value |
|
207 |
data_lines.append(data_line) |
|
208 | ||
209 |
data_class = self.formdef.data_class() |
|
210 |
for item in data_lines: |
|
211 |
data_instance = data_class() |
|
212 |
data_instance.data = item |
|
213 |
data_instance.just_created() |
|
214 |
data_instance.store() |
|
215 | ||
216 |
get_session().message = ('info', N_('Data imported successfully.')) |
|
217 |
return redirect('.') |
|
218 | ||
219 | ||
102 | 220 |
def _q_lookup(self, component): |
103 | 221 |
try: |
104 | 222 |
filled = self.formdef.data_class().get(component) |
wcs/backoffice/management.py | ||
---|---|---|
1039 | 1039 |
qs, _('Plot on a Map')) |
1040 | 1040 |
if 'stats' in self._q_exports: |
1041 | 1041 |
r += htmltext(' <li class="stats"><a href="stats">%s</a></li>') % _('Statistics') |
1042 |
if get_publisher().has_site_option('studio') and ('import-csv', 'import_csv') in self._q_exports: |
|
1043 |
r += htmltext('<li><a rel="popup" href="import-csv">%s</a></li>') % _('Import data from a CSV file') |
|
1042 | 1044 |
r += htmltext('</ul>') |
1043 | 1045 |
return r.getvalue() |
1044 | 1046 |
wcs/templates/wcs/backoffice/card-data-import-form.html | ||
---|---|---|
1 |
{% extends "wcs/backoffice/base.html" %} |
|
2 |
{% load i18n %} |
|
3 | ||
4 |
{% block appbar %} |
|
5 |
<div id="appbar" class="highlight"> |
|
6 |
<h2>{% trans "Import CSV" %}</h2> |
|
7 |
</div> |
|
8 |
{% endblock %} |
|
9 | ||
10 |
{% block content %} |
|
11 |
<p>{% trans "You can add data to this card by uploading a file containing var names in columns captions." %}</p> |
|
12 |
<p><a href="data-sample-csv">{% trans 'Download sample file' %}</a> {% trans "for this card" %}</p> |
|
13 |
{{ form.render|safe }} |
|
14 |
{% endblock %} |
|
0 |
- |