0001-cards-add-import-data-from-CSV-39473.patch
tests/test_backoffice_pages.py | ||
---|---|---|
23 | 23 |
from django.utils.six.moves.urllib import parse as urllib |
24 | 24 |
from django.utils.six.moves.urllib import parse as urlparse |
25 | 25 | |
26 |
from quixote import cleanup, get_publisher |
|
26 |
from webtest import Upload |
|
27 | ||
28 |
from quixote import cleanup, get_publisher, get_response |
|
27 | 29 |
from wcs.qommon import ods |
28 | 30 |
from wcs.api_utils import sign_url |
29 | 31 |
from wcs.qommon import errors, sessions |
... | ... | |
5534 | 5536 |
resp.click('card plop') |
5535 | 5537 | |
5536 | 5538 | |
5539 |
def test_backoffice_cards_import_data_from_csv(pub, studio): |
|
5540 |
user = create_user(pub) |
|
5541 |
CardDef.wipe() |
|
5542 |
carddef = CardDef() |
|
5543 |
carddef.name = 'test' |
|
5544 |
carddef.fields = [ |
|
5545 |
fields.StringField(id='1', label='Test'), |
|
5546 |
fields.ItemField(id='2', label='List', |
|
5547 |
items=['item1', 'item2']), |
|
5548 |
fields.DateField(id='3', label='Date',), |
|
5549 |
fields.BoolField(id='4', label='Boolean'), |
|
5550 |
fields.FileField(id='5', label='File') |
|
5551 |
] |
|
5552 |
carddef.backoffice_submission_roles = user.roles |
|
5553 |
carddef.workflow_roles = {'_editor': user.roles[0]} |
|
5554 |
carddef.store() |
|
5555 |
carddef.data_class().wipe() |
|
5556 | ||
5557 |
app = login(get_app(pub)) |
|
5558 | ||
5559 |
resp = app.get(carddef.get_url()) |
|
5560 |
assert 'import-csv' in resp |
|
5561 |
resp = resp.click('Import data from a CSV file') |
|
5562 | ||
5563 |
assert 'Download sample file' in resp |
|
5564 |
sample_resp = resp.click('Download sample file') |
|
5565 |
today = datetime.date.today() |
|
5566 |
assert sample_resp.text == 'Test,List,Date,Boolean,File\r\nvalue,value,%s,Yes,\r\n' % today |
|
5567 | ||
5568 |
resp.forms[0]['file'] = Upload('test.csv', b'\0', 'text/csv') |
|
5569 |
resp = resp.forms[0].submit() |
|
5570 |
assert 'Invalid file format.' in resp |
|
5571 | ||
5572 |
resp.forms[0]['file'] = Upload('test.csv', b'', 'text/csv') |
|
5573 |
resp = resp.forms[0].submit() |
|
5574 |
assert 'Invalid CSV file.' in resp |
|
5575 | ||
5576 |
resp.forms[0]['file'] = Upload('test.csv', |
|
5577 |
b'Test,List,Date\ndata1,item1,invalid', |
|
5578 |
'text/csv') |
|
5579 |
resp = resp.forms[0].submit() |
|
5580 |
assert 'CSV file contains less columns than card fields.' in resp.text |
|
5581 | ||
5582 |
resp.forms[0]['file'] = Upload('test.csv', |
|
5583 |
b'Test,List,Date,Boolean,File\ndata1,item1,invalid,Yes,', |
|
5584 |
'text/csv') |
|
5585 |
resp = resp.forms[0].submit() |
|
5586 |
assert 'time data \'invalid\' does not match format \'%Y-%m-%d\'' in resp.text |
|
5587 | ||
5588 |
data = [b'Test,List,Date,Boolean,File'] |
|
5589 |
for i in range(1, 150): |
|
5590 |
data.append(b'data%d,item%d,2020-01-%02d,%d,filename-%d' % (i, i, i%31+1, i%2,i)) |
|
5591 | ||
5592 |
resp.forms[0]['file'] = Upload('test.csv',b'\n'.join(data), |
|
5593 |
'text/csv') |
|
5594 |
resp = resp.forms[0].submit().follow() |
|
5595 |
assert 'Importing data into ' in resp |
|
5596 | ||
5597 | ||
5537 | 5598 |
def test_backoffice_cards_wscall_failure_display(http_requests, pub, studio): |
5538 | 5599 |
LoggedError.wipe() |
5539 | 5600 |
user = create_user(pub) |
wcs/backoffice/data_management.py | ||
---|---|---|
14 | 14 |
# You should have received a copy of the GNU General Public License |
15 | 15 |
# along with this program; if not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 |
import csv |
|
18 |
import datetime |
|
19 | ||
17 | 20 |
from quixote import get_request, get_response, get_session, redirect |
18 | 21 |
from quixote.html import TemplateIO, htmltext, htmlescape |
19 | 22 | |
23 |
from django.utils.encoding import force_text |
|
24 |
from django.utils.six import StringIO |
|
25 | ||
26 | ||
20 | 27 |
from ..qommon import _ |
21 | 28 |
from ..qommon import errors |
22 | 29 |
from ..qommon import template |
30 |
from ..qommon.form import Form, FileWidget |
|
23 | 31 |
from ..qommon.backoffice.menu import html_top |
32 |
from ..qommon import template |
|
33 |
from ..qommon.afterjobs import AfterJob |
|
24 | 34 | |
25 | 35 |
from wcs.carddef import CardDef |
36 |
from wcs import fields |
|
37 | ||
26 | 38 |
from .management import ManagementDirectory, FormPage, FormFillPage, FormBackOfficeStatusPage |
27 | 39 | |
28 | 40 | |
... | ... | |
67 | 79 | |
68 | 80 | |
69 | 81 |
class CardPage(FormPage): |
70 |
_q_exports = ['', 'csv', 'xls', 'ods', 'json', 'export', 'map', 'geojson', 'add'] |
|
82 |
_q_exports = ['', 'csv', 'xls', 'ods', 'json', 'export', 'map', 'geojson', 'add', |
|
83 |
('import-csv', 'import_csv'), |
|
84 |
('data-sample-csv', 'data_sample_csv')] |
|
71 | 85 | |
72 | 86 |
def __init__(self, component): |
73 | 87 |
try: |
... | ... | |
99 | 113 |
def get_filter_from_query(self, default='waiting'): |
100 | 114 |
return 'all' |
101 | 115 | |
116 |
def data_sample_csv(self): |
|
117 |
carddef_fields = self.formdef.get_all_fields() |
|
118 |
output = StringIO() |
|
119 |
csv_output = csv.writer(output) |
|
120 |
csv_output.writerow([f.label for f in carddef_fields]) |
|
121 |
sample_line = [] |
|
122 |
for f in carddef_fields: |
|
123 |
if isinstance(f, fields.DateField): |
|
124 |
value = datetime.date.today() |
|
125 |
elif isinstance(f, fields.BoolField): |
|
126 |
value = _('Yes') |
|
127 |
elif isinstance(f, fields.EmailField): |
|
128 |
value = 'foo@example.com' |
|
129 |
elif isinstance(f, fields.FileField) or isinstance(f, fields.MapField): |
|
130 |
value = '' |
|
131 |
else: |
|
132 |
value = 'value' |
|
133 |
sample_line.append(value) |
|
134 |
csv_output.writerow(sample_line) |
|
135 |
response = get_response() |
|
136 |
response.set_content_type('text/plain') |
|
137 |
response.set_header('content-disposition', 'attachment; filename=%s-sample.csv' % self.formdef.url_name) |
|
138 |
return output.getvalue() |
|
139 | ||
140 |
def import_csv(self): |
|
141 |
context = {} |
|
142 |
try: |
|
143 |
job = AfterJob.get(get_request().form.get('job')) |
|
144 |
get_response().add_javascript(['jquery.js', 'afterjob.js']) |
|
145 |
context['job'] = job |
|
146 |
except KeyError: |
|
147 |
form = Form(enctype='multipart/form-data', use_tokens=False) |
|
148 |
form.add(FileWidget, 'file', title=_('File'), required=False) |
|
149 |
form.add_submit('submit', _('Submit')) |
|
150 |
form.add_submit('cancel', _('Cancel')) |
|
151 |
if form.get_widget('cancel').parse(): |
|
152 |
return redirect('.') |
|
153 | ||
154 |
if form.is_submitted() and not form.has_errors(): |
|
155 |
try: |
|
156 |
return self.import_csv_submit(form) |
|
157 |
except ValueError as e: |
|
158 |
form.set_error('file', e) |
|
159 |
context['form'] = form |
|
160 | ||
161 |
get_response().breadcrumb.append(('import_csv', _('Import CSV'))) |
|
162 |
html_top('data_management', _('Import CSV')) |
|
163 |
return template.QommonTemplateResponse( |
|
164 |
templates=['wcs/backoffice/card-data-import-form.html'], |
|
165 |
context=context) |
|
166 | ||
167 |
def import_csv_submit(self, form): |
|
168 |
if form.get_widget('file').parse(): |
|
169 |
content = form.get_widget('file').parse().fp.read() |
|
170 |
if b'\0' in content: |
|
171 |
raise ValueError(_('Invalid file format.')) |
|
172 |
else: |
|
173 |
raise ValueError(_('You have to enter a file.')) |
|
174 | ||
175 |
for charset in ('utf-8', 'iso-8859-15'): |
|
176 |
try: |
|
177 |
content = content.decode(charset) |
|
178 |
break |
|
179 |
except UnicodeDecodeError: |
|
180 |
continue |
|
181 | ||
182 |
try: |
|
183 |
dialect = csv.Sniffer().sniff(content) |
|
184 |
except csv.Error: |
|
185 |
dialect = None |
|
186 | ||
187 |
carddef_fields = self.formdef.get_all_fields() |
|
188 |
reader = csv.reader(content.splitlines(), dialect=dialect) |
|
189 |
try: |
|
190 |
caption = next(reader) |
|
191 |
except StopIteration: |
|
192 |
raise ValueError(_('Invalid CSV file.')) |
|
193 | ||
194 |
if len(caption) < len(carddef_fields): |
|
195 |
raise ValueError(_('CSV file contains less columns than card fields.')) |
|
196 | ||
197 |
data_lines = [] |
|
198 |
for csv_line in reader: |
|
199 |
data_line = {} |
|
200 |
for i, field in enumerate(self.formdef.get_all_fields()): |
|
201 |
value = csv_line[i] |
|
202 |
# ignore empty values |
|
203 |
if not value: |
|
204 |
continue |
|
205 |
# ignore data for file and map fields |
|
206 |
if isinstance(field, fields.FileField) or isinstance(field, fields.MapField): |
|
207 |
continue |
|
208 |
field_id = str(field.id) |
|
209 |
if field.convert_value_from_str: |
|
210 |
value = field.convert_value_from_str(value) |
|
211 |
data_line[field_id] = value |
|
212 |
if field.store_display_value: |
|
213 |
display_value = field.store_display_value(data_line, field_id) |
|
214 |
data_line['%s_display' % field_id] = display_value |
|
215 |
if value and field.store_structured_value: |
|
216 |
structured_value = field.store_structured_value(data_line, field_id) |
|
217 |
if structured_value: |
|
218 |
if isinstance(structured_value, dict) and structured_value.get('id'): |
|
219 |
formdata.data[field_id] = str(structured_value.get('id')) |
|
220 |
data_line['%s_structured' % field_id] = structured_value |
|
221 |
data_lines.append(data_line) |
|
222 | ||
223 |
class ImportAction: |
|
224 |
def __init__(self, data_class, lines): |
|
225 |
self.data_class = data_class |
|
226 |
self.lines = lines |
|
227 | ||
228 |
def execute(self, job): |
|
229 |
for item in self.lines: |
|
230 |
data_instance = self.data_class() |
|
231 |
data_instance.data = item |
|
232 |
data_instance.just_created() |
|
233 |
data_instance.store() |
|
234 |
data_instance.perform_workflow() |
|
235 | ||
236 |
action = ImportAction(self.formdef.data_class(), data_lines) |
|
237 |
job = get_response().add_after_job(str(N_('Importing data into \'%s\'') % self.formdef.name), |
|
238 |
action.execute) |
|
239 |
job.store() |
|
240 |
return redirect('import-csv?job=%s' % job.id) |
|
241 | ||
102 | 242 |
def _q_lookup(self, component): |
103 | 243 |
try: |
104 | 244 |
filled = self.formdef.data_class().get(component) |
wcs/backoffice/management.py | ||
---|---|---|
1039 | 1039 |
qs, _('Plot on a Map')) |
1040 | 1040 |
if 'stats' in self._q_exports: |
1041 | 1041 |
r += htmltext(' <li class="stats"><a href="stats">%s</a></li>') % _('Statistics') |
1042 |
if ('import-csv', 'import_csv') in self._q_exports: |
|
1043 |
r += htmltext('<li><a rel="popup" href="import-csv">%s</a></li>') % _('Import data from a CSV file') |
|
1042 | 1044 |
r += htmltext('</ul>') |
1043 | 1045 |
return r.getvalue() |
1044 | 1046 |
wcs/templates/wcs/backoffice/card-data-import-form.html | ||
---|---|---|
1 |
{% extends "wcs/backoffice/base.html" %} |
|
2 |
{% load i18n %} |
|
3 | ||
4 |
{% block appbar %} |
|
5 |
<div id="appbar" class="highlight"> |
|
6 |
<h2>{% trans "Import CSV" %}</h2> |
|
7 |
</div> |
|
8 |
{% endblock %} |
|
9 | ||
10 |
{% block content %} |
|
11 |
{% if job %} |
|
12 |
<dl class="job-status"> |
|
13 |
<dt>{{ job.label }}</dt> |
|
14 |
<dd><span class="afterjob" id="{{ job.id }}">{{ job.status }}</span></dd> |
|
15 |
</dl> |
|
16 |
<div class="done"> |
|
17 |
<a data-redirect-auto="true" href=".">{% trans "Back to Listing" %}</a> |
|
18 |
</div> |
|
19 |
{% else %} |
|
20 |
<p>{% trans "You can add data to this card by uploading a file." %}</p> |
|
21 |
<p><a href="data-sample-csv">{% trans 'Download sample file' %}</a> {% trans "for this card" %}</p> |
|
22 |
{{ form.render|safe }} |
|
23 |
{% endif %} |
|
24 |
{% endblock %} |
|
0 |
- |