Project

General

Profile

0004-csv_datasource-PEP8ness-code-style-30458.patch

Benjamin Dauvergne, 08 Feb 2019 08:43 AM

Download (21.4 KB)

View differences:

Subject: [PATCH 04/13] csv_datasource: PEP8ness, code style (#30458)

 passerelle/apps/csvdatasource/models.py |  64 ++++++++-----
 tests/test_csv_datasource.py            | 118 ++++++++++++++++++------
 2 files changed, 129 insertions(+), 53 deletions(-)
passerelle/apps/csvdatasource/models.py
17 17
import os
18 18
import re
19 19
import csv
20
import itertools
21 20
import unicodedata
22 21
from collections import OrderedDict
22

  
23
import six
24

  
23 25
from pyexcel_ods import get_data as get_data_ods
24 26
from pyexcel_xls import get_data as get_data_xls
25 27

  
......
29 31
from django.utils.timezone import datetime, make_aware
30 32
from django.db import models, transaction
31 33
from django.core.exceptions import ValidationError
32
from django.shortcuts import get_object_or_404
33 34
from django.utils.translation import ugettext_lazy as _
34 35

  
35 36
from passerelle.base.models import BaseResource
......
52 53
        code_cache[expr] = compile(expr, '<inline>', 'eval')
53 54
    return code_cache[expr]
54 55

  
56

  
55 57
def normalize(value):
56 58
    return unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
57 59

  
......
61 63
    slug = models.SlugField(_('Name (slug)'))
62 64
    label = models.CharField(_('Label'), max_length=100)
63 65
    description = models.TextField(_('Description'), blank=True)
64
    filters = models.TextField(_('Filters'), blank=True,
65
            help_text=_('List of filter clauses (Python expression)'))
66
    order = models.TextField(_('Order'), blank=True,
67
            help_text=_('Ordering columns'))
68
    distinct = models.TextField(_('Distinct'), blank=True,
69
            help_text=_('Distinct columns'))
70
    projections = models.TextField(_('Projections'), blank=True,
71
            help_text=_('List of projections (name:expression)'))
72
    structure = models.CharField(_('Structure'),
73
            max_length=20,
74
            choices=[
75
                ('array', _('Array')),
76
                ('dict', _('Dictionary')),
77
                ('keyed-distinct', _('Keyed Dictionary')),
78
                ('tuples', _('Tuples')),
79
                ('onerow', _('Single Row')),
80
                ('one', _('Single Value'))],
81
            default='dict',
82
            help_text=_('Data structure used for the response'))
66
    filters = models.TextField(
67
        _('Filters'),
68
        blank=True,
69
        help_text=_('List of filter clauses (Python expression)'))
70
    order = models.TextField(
71
        _('Order'),
72
        blank=True,
73
        help_text=_('Ordering columns'))
74
    distinct = models.TextField(
75
        _('Distinct'),
76
        blank=True,
77
        help_text=_('Distinct columns'))
78
    projections = models.TextField(
79
        _('Projections'),
80
        blank=True,
81
        help_text=_('List of projections (name:expression)'))
82
    structure = models.CharField(
83
        _('Structure'),
84
        max_length=20,
85
        choices=[
86
            ('array', _('Array')),
87
            ('dict', _('Dictionary')),
88
            ('keyed-distinct', _('Keyed Dictionary')),
89
            ('tuples', _('Tuples')),
90
            ('onerow', _('Single Row')),
91
            ('one', _('Single Value'))],
92
        default='dict',
93
        help_text=_('Data structure used for the response'))
83 94

  
84 95
    class Meta:
85 96
        ordering = ['slug']
......
107 118

  
108 119

  
109 120
class CsvDataSource(BaseResource):
110
    csv_file = models.FileField(_('Spreadsheet file'), upload_to='csv',
121
    csv_file = models.FileField(
122
        _('Spreadsheet file'),
123
        upload_to='csv',
111 124
        help_text=_('Supported file formats: csv, ods, xls, xlsx'))
112 125
    columns_keynames = models.CharField(
113 126
        max_length=256,
......
149 162
        return result
150 163

  
151 164
    def cache_data(self):
165
        # FIXME: why are those dead variables computed ?
152 166
        titles = [t.strip() for t in self.columns_keynames.split(',')]
153 167
        indexes = [titles.index(t) for t in titles if t]
154 168
        caption = [titles[i] for i in indexes]
......
177 191

  
178 192
        options = {}
179 193
        for k, v in self._dialect_options.items():
180
            if isinstance(v, unicode):
194
            if isinstance(v, six.text_type):
181 195
                v = v.encode('ascii')
182 196
            options[k.encode('ascii')] = v
183 197

  
......
256 270
        return [smart_text(t.strip()) for t in self.columns_keynames.split(',')]
257 271

  
258 272
    @endpoint(perm='can_access', methods=['get'],
259
              name='query', pattern='^(?P<query_name>[\w-]+)/$')
273
              name='query', pattern=r'^(?P<query_name>[\w-]+)/$')
260 274
    def select(self, request, query_name, **kwargs):
261 275
        try:
262 276
            query = Query.objects.get(resource=self.id, slug=query_name)
......
315 329
        if order:
316 330
            generator = stream_expressions(order, data, kind='order')
317 331
            new_data = [(tuple(new_row), row) for new_row, row in generator]
318
            new_data.sort(key=lambda (k, row): k)
332
            new_data.sort(key=lambda x: x[0])
319 333
            data = [row for key, row in new_data]
320 334

  
321 335
        distinct = query.get_list('distinct')
tests/test_csv_datasource.py
1 1
# -*- coding: utf-8 -*-
2
#
3
# passerelle - uniform access to multiple data sources and services
4
# Copyright (C) 2016 Entr'ouvert
5
#
6
# This program is free software: you can redistribute it and/or modify it
7
# under the terms of the GNU Affero General Public License as published
8
# by the Free Software Foundation, either version 3 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
# GNU Affero General Public License for more details.
15
#
16
# You should have received a copy of the GNU Affero General Public License
17
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
18

  
2 19
import json
3 20
import os
4 21
import pytest
5 22
from StringIO import StringIO
6 23

  
7
from django.contrib.auth.models import User
24
import six
25

  
8 26
from django.core.files import File
9 27
from django.core.urlresolvers import reverse
10 28
from django.contrib.contenttypes.models import ContentType
......
12 30
from django.core.management import call_command
13 31

  
14 32
from passerelle.base.models import ApiUser, AccessRight
33
from passerelle.apps.csvdatasource.models import CsvDataSource, Query, TableRow
34

  
15 35
from test_manager import login, admin_user
16 36

  
17 37
import webtest
......
40 60

  
41 61
data_bom = data.decode('utf-8').encode('utf-8-sig')
42 62

  
43
from passerelle.apps.csvdatasource.models import CsvDataSource, Query, TableRow
44

  
45 63
pytestmark = pytest.mark.django_db
46 64

  
47 65
TEST_BASE_DIR = os.path.join(os.path.dirname(__file__), 'data', 'csvdatasource')
48 66

  
49 67

  
50 68
def get_file_content(filename):
51
    return file(os.path.join(TEST_BASE_DIR, filename)).read()
69
    return open(os.path.join(TEST_BASE_DIR, filename)).read()
70

  
52 71

  
53 72
@pytest.fixture
54 73
def setup():
55

  
56
    def maker(columns_keynames='fam,id,lname,fname,sex', filename='data.csv', sheet_name='Feuille2', data='', skip_header=False):
57
        api = ApiUser.objects.create(username='all',
58
                        keytype='', key='')
59
        csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), filename), sheet_name=sheet_name,
60
               columns_keynames=columns_keynames, slug='test', title='a title', description='a description',
61
               skip_header=skip_header)
74
    def maker(columns_keynames='fam,id,lname,fname,sex', filename='data.csv',
75
              sheet_name='Feuille2', data='', skip_header=False):
76
        api = ApiUser.objects.create(
77
            username='all',
78
            keytype='',
79
            key='')
80
        csv = CsvDataSource.objects.create(
81
            csv_file=File(StringIO(data), filename),
82
            sheet_name=sheet_name,
83
            columns_keynames=columns_keynames,
84
            slug='test',
85
            title='a title',
86
            description='a description',
87
            skip_header=skip_header)
62 88
        assert TableRow.objects.filter(resource=csv).count() == len(csv.get_rows())
63 89
        obj_type = ContentType.objects.get_for_model(csv)
64
        AccessRight.objects.create(codename='can_access', apiuser=api,
65
                resource_type=obj_type, resource_pk=csv.pk)
90
        AccessRight.objects.create(
91
            codename='can_access',
92
            apiuser=api,
93
            resource_type=obj_type,
94
            resource_pk=csv.pk)
66 95
        url = reverse('csvdatasource-data', kwargs={'slug': csv.slug})
67 96
        return csv, url
68 97

  
69 98
    return maker
70 99

  
100

  
71 101
def parse_response(response):
72 102
    return json.loads(response.content)['data']
73 103

  
104

  
74 105
@pytest.fixture
75 106
def client():
76 107
    return Client()
77 108

  
109

  
78 110
@pytest.fixture(params=['data.csv', 'data.ods', 'data.xls', 'data.xlsx'])
79 111
def filetype(request):
80 112
    return request.param
81 113

  
114

  
82 115
def test_default_column_keynames(setup, filetype):
83 116
    csvdata = CsvDataSource.objects.create(csv_file=File(StringIO(get_file_content(filetype)), filetype),
84 117
                                           sheet_name='Feuille2',
......
89 122
    assert 'id' in csvdata.columns_keynames
90 123
    assert 'text' in csvdata.columns_keynames
91 124

  
125

  
92 126
def test_sheet_name_error(setup, app, filetype, admin_user):
93 127
    csvdata, url = setup('field,,another_field,', filename=filetype, data=get_file_content(filetype))
94 128
    app = login(app)
......
103 137
    else:
104 138
        assert resp.status_code == 302
105 139

  
140

  
106 141
def test_unfiltered_data(client, setup, filetype):
107 142
    csvdata, url = setup('field,,another_field,', filename=filetype, data=get_file_content(filetype))
108 143
    resp = client.get(url)
......
111 146
        assert 'field' in item
112 147
        assert 'another_field' in item
113 148

  
149

  
114 150
def test_empty_file(client, setup):
115
    csvdata, url = setup('field,,another_field,', filename='data-empty.ods',
116
            data=get_file_content('data-empty.ods'))
151
    csvdata, url = setup(
152
        'field,,another_field,',
153
        filename='data-empty.ods',
154
        data=get_file_content('data-empty.ods'))
117 155
    resp = client.get(url)
118 156
    result = parse_response(resp)
119 157
    assert len(result) == 0
120 158

  
159

  
121 160
def test_view_manage_page(setup, app, filetype, admin_user):
122 161
    csvdata, url = setup(',id,,text,', filename=filetype, data=get_file_content(filetype))
123 162
    app = login(app)
124
    resp = app.get(csvdata.get_absolute_url())
163
    app.get(csvdata.get_absolute_url())
164

  
125 165

  
126 166
def test_good_filter_data(client, setup, filetype):
127 167
    filter_criteria = 'Zakia'
......
135 175
        assert 'text' in item
136 176
        assert filter_criteria in item['text']
137 177

  
178

  
138 179
def test_bad_filter_data(client, setup, filetype):
139 180
    filter_criteria = 'bad'
140 181
    csvdata, url = setup(',id,,text,', filename=filetype, data=get_file_content(filetype))
......
143 184
    result = parse_response(resp)
144 185
    assert len(result) == 0
145 186

  
187

  
146 188
def test_useless_filter_data(client, setup, filetype):
147 189
    csvdata, url = setup('id,,nom,prenom,sexe', filename=filetype, data=get_file_content(filetype))
148 190
    filters = {'text': 'Ali'}
......
150 192
    result = parse_response(resp)
151 193
    assert len(result) == 20
152 194

  
195

  
153 196
def test_columns_keynames_with_spaces(client, setup, filetype):
154 197
    csvdata, url = setup('id , , nom,text , ', filename=filetype, data=get_file_content(filetype))
155 198
    filters = {'text': 'Yaniss'}
......
157 200
    result = parse_response(resp)
158 201
    assert len(result) == 1
159 202

  
203

  
160 204
def test_skipped_header_data(client, setup, filetype):
161 205
    csvdata, url = setup(',id,,text,', filename=filetype, data=get_file_content(filetype), skip_header=True)
162 206
    filters = {'q': 'Eliot'}
......
164 208
    result = parse_response(resp)
165 209
    assert len(result) == 0
166 210

  
211

  
167 212
def test_data(client, setup, filetype):
168 213
    csvdata, url = setup('fam,id,, text,sexe ', filename=filetype, data=get_file_content(filetype))
169 214
    filters = {'text': 'Sacha'}
......
172 217
    assert result[0] == {'id': '59', 'text': 'Sacha',
173 218
                         'fam': '2431', 'sexe': 'H'}
174 219

  
220

  
175 221
def test_unicode_filter_data(client, setup, filetype):
176 222
    filter_criteria = u'Benoît'
177 223
    csvdata, url = setup(',id,,text,', filename=filetype, data=get_file_content(filetype))
......
184 230
        assert 'text' in item
185 231
        assert filter_criteria in item['text']
186 232

  
233

  
187 234
def test_unicode_case_insensitive_filter_data(client, setup, filetype):
188 235
    csvdata, url = setup(',id,,text,', filename=filetype, data=get_file_content(filetype))
189 236
    filter_criteria = u'anaëlle'
190
    filters = {'text':filter_criteria, 'case-insensitive':''}
237
    filters = {'text': filter_criteria, 'case-insensitive': ''}
191 238
    resp = client.get(url, filters)
192 239
    result = parse_response(resp)
193 240
    assert len(result)
......
196 243
        assert 'text' in item
197 244
        assert filter_criteria.lower() in item['text'].lower()
198 245

  
246

  
199 247
def test_data_bom(client, setup):
200 248
    csvdata, url = setup('fam,id,, text,sexe ', data=data_bom)
201
    filters = {'text':'Eliot'}
249
    filters = {'text': 'Eliot'}
202 250
    resp = client.get(url, filters)
203 251
    result = parse_response(resp)
204 252
    assert result[0] == {'id': '69981', 'text': 'Eliot', 'fam': '121', 'sexe': 'H'}
205 253

  
254

  
206 255
def test_multi_filter(client, setup, filetype):
207 256
    csvdata, url = setup('fam,id,, text,sexe ', filename=filetype, data=get_file_content(filetype))
208
    filters = {'sexe':'F'}
257
    filters = {'sexe': 'F'}
209 258
    resp = client.get(url, filters)
210 259
    result = parse_response(resp)
211 260
    assert result[0] == {'id': '6', 'text': 'Shanone',
212 261
                         'fam': '525', 'sexe': 'F'}
213 262
    assert len(result) == 10
214 263

  
264

  
215 265
def test_query(client, setup, filetype):
216 266
    csvdata, url = setup('fam,id,, text,sexe ', filename=filetype, data=get_file_content(filetype))
217
    filters =  {'q':'liot'}
267
    filters = {'q': 'liot'}
218 268
    resp = client.get(url, filters)
219 269
    result = parse_response(resp)
220 270
    assert result[0]['text'] == 'Eliot'
221 271
    assert len(result) == 1
222 272

  
273

  
223 274
def test_query_insensitive_and_unicode(client, setup, filetype):
224 275
    csvdata, url = setup('fam,id,, text,sexe ', filename=filetype, data=get_file_content(filetype))
225
    filters = {'q':'elIo', 'case-insensitive':''}
276
    filters = {'q': 'elIo', 'case-insensitive': ''}
226 277
    resp = client.get(url, filters)
227 278
    result = parse_response(resp)
228 279
    assert result[0]['text'] == 'Eliot'
......
233 284
    assert result[0]['text'] == 'Eliot'
234 285
    assert len(result) == 1
235 286

  
287

  
236 288
def test_query_insensitive_and_filter(client, setup, filetype):
237 289
    csvdata, url = setup('fam,id,,text,sexe', filename=filetype, data=get_file_content(filetype))
238
    filters = {'q':'elIo', 'sexe': 'H', 'case-insensitive':''}
290
    filters = {'q': 'elIo', 'sexe': 'H', 'case-insensitive': ''}
239 291
    resp = client.get(url, filters)
240 292
    result = parse_response(resp)
241 293
    assert result[0]['text'] == 'Eliot'
......
261 313
    assert result[0]['id'] == '22'
262 314
    assert result[0]['lname'] == 'MARTIN'
263 315

  
316

  
264 317
def test_on_the_fly_dialect_detection(client, setup):
265 318
    # fake a connector that was not initialized during .save(), because it's
266 319
    # been migrated and we didn't do dialect detection at save() time.
......
271 324
    assert result['err'] == 0
272 325
    assert len(result['data']) == 20
273 326

  
327

  
274 328
def test_missing_columns(client, setup):
275 329
    csvdata, url = setup(data=data + 'A;B;C\n')
276 330
    resp = client.get(url)
......
279 333
    assert len(result['data']) == 21
280 334
    assert result['data'][-1] == {'lname': 'C', 'sex': None, 'id': 'B', 'fname': None, 'fam': 'A'}
281 335

  
336

  
282 337
def test_unknown_sheet_name(client, setup):
283 338
    csvdata, url = setup('field,,another_field,', filename='data2.ods', data=get_file_content('data2.ods'))
284 339
    csvdata.sheet_name = 'unknown'
......
287 342
    result = json.loads(resp.content)
288 343
    assert len(result['data']) == 20
289 344

  
345

  
290 346
def test_cache_new_shorter_file(client, setup):
291 347
    csvdata, url = setup(data=data + 'A;B;C\n')
292 348
    resp = client.get(url)
......
299 355
    result = json.loads(resp.content)
300 356
    assert len(result['data']) == 20
301 357

  
358

  
302 359
def test_query_array(app, setup, filetype):
303 360
    csvdata, url = setup('id,whatever,nom,prenom,sexe', filename=filetype, data=get_file_content(filetype))
304 361
    url = reverse('generic-endpoint', kwargs={
......
315 372
    for row in response.json['data']:
316 373
        assert len(row) == 2
317 374
        assert isinstance(row[0], int)
318
        assert isinstance(row[1], unicode)
375
        assert isinstance(row[1], six.text_type)
319 376

  
320 377

  
321 378
def test_query_q_filter(app, setup, filetype):
......
353 410
    for row in response.json['data']:
354 411
        assert len(row) == 2
355 412
        assert isinstance(row['id'], int)
356
        assert isinstance(row['prenom'], unicode)
413
        assert isinstance(row['prenom'], six.text_type)
357 414

  
358 415

  
359 416
def test_query_tuples(app, setup, filetype):
......
374 431
        assert row[0][0] == 'id'
375 432
        assert isinstance(row[0][1], int)
376 433
        assert row[1][0] == 'prenom'
377
        assert isinstance(row[1][1], unicode)
434
        assert isinstance(row[1][1], six.text_type)
378 435

  
379 436

  
380 437
def test_query_onerow(app, setup, filetype):
......
440 497
    assert isinstance(response.json['data'], list)
441 498
    assert len(response.json['data']) == 2
442 499

  
500

  
443 501
def test_query_keyed_distinct(app, setup, filetype):
444 502
    csvdata, url = setup('id,whatever,nom,prenom,sexe', filename=filetype, data=get_file_content(filetype))
445 503
    url = reverse('generic-endpoint', kwargs={
......
454 512
    assert isinstance(response.json['data'], dict)
455 513
    assert response.json['data']['MARTIN']['prenom'] == 'Sandra'
456 514

  
515

  
457 516
def test_query_order(app, setup, filetype):
458 517
    csvdata, url = setup('id,whatever,nom,prenom,sexe', filename=filetype, data=get_file_content(filetype))
459 518
    url = reverse('generic-endpoint', kwargs={
......
566 625
    assert response.json['data']['expr'] == 'zob'
567 626
    assert 'row' in response.json['data']
568 627

  
628

  
569 629
def test_edit_connector_queries(admin_user, app, setup, filetype):
570 630
    csvdata, url = setup('id,whatever,nom,prenom,sexe', filename=filetype, data=get_file_content(filetype))
571 631
    url = reverse('view-connector', kwargs={'connector': 'csvdatasource', 'slug': csvdata.slug})
572 632
    resp = app.get(url)
573
    assert not 'New Query' in resp.body
633
    assert 'New Query' not in resp.body
574 634
    new_query_url = reverse('csv-new-query', kwargs={'connector_slug': csvdata.slug})
575 635
    resp = app.get(new_query_url, status=302)
576 636
    assert resp.location.endswith('/login/?next=%s' % new_query_url)
......
589 649
    assert 'Syntax error' in resp.body
590 650
    resp.form['projections'] = 'id:id\nprenom:prenom'
591 651
    resp = resp.form.submit().follow()
592
    resp = resp.click('foobar', index=1) # 0th is the newly created endpoint
652
    resp = resp.click('foobar', index=1)  # 0th is the newly created endpoint
593 653
    resp.form['filters'] = 'int(id) == 511'
594 654
    resp.form['description'] = 'in the sky without diamonds'
595 655
    resp = resp.form.submit().follow()
596 656
    assert 'Lucy alone' in resp.body
597 657
    assert 'in the sky without diamonds' in resp.body
598
    resp = resp.click('foobar', index=0) # 0th is the newly created endpoint
658
    resp = resp.click('foobar', index=0)  # 0th is the newly created endpoint
599 659
    assert len(resp.json['data']) == 1
600 660
    assert resp.json['data'][0]['prenom'] == 'Lucie'
601 661

  
662

  
602 663
def test_download_file(app, setup, filetype, admin_user):
603 664
    csvdata, url = setup('field,,another_field,', filename=filetype, data=get_file_content(filetype))
604 665
    assert '/login' in app.get('/manage/csvdatasource/test/download/').location
......
630 691
    assert response.json['err'] == 0
631 692
    assert len(response.json['data']) == 2
632 693

  
694

  
633 695
def test_delete_connector_query(admin_user, app, setup, filetype):
634 696
    csvdata, url = setup('id,whatever,nom,prenom,sexe', filename=filetype, data=get_file_content(filetype))
635 697
    url = reverse('view-connector', kwargs={'connector': 'csvdatasource', 'slug': csvdata.slug})
636
-