Projet

Général

Profil

0001-csvdatasource-allow-normalized-search-for-unicode-va.patch

Serghei Mihai (congés, retour 15/05), 13 novembre 2018 09:49

Télécharger (5,02 ko)

Voir les différences: 

Subject: [PATCH] csvdatasource: allow normalized search for unicode values
 (#26188)


 passerelle/apps/csvdatasource/lookups.py | 17 +++++++++++++
 passerelle/apps/csvdatasource/views.py   | 13 +++++++---
 tests/test_csv_datasource.py             | 31 ++++++++++++++++++++++++
 3 files changed, 58 insertions(+), 3 deletions(-)
passerelle/apps/csvdatasource/lookups.py
1 1 
DELIMITER = '__'
2 2 

  		




  
  3
  
    
import unicodedata

  		




  
  4
  
    

  		




  3
  5
  
    
class InvalidOperatorError(Exception):

  		




  4
  6
  
    
    pass

  		




  5
  7
  
    

  		




  ......




  13
  15
  
    
    except (ValueError, TypeError):

  		




  14
  16
  
    
        return False

  		




  15
  17
  
    

  		




  
  18
  
    
def normalize(value):

  		




  
  19
  
    
    return unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')

  		




  
  20
  
    

  		




  16
  21
  
    

  		




  17
  22
  
    
class Lookup(object):

  		




  18
  23
  
    

  		




  ......




  22
  27
  
    
    def icontains(self, key, value):

  		




  23
  28
  
    
        return lambda x: value.lower() in x[key].lower()

  		




  24
  29
  
    

  		




  
  30
  
    
    def normcontains(self, key, value):

  		




  
  31
  
    
        return lambda x: normalize(value) in normalize(x[key])

  		




  
  32
  
    

  		




  
  33
  
    
    def inormcontains(self, key, value):

  		




  
  34
  
    
        return lambda x: normalize(value.lower()) in normalize(x[key].lower())

  		




  
  35
  
    

  		




  25
  36
  
    
    def gt(self, key, value):

  		




  26
  37
  
    
        return lambda x: int(x[key]) > int(value)

  		




  27
  38
  
    

  		




  ......




  54
  65
  
    
    def ieq(self, key, value):

  		




  55
  66
  
    
        return lambda x: value.lower() == x[key].lower()

  		




  56
  67
  
    

  		




  
  68
  
    
    def normeq(self, key, value):

  		




  
  69
  
    
        return lambda x: normalize(value) == normalize(x[key])

  		




  
  70
  
    

  		




  
  71
  
    
    def inormeq(self, key, value):

  		




  
  72
  
    
        return lambda x: normalize(value.lower()) == normalize(x[key].lower())

  		




  
  73
  
    

  		




  57
  74
  
    
    def ne(self, key, value):

  		




  58
  75
  
    
        if is_int(value):

  		




  59
  76
  
    
            return lambda x: int(value) != int(x[key])

  		












  

    
      passerelle/apps/csvdatasource/views.py
    
  





  38
  38
  
    
        params = request.GET

  		




  39
  39
  
    

  		




  40
  40
  
    
        case_insensitive = 'case-insensitive' in params

  		




  
  41
  
    
        normalized = 'normalized' in params

  		




  41
  42
  
    
        query = params.get('q', None)

  		




  42
  43
  
    

  		




  
  44
  
    
        if normalized:

  		




  
  45
  
    
            lookup = 'normcontains'

  		




  
  46
  
    
        else:

  		




  
  47
  
    
            lookup = 'contains'

  		




  
  48
  
    

  		




  43
  49
  
    
        if query:

  		




  44
  50
  
    
            if case_insensitive:

  		




  45
  
  
    
                filters['text__icontains'] = query.lower()

  		




  46
  
  
    
            else:

  		




  47
  
  
    
                filters['text__contains'] = query

  		




  
  51
  
    
                lookup = 'i' + lookup

  		




  
  52
  
    
                query = query.lower()

  		




  
  53
  
    

  		




  
  54
  
    
            filters['text__%s' % lookup] = query

  		




  48
  55
  
    

  		




  49
  56
  
    
        # builds filters according to csv file header

  		




  50
  57
  
    
        for column_title in [t.strip() for t in obj.columns_keynames.split(',') if t]:

  		












  

    
      tests/test_csv_datasource.py
    
  





  182
  182
  
    
        assert 'text' in item

  		




  183
  183
  
    
        assert filter_criteria in item['text']

  		




  184
  184
  
    

  		




  
  185
  
    
    # require normalized comparison

  		




  
  186
  
    
    filters['normalized'] = ''

  		




  
  187
  
    
    filters['text'] = 'Benoit'

  		




  
  188
  
    
    resp = client.get(url, filters)

  		




  
  189
  
    
    assert len(result)

  		




  
  190
  
    

  		




  185
  191
  
    
def test_unicode_case_insensitive_filter_data(client, setup, filetype):

  		




  186
  192
  
    
    csvdata, url = setup(',id,,text,', filename=filetype, data=get_file_content(filetype))

  		




  187
  193
  
    
    filter_criteria = u'anaëlle'

  		




  ......




  194
  200
  
    
        assert 'text' in item

  		




  195
  201
  
    
        assert filter_criteria.lower() in item['text'].lower()

  		




  196
  202
  
    

  		




  
  203
  
    
    # require normalized comparison

  		




  
  204
  
    
    filters['normalized'] = ''

  		




  
  205
  
    
    filters['text'] = 'anaelle'

  		




  
  206
  
    
    resp = client.get(url, filters)

  		




  
  207
  
    
    assert len(result)

  		




  
  208
  
    

  		




  197
  209
  
    
def test_data_bom(client, setup):

  		




  198
  210
  
    
    csvdata, url = setup('fam,id,, text,sexe ', data=data_bom)

  		




  199
  211
  
    
    filters = {'text':'Eliot'}

  		




  ......




  258
  270
  
    
    assert result[0]['id'] == '22'

  		




  259
  271
  
    
    assert result[0]['lname'] == 'MARTIN'

  		




  260
  272
  
    

  		




  
  273
  
    
def test_normalized_filters(client, setup, filetype):

  		




  
  274
  
    
    csvdata, url = setup(filename=filetype, data=get_file_content(filetype))

  		




  
  275
  
    
    filters = {

  		




  
  276
  
    
        'fname__normcontains': 'aëlle'

  		




  
  277
  
    
    }

  		




  
  278
  
    
    resp = client.get(url, filters)

  		




  
  279
  
    
    result = parse_response(resp)

  		




  
  280
  
    
    assert len(result) == 1

  		




  
  281
  
    
    assert result[0]['id'] == '46'

  		




  
  282
  
    
    assert result[0]['lname'] == 'WILSON-LUZAYADIO'

  		




  
  283
  
    

  		




  
  284
  
    
    filters = {

  		




  
  285
  
    
        'fname__inormcontains': 'ANAëllE'

  		




  
  286
  
    
    }

  		




  
  287
  
    
    resp = client.get(url, filters)

  		




  
  288
  
    
    result = parse_response(resp)

  		




  
  289
  
    
    assert len(result) == 1

  		




  
  290
  
    
    assert result[0]['fname'] == u'Anaëlle'

  		




  
  291
  
    

  		




  261
  292
  
    
def test_unknown_operator(client, setup, filetype):

  		




  262
  293
  
    
    csvdata, url = setup(filename=filetype, data=get_file_content(filetype))

  		




  263
  294
  
    
    filters = {'id__whatever': '25', 'fname__icontains':'Eliot'}

  		




  264
  
  
    
-