Projet

Général

Profil

0001-csvdatasource-allow-normalized-search-for-unicode-va.patch

Serghei Mihai (congés, retour 15/05), 14 novembre 2018 12:14

Télécharger (3,23 ko)

Voir les différences: 

Subject: [PATCH] csvdatasource: allow normalized search for unicode values
 (#26188)

Activate normalized search by default for 'q' filters.

 passerelle/apps/csvdatasource/models.py | 8 ++++++--
 tests/test_csv_datasource.py            | 7 ++++++-
 2 files changed, 12 insertions(+), 3 deletions(-)
passerelle/apps/csvdatasource/models.py
18 18 
import re
19 19 
import csv
20 20 
import itertools
21 
import unicodedata
21 22 
from collections import OrderedDict
22 23 
from pyexcel_ods import get_data as get_data_ods
23 24 
from pyexcel_xls import get_data as get_data_xls
......
50 51 
        code_cache[expr] = compile(expr, '<inline>', 'eval')
51 52 
    return code_cache[expr]
52 53 

  		




  
  54
  
    
def normalize(value):

  		




  
  55
  
    
    return unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')

  		




  
  56
  
    

  		




  53
  57
  
    

  		




  54
  58
  
    
class Query(models.Model):

  		




  55
  59
  
    
    resource = models.ForeignKey('CsvDataSource')

  		




  ......




  279
  283
  
    
                row_vars['query'] = kwargs

  		




  280
  284
  
    
                for i, (code, expr) in enumerate(codes):

  		




  281
  285
  
    
                    try:

  		




  282
  
  
    
                        result = eval(code, {}, row_vars)

  		




  
  286
  
    
                        result = eval(code, {'normalize': normalize}, row_vars)

  		




  283
  287
  
    
                    except Exception as e:

  		




  284
  288
  
    
                        data = {

  		




  285
  289
  
    
                            'expr': expr,

  		




  ......




  348
  352
  
    
        # filtering is done there afater projection because we need a projection named text for

  		




  349
  353
  
    
        # retro-compatibility with previous use of the csvdatasource with select2

  		




  350
  354
  
    
        if 'q' in request.GET:

  		




  351
  
  
    
            filters = ["%s in text.lower()" % repr(request.GET['q'].lower())]

  		




  
  355
  
    
            filters = ["%s in normalize(text.lower())" % repr(normalize(request.GET['q'].lower()))]

  		




  352
  356
  
    
            data = [row for new_row, row in stream_expressions(filters, data, kind='filters')

  		




  353
  357
  
    
                    if new_row[0]]

  		




  354
  358
  
    

  		












  

    
      tests/test_csv_datasource.py
    
  





  219
  219
  
    
    assert result[0]['text'] == 'Eliot'

  		




  220
  220
  
    
    assert len(result) == 1

  		




  221
  221
  
    

  		




  222
  
  
    
def test_query_insensitive(client, setup, filetype):

  		




  
  222
  
    
def test_query_insensitive_and_unicode(client, setup, filetype):

  		




  223
  223
  
    
    csvdata, url = setup('fam,id,, text,sexe ', filename=filetype, data=get_file_content(filetype))

  		




  224
  224
  
    
    filters = {'q':'elIo', 'case-insensitive':''}

  		




  225
  225
  
    
    resp = client.get(url, filters)

  		




  226
  226
  
    
    result = parse_response(resp)

  		




  227
  227
  
    
    assert result[0]['text'] == 'Eliot'

  		




  228
  228
  
    
    assert len(result) == 1

  		




  
  229
  
    
    filters['q'] = 'élIo'

  		




  
  230
  
    
    resp = client.get(url, filters)

  		




  
  231
  
    
    result = parse_response(resp)

  		




  
  232
  
    
    assert result[0]['text'] == 'Eliot'

  		




  
  233
  
    
    assert len(result) == 1

  		




  229
  234
  
    

  		




  230
  235
  
    
def test_query_insensitive_and_filter(client, setup, filetype):

  		




  231
  236
  
    
    csvdata, url = setup('fam,id,,text,sexe', filename=filetype, data=get_file_content(filetype))

  		




  232
  
  
    
-