From f4932281f914a64566b38218ee962612379fe741 Mon Sep 17 00:00:00 2001 From: Serghei Mihai Date: Wed, 14 Nov 2018 10:54:19 +0100 Subject: [PATCH] csvdatasource: allow normalized search for unicode values (#26188) Activate normalized search by default for 'q' filters. --- passerelle/apps/csvdatasource/models.py | 8 ++++++-- tests/test_csv_datasource.py | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/passerelle/apps/csvdatasource/models.py b/passerelle/apps/csvdatasource/models.py index 7c02ed19..f39b697d 100644 --- a/passerelle/apps/csvdatasource/models.py +++ b/passerelle/apps/csvdatasource/models.py @@ -18,6 +18,7 @@ import os import re import csv import itertools +import unicodedata from collections import OrderedDict from pyexcel_ods import get_data as get_data_ods from pyexcel_xls import get_data as get_data_xls @@ -50,6 +51,9 @@ def get_code(expr): code_cache[expr] = compile(expr, '', 'eval') return code_cache[expr] +def normalize(value): + return unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') + class Query(models.Model): resource = models.ForeignKey('CsvDataSource') @@ -279,7 +283,7 @@ class CsvDataSource(BaseResource): row_vars['query'] = kwargs for i, (code, expr) in enumerate(codes): try: - result = eval(code, {}, row_vars) + result = eval(code, {'normalize': normalize}, row_vars) except Exception as e: data = { 'expr': expr, @@ -348,7 +352,7 @@ class CsvDataSource(BaseResource): # filtering is done there afater projection because we need a projection named text for # retro-compatibility with previous use of the csvdatasource with select2 if 'q' in request.GET: - filters = ["%s in text.lower()" % repr(request.GET['q'].lower())] + filters = ["%s in normalize(text.lower())" % repr(normalize(request.GET['q'].lower()))] data = [row for new_row, row in stream_expressions(filters, data, kind='filters') if new_row[0]] diff --git a/tests/test_csv_datasource.py b/tests/test_csv_datasource.py index beec8349..a8d5350a 100644 --- a/tests/test_csv_datasource.py +++ b/tests/test_csv_datasource.py @@ -219,13 +219,18 @@ def test_query(client, setup, filetype): assert result[0]['text'] == 'Eliot' assert len(result) == 1 -def test_query_insensitive(client, setup, filetype): +def test_query_insensitive_and_unicode(client, setup, filetype): csvdata, url = setup('fam,id,, text,sexe ', filename=filetype, data=get_file_content(filetype)) filters = {'q':'elIo', 'case-insensitive':''} resp = client.get(url, filters) result = parse_response(resp) assert result[0]['text'] == 'Eliot' assert len(result) == 1 + filters['q'] = 'élIo' + resp = client.get(url, filters) + result = parse_response(resp) + assert result[0]['text'] == 'Eliot' + assert len(result) == 1 def test_query_insensitive_and_filter(client, setup, filetype): csvdata, url = setup('fam,id,,text,sexe', filename=filetype, data=get_file_content(filetype)) -- 2.19.1