From ff2f4f8d72b4920cc686233d57fdc57a80fa79ba Mon Sep 17 00:00:00 2001 From: Serghei Mihai Date: Tue, 13 Nov 2018 14:05:58 +0100 Subject: [PATCH] csvdatasource: remove advanced lookup filters (#13748) --- passerelle/apps/csvdatasource/lookups.py | 70 ------------------------ passerelle/apps/csvdatasource/models.py | 31 ++++++----- passerelle/apps/csvdatasource/views.py | 28 ++-------- tests/test_csv_datasource.py | 35 +----------- 4 files changed, 22 insertions(+), 142 deletions(-) delete mode 100644 passerelle/apps/csvdatasource/lookups.py diff --git a/passerelle/apps/csvdatasource/lookups.py b/passerelle/apps/csvdatasource/lookups.py deleted file mode 100644 index 667cdc07..00000000 --- a/passerelle/apps/csvdatasource/lookups.py +++ /dev/null @@ -1,70 +0,0 @@ -DELIMITER = '__' - -class InvalidOperatorError(Exception): - pass - -compare_str = cmp - - -def is_int(value): - try: - int(value) - return True - except (ValueError, TypeError): - return False - - -class Lookup(object): - - def contains(self, key, value): - return lambda x: value in x[key] - - def icontains(self, key, value): - return lambda x: value.lower() in x[key].lower() - - def gt(self, key, value): - return lambda x: int(x[key]) > int(value) - - def igt(self, key, value): - return lambda x: compare_str(x[key].lower(), value.lower()) > 0 - - def ge(self, key, value): - return lambda x: int(x[key]) >= int(value) - - def ige(self, key, value): - return lambda x: compare_str(x[key].lower(), value.lower()) >= 0 - - def lt(self, key, value): - return lambda x: int(x[key]) < int(value) - - def ilt(self, key, value): - return lambda x: compare_str(x[key].lower(), value.lower()) < 0 - - def le(self, key, value): - return lambda x: int(x[key]) <= int(value) - - def ile(self, key, value): - return lambda x: compare_str(x[key].lower(), value.lower()) <= 0 - - def eq(self, key, value): - if is_int(value): - return lambda x: int(value) == int(x[key]) - return lambda x: value == x[key] - - def ieq(self, key, value): - return lambda x: value.lower() == x[key].lower() - - def ne(self, key, value): - if is_int(value): - return lambda x: int(value) != int(x[key]) - return lambda x: value != x[key] - - def ine(self, key, value): - return lambda x: value.lower() != x[key].lower() - - -def get_lookup(operator, key, value): - try: - return getattr(Lookup(), operator)(key, value) - except (AttributeError,): - raise InvalidOperatorError('%s is not a valid operator' % operator) diff --git a/passerelle/apps/csvdatasource/models.py b/passerelle/apps/csvdatasource/models.py index 7b0d5eef..e77f5716 100644 --- a/passerelle/apps/csvdatasource/models.py +++ b/passerelle/apps/csvdatasource/models.py @@ -35,8 +35,6 @@ from passerelle.base.models import BaseResource from passerelle.utils.jsonresponse import APIError from passerelle.utils.api import endpoint -import lookups - identifier_re = re.compile(r"^[^\d\W]\w*\Z", re.UNICODE) @@ -242,28 +240,31 @@ class CsvDataSource(BaseResource): for data in self.get_cached_rows(initial=False): yield data - def get_data(self, filters=None): + def get_data(self, filters={}, case_insensitive=False): titles = [t.strip() for t in self.columns_keynames.split(',')] - # validate filters (appropriate columns must exist) - if filters: - for filter_key in filters.keys(): - if not filter_key.split(lookups.DELIMITER)[0] in titles: - del filters[filter_key] + for filter_key in filters.keys(): + # allow 'q' filter + if filter_key == 'q': + continue + if filter_key not in titles: + del filters[filter_key] rows = self.get_cached_rows() data = [] # build a generator of all filters - def filters_generator(filters, titles): + def filters_generator(filters, case_insensitive): if not filters: return for key, value in filters.items(): - try: - key, op = key.split(lookups.DELIMITER) - except (ValueError,): - op = 'eq' - yield lookups.get_lookup(op, key, value) + def operation(key, value, case_insensitive): + if key == 'q': + return lambda x: value.lower() in x['text'].lower() + if case_insensitive: + return lambda x: value.lower() == x[key].lower() + return lambda x: value == x[key] + yield operation(key, value, case_insensitive) # apply filters to data def super_filter(filters, data): @@ -272,7 +273,7 @@ class CsvDataSource(BaseResource): return data data = list(super_filter( - filters_generator(filters, titles), rows + filters_generator(filters, case_insensitive), rows )) return data diff --git a/passerelle/apps/csvdatasource/views.py b/passerelle/apps/csvdatasource/views.py index 1754c2d2..387cfe82 100644 --- a/passerelle/apps/csvdatasource/views.py +++ b/passerelle/apps/csvdatasource/views.py @@ -34,35 +34,16 @@ class CsvDataView(View, SingleObjectMixin): def _filters_builder(self, request): filters = {} obj = self.get_object() - params = request.GET - - case_insensitive = 'case-insensitive' in params query = params.get('q', None) if query: - if case_insensitive: - filters['text__icontains'] = query.lower() - else: - filters['text__contains'] = query + filters['q'] = query # builds filters according to csv file header for column_title in [t.strip() for t in obj.columns_keynames.split(',') if t]: - match = filter( - (lambda ct: lambda x: x.startswith(ct))(column_title), params.keys() - ) - for key in match: - if case_insensitive: - filters[key + '__ieq'] = params[key].lower() - else: - filters[key] = params[key] - - if 'text' in filters: - if case_insensitive: - filters['text__ieq'] = filters['text'].lower() - else: - filters['text__eq'] = filters['text'] - filters.pop('text') + if column_title in params.keys(): + filters[column_title] = params[column_title] return filters @@ -72,7 +53,8 @@ class CsvDataView(View, SingleObjectMixin): def get(self, request, *args, **kwargs): obj = self.get_object() filters = self._filters_builder(request) - return {'data': obj.get_data(filters)} + case_insensitive = 'case-insensitive' in request.GET + return {'data': obj.get_data(filters, case_insensitive)} class NewQueryView(CreateView): diff --git a/tests/test_csv_datasource.py b/tests/test_csv_datasource.py index 0d707df6..3576ab53 100644 --- a/tests/test_csv_datasource.py +++ b/tests/test_csv_datasource.py @@ -234,39 +234,6 @@ def test_query_insensitive_and_filter(client, setup, filetype): assert result[0]['text'] == 'Eliot' assert len(result) == 1 -def test_advanced_filters(client, setup, filetype): - csvdata, url = setup(filename=filetype, data=get_file_content(filetype)) - filters = {'id__gt':20, 'id__lt': 40} - resp = client.get(url, filters) - result = parse_response(resp) - assert len(result) == 3 - for stuff in result: - assert stuff['id'] in ('22', '36', '38') - -def test_advanced_filters_combo(client, setup, filetype): - csvdata, url = setup(filename=filetype, data=get_file_content(filetype)) - filters = { - 'id__ge': '20', - 'id__lt': '40', - 'fam__gt': '234', - 'fam__le': '235', - 'fname__icontains': 'Sandra' - } - resp = client.get(url, filters) - result = parse_response(resp) - assert len(result) == 1 - assert result[0]['id'] == '22' - assert result[0]['lname'] == 'MARTIN' - -def test_unknown_operator(client, setup, filetype): - csvdata, url = setup(filename=filetype, data=get_file_content(filetype)) - filters = {'id__whatever': '25', 'fname__icontains':'Eliot'} - resp = client.get(url, filters) - result = json.loads(resp.content) - assert result['err'] == 1 - assert result['err_class'] == 'passerelle.apps.csvdatasource.lookups.InvalidOperatorError' - assert result['err_desc'] == 'whatever is not a valid operator' - def test_dialect(client, setup): csvdata, url = setup(data=data) @@ -280,7 +247,7 @@ def test_dialect(client, setup): } assert expected == csvdata.dialect_options - filters = {'id__gt': '20', 'id__lt': '40', 'fname__icontains': 'Sandra'} + filters = {'id': '22', 'fname': 'Sandra'} resp = client.get(url, filters) result = parse_response(resp) assert len(result) == 1 -- 2.19.1