From e9bcf8c8ea70d2d0008dacd34ad677f41fbe18d2 Mon Sep 17 00:00:00 2001 From: Serghei Mihai Date: Wed, 26 Aug 2015 15:39:01 +0200 Subject: [PATCH] csv files datasource (#5896) --- passerelle/apps/csvdatasource/__init__.py | 27 ++++++++ passerelle/apps/csvdatasource/forms.py | 16 +++++ .../apps/csvdatasource/migrations/0001_initial.py | 30 +++++++++ passerelle/apps/csvdatasource/models.py | 76 ++++++++++++++++++++++ .../csvdatasource/csvdatasource_detail.html | 61 +++++++++++++++++ passerelle/apps/csvdatasource/urls.py | 27 ++++++++ passerelle/apps/csvdatasource/views.py | 45 +++++++++++++ passerelle/locale/fr/LC_MESSAGES/django.po | 68 +++++++++++++++++-- passerelle/settings.py | 1 + passerelle/static/css/style.css | 3 + tests/test_csv_datasource.py | 63 ++++++++++++++++++ 11 files changed, 410 insertions(+), 7 deletions(-) create mode 100644 passerelle/apps/csvdatasource/__init__.py create mode 100644 passerelle/apps/csvdatasource/forms.py create mode 100644 passerelle/apps/csvdatasource/migrations/0001_initial.py create mode 100644 passerelle/apps/csvdatasource/models.py create mode 100644 passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html create mode 100644 passerelle/apps/csvdatasource/urls.py create mode 100644 passerelle/apps/csvdatasource/views.py create mode 100644 tests/test_csv_datasource.py diff --git a/passerelle/apps/csvdatasource/__init__.py b/passerelle/apps/csvdatasource/__init__.py new file mode 100644 index 0000000..6481473 --- /dev/null +++ b/passerelle/apps/csvdatasource/__init__.py @@ -0,0 +1,27 @@ +# passerelle.apps.csvdatasource +# Copyright (C) 2015 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import django.apps + + +class AppConfig(django.apps.AppConfig): + name = 'csvdatasource' + + def get_after_urls(self): + from . import urls + return urls.urlpatterns + +default_app_config = 'csvdatasource.AppConfig' diff --git a/passerelle/apps/csvdatasource/forms.py b/passerelle/apps/csvdatasource/forms.py new file mode 100644 index 0000000..5b2d30a --- /dev/null +++ b/passerelle/apps/csvdatasource/forms.py @@ -0,0 +1,16 @@ +from django.utils.text import slugify +from django import forms + +from .models import CsvDataSource + + +class CsvDataSourceForm(forms.ModelForm): + + class Meta: + model = CsvDataSource + exclude = ('slug', 'users') + + def save(self, commit=True): + if not self.instance.slug: + self.instance.slug = slugify(self.instance.title) + return super(CsvDataSourceForm, self).save(commit=commit) diff --git a/passerelle/apps/csvdatasource/migrations/0001_initial.py b/passerelle/apps/csvdatasource/migrations/0001_initial.py new file mode 100644 index 0000000..866ed62 --- /dev/null +++ b/passerelle/apps/csvdatasource/migrations/0001_initial.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='CsvDataSource', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('title', models.CharField(max_length=50)), + ('slug', models.SlugField()), + ('description', models.TextField()), + ('csv_file', models.FileField(upload_to=b'csv', verbose_name='CSV File')), + ('columns_titles', models.CharField(help_text='in "[column1_title],[column2_title],..." format', max_length=256, verbose_name='Column titles')), + ('users', models.ManyToManyField(to='base.ApiUser', blank=True)), + ], + options={ + 'verbose_name': 'CSV File', + }, + bases=(models.Model,), + ), + ] diff --git a/passerelle/apps/csvdatasource/models.py b/passerelle/apps/csvdatasource/models.py new file mode 100644 index 0000000..1af0ac0 --- /dev/null +++ b/passerelle/apps/csvdatasource/models.py @@ -0,0 +1,76 @@ +import re +import csv + +from django.db import models +from django.utils.translation import ugettext_lazy as _ +from django.core.urlresolvers import reverse + +from passerelle.base.models import BaseResource + +_CACHE_SENTINEL = object() + + +class CsvDataSource(BaseResource): + csv_file = models.FileField(_('CSV File'), upload_to='csv') + columns_titles = models.CharField(max_length=256, + verbose_name=_('Column titles'), + help_text=_('in "[column1_title],[column2_title],..." format')) + + category = _('Data Sources') + + class Meta: + verbose_name = 'CSV File' + + @classmethod + def get_verbose_name(cls): + return cls._meta.verbose_name + + @classmethod + def get_icon_class(cls): + return 'grid' + + @classmethod + def is_enabled(cls): + return True + + @classmethod + def get_add_url(cls): + return reverse('csvdatasource-add') + + def get_absolute_url(self): + return reverse('csvdatasource-detail', kwargs={'slug': self.slug}) + + def get_data(self, filter_criteria=None): + + def filter_row(row, titles, filter_criteria): + if 'text' in titles: + col = titles.index('text') + if filter_criteria not in unicode(row[col], 'utf-8'): + return False + return True + + content = self.csv_file.read() + if not content: + return None + + dialect = csv.Sniffer().sniff(content[:1024]) + reader = csv.reader(content.splitlines(), dialect) + + # skip header + reader.next() + + titles = self.columns_titles.split(',') + indexes = [titles.index(t) for t in titles if t.split()] + caption = [titles[i] for i in indexes] + data = [] + + for row in reader: + if filter_criteria and not filter_row(row, titles, filter_criteria): + continue + try: + line = [row[i] for i in indexes] + data.append(dict(zip(caption, line))) + except IndexError: + continue + + return data diff --git a/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html b/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html new file mode 100644 index 0000000..a3d0b10 --- /dev/null +++ b/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html @@ -0,0 +1,61 @@ +{% extends "passerelle/base.html" %} +{% load i18n passerelle %} + +{% block more-user-links %} +{{ block.super }} +{% if object.id %} +{{ object.title }} +{% endif %} +{% endblock %} + + +{% block appbar %} +

CSV - {{ object.title }}

+ +{% if perms.csvdatasource.change_csvdatasource %} +{% trans 'edit' %} +{% endif %} + +{% if perms.csvdatasource.delete_csvdatasource %} +{% trans 'delete' %} +{% endif %} +{% endblock %} + +{% block content %} +

+ {% if object.local_csv_file.name %} {% blocktrans with file=object.local_csv_file.name %}File: {{ file }} {% endblocktrans %} + {% elif object.remote_csv_file %} + {% blocktrans with address=object.remote_csv_file %}URL: + {{ address }} + {% endblocktrans %} + {% endif %} +

+ +
+

{% trans 'Endpoints' %}

+ +
+ + +{% if perms.base.view_accessright %} +
+

{% trans "Security" %}

+ +

+ {% trans 'Access is limited to the following API users:' %} +

+ +{% access_rights_table resource=object permission='can_access' %} +{% endif %} + +
+ +{% endblock %} + diff --git a/passerelle/apps/csvdatasource/urls.py b/passerelle/apps/csvdatasource/urls.py new file mode 100644 index 0000000..1bcd8cf --- /dev/null +++ b/passerelle/apps/csvdatasource/urls.py @@ -0,0 +1,27 @@ +from django.conf.urls import patterns, include, url +from django.contrib.auth.decorators import login_required + +from passerelle.urls_utils import decorated_includes, required, app_enabled + +from views import * + +public_urlpatterns = patterns('', + url(r'^(?P[\w,-]+)/$', CsvDataSourceDetailView.as_view(), name='csvdatasource-detail'), + url(r'^(?P[\w,-]+)/data$', CsvDataView.as_view(), name='csvdatasource-data'), +) + +management_urlpatterns = patterns('', + url(r'^add$', CsvDataSourceCreateView.as_view(), name='csvdatasource-add'), + url(r'^(?P[\w,-]+)/edit$', CsvDataSourceUpdateView.as_view(), name='csvdatasource-edit'), + url(r'^(?P[\w,-]+)/delete$', CsvDataSourceDeleteView.as_view(), name='csvdatasource-delete'), +) + + +urlpatterns = required( + app_enabled('csvdatasource'), + patterns('', + url(r'^csvdatasource/', include(public_urlpatterns)), + url(r'^manage/csvdatasource/', + decorated_includes(login_required, include(management_urlpatterns))), + ) +) diff --git a/passerelle/apps/csvdatasource/views.py b/passerelle/apps/csvdatasource/views.py new file mode 100644 index 0000000..9d4a083 --- /dev/null +++ b/passerelle/apps/csvdatasource/views.py @@ -0,0 +1,45 @@ +import json + +from django.core.urlresolvers import reverse +from django.views.generic.edit import CreateView, UpdateView, DeleteView +from django.views.generic.detail import DetailView, SingleObjectMixin +from django.views.generic.base import View + +from passerelle import utils + +from .models import CsvDataSource +from .forms import CsvDataSourceForm + + +class CsvDataSourceDetailView(DetailView): + model = CsvDataSource + + +class CsvDataSourceCreateView(CreateView): + model = CsvDataSource + template_name = 'passerelle/manage/service_form.html' + form_class = CsvDataSourceForm + + +class CsvDataSourceUpdateView(UpdateView): + model = CsvDataSource + template_name = 'passerelle/manage/service_form.html' + form_class = CsvDataSourceForm + + +class CsvDataSourceDeleteView(DeleteView): + model = CsvDataSource + template_name = 'passerelle/manage/service_confirm_delete.html' + + def get_success_url(self): + return reverse('manage-home') + + +class CsvDataView(View, SingleObjectMixin): + model = CsvDataSource + + @utils.protected_api('can_access') + @utils.to_json('api') + def get(self, request, *args, **kwargs): + obj = self.get_object() + return obj.get_data(request.GET.get('q')) diff --git a/passerelle/locale/fr/LC_MESSAGES/django.po b/passerelle/locale/fr/LC_MESSAGES/django.po index e784015..a68abdf 100644 --- a/passerelle/locale/fr/LC_MESSAGES/django.po +++ b/passerelle/locale/fr/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: Passerelle 0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2015-08-26 10:13-0500\n" +"POT-Creation-Date: 2015-08-26 10:20-0500\n" "PO-Revision-Date: 2014-07-21 08:17-0500\n" "Last-Translator: Frederic Peters \n" "Language: fr\n" @@ -38,6 +38,7 @@ msgstr "Webservice GDC" #: passerelle/apps/bdp/templates/bdp/bdp_detail.html:7 #: passerelle/apps/clicrdv/templates/clicrdv/clicrdv_detail.html:7 #: passerelle/apps/concerto/templates/concerto/concerto_detail.html:7 +#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:16 #: passerelle/apps/gdc/templates/gdc/gdc_detail.html:7 #: passerelle/apps/pastell/templates/pastell/pastell_detail.html:7 #: passerelle/apps/pastell/templates/pastell/pastell_detail.html:37 @@ -51,6 +52,7 @@ msgstr "modifier" #: passerelle/apps/bdp/templates/bdp/bdp_detail.html:10 #: passerelle/apps/clicrdv/templates/clicrdv/clicrdv_detail.html:10 #: passerelle/apps/concerto/templates/concerto/concerto_detail.html:10 +#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:20 #: passerelle/apps/gdc/templates/gdc/gdc_detail.html:10 #: passerelle/apps/pastell/templates/pastell/pastell_detail.html:10 #: passerelle/contrib/maarch/templates/passerelle/contrib/maarch/detail.html:17 @@ -64,6 +66,7 @@ msgstr "supprimer" #: passerelle/apps/choosit/templates/choosit/choosit_register_detail.html:11 #: passerelle/apps/clicrdv/templates/clicrdv/clicrdv_detail.html:21 #: passerelle/apps/concerto/templates/concerto/concerto_detail.html:17 +#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:35 #: passerelle/apps/gdc/templates/gdc/gdc_detail.html:30 #: passerelle/contrib/maarch/templates/passerelle/contrib/maarch/detail.html:24 #: passerelle/contrib/teamnet_axel/templates/passerelle/contrib/teamnet_axel/detail.html:24 @@ -93,6 +96,7 @@ msgstr "" #: passerelle/apps/choosit/templates/choosit/choosit_register_detail.html:20 #: passerelle/apps/clicrdv/templates/clicrdv/clicrdv_detail.html:42 #: passerelle/apps/concerto/templates/concerto/concerto_detail.html:53 +#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:49 #: passerelle/apps/gdc/templates/gdc/gdc_detail.html:45 #: passerelle/apps/pastell/templates/pastell/pastell_detail.html:43 #: passerelle/contrib/maarch/templates/passerelle/contrib/maarch/detail.html:43 @@ -284,6 +288,61 @@ msgstr "" msgid "Invoice update:" msgstr "" +#: passerelle/apps/csvdatasource/models.py:19 +msgid "CSV File" +msgstr "Fichier CSV" + +#: passerelle/apps/csvdatasource/models.py:20 +msgid "Remote file URL" +msgstr "Adresse du fichier distant" + +#: passerelle/apps/csvdatasource/models.py:22 +msgid "Optional column titles" +msgstr "Titres des colonnes optionnels" + +#: passerelle/apps/csvdatasource/models.py:23 +msgid "in \"[column1_title],[column2_title],...\" format" +msgstr "au format \"[titre_colonne1],[titre_colonne2],...\"" + +#: passerelle/apps/csvdatasource/models.py:25 +msgid "Cache duration in seconds" +msgstr "Durée du cache en secondes" + +#: passerelle/apps/csvdatasource/models.py:26 +msgid "applies to remote file" +msgstr "s'applique au fichier distant" + +#: passerelle/apps/csvdatasource/models.py:29 +msgid "Data Sources" +msgstr "Sources des données" + +#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:26 +#, python-format +msgid "File: %(file)s " +msgstr "Fichier: %(file)s" + +#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:28 +#, python-format +msgid "" +"URL: \n" +" %(address)s\n" +" " +msgstr "" + +#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:37 +msgid "Returning all file lines: " +msgstr "Retourne toutes les lignes du fichier :" + +#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:40 +msgid "Returning all lines containing 'abc' in 'text' column if defined : " +msgstr "Retourne toutes les ligns contenant 'abc' dans la colonne 'text', si définie :" + +#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:52 +#: passerelle/contrib/maarch/templates/passerelle/contrib/maarch/detail.html:46 +#: passerelle/contrib/teamnet_axel/templates/passerelle/contrib/teamnet_axel/detail.html:53 +msgid "Access is limited to the following API users:" +msgstr "L'accès est limité aux utilisateurs d'API suivants :" + #: passerelle/apps/gdc/models.py:11 msgid "GDC Web Service URL" msgstr "URL du webservice GDC" @@ -510,11 +569,6 @@ msgstr "" msgid "Get a contact:" msgstr "" -#: passerelle/contrib/maarch/templates/passerelle/contrib/maarch/detail.html:46 -#: passerelle/contrib/teamnet_axel/templates/passerelle/contrib/teamnet_axel/detail.html:53 -msgid "Access is limited to the following API users:" -msgstr "L'accès est limité aux utilisateurs d'API suivants :" - #: passerelle/contrib/teamnet_axel/models.py:38 msgid "Teamnet Axel WSDL URL" msgstr "" @@ -677,4 +731,4 @@ msgstr "Mot de passe incorrect; essayez à nouveau." #: passerelle/views.py:48 msgid "Connectors" -msgstr "Connecteurs" \ No newline at end of file +msgstr "Connecteurs" diff --git a/passerelle/settings.py b/passerelle/settings.py index 2e75677..4aa00db 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -108,6 +108,7 @@ INSTALLED_APPS = ( 'concerto', 'bdp', 'base_adresse', + 'csvdatasource', # backoffice templates and static 'gadjo', ) diff --git a/passerelle/static/css/style.css b/passerelle/static/css/style.css index 436cc1c..3ceb90b 100644 --- a/passerelle/static/css/style.css +++ b/passerelle/static/css/style.css @@ -32,3 +32,6 @@ li.bdp a:hover { background-image: url(icons/icon-bdp-hover.png); } li.gis a { background-image: url(icons/icon-gis.png); } li.gis a:hover { background-image: url(icons/icon-gis-hover.png); } + +li.grid a { background-image: url(icons/icon-grid.png); } +li.grid a:hover { background-image: url(icons/icon-grid-hover.png); } diff --git a/tests/test_csv_datasource.py b/tests/test_csv_datasource.py new file mode 100644 index 0000000..3bb7f7d --- /dev/null +++ b/tests/test_csv_datasource.py @@ -0,0 +1,63 @@ +import pytest +from StringIO import StringIO + +from django.core.files import File + +data = """ +121;69,981;DELANOUE;Eliot;H +525;6;DANIEL WILLIAMS;Shanone;F +253;67,742;MARTIN;Sandra;F +511;38,142;MARCHETTI;Lucie;F +235;22;MARTIN;Sandra;F +620;52,156;ARNAUD;Mathis;H +902;36;BRIGAND;Coline;F +2179;48,548;THEBAULT;Salima;F +3420;46;WILSON-LUZAYADIO;Anaelle;F +1421;55,486;WONE;Fadouma;F +2841;51;FIDJI;Zakia;F +2431;59;BELICARD;Sacha;H +4273;60;GOUBERT;Adrien;H +4049;64;MOVSESSIAN;Dimitri;H +4605;67;ABDOU BACAR;Kyle;H +4135;22,231;SAVERIAS;Marius;H +4809;75;COROLLER;Maelys;F +5427;117;KANTE;Aliou;H +7017;118;ANGELOTTI;Esther;F +116642;118;ZAHMOUM;Yaniss;H +""" + +from csvdatasource.models import CsvDataSource + +pytestmark = pytest.mark.django_db + +def test_unfiltered_data(): + csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), + columns_titles='field,,another_field,') + result = csv.get_data() + for item in result: + assert 'field' in item + assert 'another_field' in item + +def test_good_filter_data(): + filter_criteria = 'ak' + csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), + columns_titles=',id,,text,') + result = csv.get_data(filter_criteria) + assert len(result) + for item in result: + assert 'id' in item + assert 'text' in item + assert filter_criteria in item['text'] + +def test_bad_filter_data(): + filter_criteria = 'bad' + csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), + columns_titles=',id,,text,') + result = csv.get_data(filter_criteria) + assert len(result) == 0 + +def test_useless_filter_data(): + csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), + columns_titles='id,,nom, prenom, sexe') + result = csv.get_data('Ali') + assert len(result) == 20 -- 2.5.1