0001-csv-files-datasource-5896.patch
passerelle/apps/csvdatasource/__init__.py | ||
---|---|---|
1 |
# passerelle.apps.csvdatasource |
|
2 |
# Copyright (C) 2015 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software: you can redistribute it and/or modify it |
|
5 |
# under the terms of the GNU Affero General Public License as published |
|
6 |
# by the Free Software Foundation, either version 3 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU Affero General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU Affero General Public License |
|
15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
import django.apps |
|
18 | ||
19 | ||
20 |
class AppConfig(django.apps.AppConfig): |
|
21 |
name = 'csvdatasource' |
|
22 | ||
23 |
def get_after_urls(self): |
|
24 |
from . import urls |
|
25 |
return urls.urlpatterns |
|
26 | ||
27 |
default_app_config = 'csvdatasource.AppConfig' |
passerelle/apps/csvdatasource/forms.py | ||
---|---|---|
1 |
from django.utils.text import slugify |
|
2 |
from django import forms |
|
3 | ||
4 |
from .models import CsvDataSource |
|
5 | ||
6 | ||
7 |
class CsvDataSourceForm(forms.ModelForm): |
|
8 | ||
9 |
class Meta: |
|
10 |
model = CsvDataSource |
|
11 |
exclude = ('slug', 'users') |
|
12 | ||
13 |
def save(self, commit=True): |
|
14 |
if not self.instance.slug: |
|
15 |
self.instance.slug = slugify(self.instance.title) |
|
16 |
return super(CsvDataSourceForm, self).save(commit=commit) |
passerelle/apps/csvdatasource/migrations/0001_initial.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
from __future__ import unicode_literals |
|
3 | ||
4 |
from django.db import models, migrations |
|
5 | ||
6 | ||
7 |
class Migration(migrations.Migration): |
|
8 | ||
9 |
dependencies = [ |
|
10 |
('base', '0001_initial'), |
|
11 |
] |
|
12 | ||
13 |
operations = [ |
|
14 |
migrations.CreateModel( |
|
15 |
name='CsvDataSource', |
|
16 |
fields=[ |
|
17 |
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), |
|
18 |
('title', models.CharField(max_length=50)), |
|
19 |
('slug', models.SlugField()), |
|
20 |
('description', models.TextField()), |
|
21 |
('csv_file', models.FileField(upload_to=b'csv', verbose_name='CSV File')), |
|
22 |
('columns_keynames', models.CharField(help_text='ex: id,text,data1,data2', max_length=256, verbose_name='Column keynames')), |
|
23 |
('skip_header', models.BooleanField(default=False, verbose_name='Skip first line')), |
|
24 |
('users', models.ManyToManyField(to='base.ApiUser', blank=True)), |
|
25 |
], |
|
26 |
options={ |
|
27 |
'verbose_name': 'CSV File', |
|
28 |
}, |
|
29 |
bases=(models.Model,), |
|
30 |
), |
|
31 |
] |
passerelle/apps/csvdatasource/models.py | ||
---|---|---|
1 |
import re |
|
2 |
import csv |
|
3 | ||
4 |
from django.db import models |
|
5 |
from django.utils.translation import ugettext_lazy as _ |
|
6 |
from django.core.urlresolvers import reverse |
|
7 | ||
8 |
from passerelle.base.models import BaseResource |
|
9 | ||
10 | ||
11 |
class CsvDataSource(BaseResource): |
|
12 |
csv_file = models.FileField(_('CSV File'), upload_to='csv') |
|
13 |
columns_keynames = models.CharField(max_length=256, |
|
14 |
verbose_name=_('Column keynames'), |
|
15 |
help_text=_('ex: id,text,data1,data2')) |
|
16 |
skip_header = models.BooleanField(_('Skip first line'), default=False) |
|
17 | ||
18 |
category = _('Data Sources') |
|
19 | ||
20 |
class Meta: |
|
21 |
verbose_name = _('CSV File') |
|
22 | ||
23 |
@classmethod |
|
24 |
def get_verbose_name(cls): |
|
25 |
return cls._meta.verbose_name |
|
26 | ||
27 |
@classmethod |
|
28 |
def get_icon_class(cls): |
|
29 |
return 'grid' |
|
30 | ||
31 |
@classmethod |
|
32 |
def is_enabled(cls): |
|
33 |
return True |
|
34 | ||
35 |
@classmethod |
|
36 |
def get_add_url(cls): |
|
37 |
return reverse('csvdatasource-add') |
|
38 | ||
39 |
def get_absolute_url(self): |
|
40 |
return reverse('csvdatasource-detail', kwargs={'slug': self.slug}) |
|
41 | ||
42 |
def get_data(self, filter_criteria=None): |
|
43 | ||
44 |
def filter_row(row, titles, filter_criteria): |
|
45 |
if 'text' in titles: |
|
46 |
col = titles.index('text') |
|
47 |
if filter_criteria not in unicode(row[col], 'utf-8'): |
|
48 |
return False |
|
49 |
return True |
|
50 | ||
51 |
content = self.csv_file.read() |
|
52 |
if not content: |
|
53 |
return None |
|
54 | ||
55 |
dialect = csv.Sniffer().sniff(content[:1024]) |
|
56 |
reader = csv.reader(content.splitlines(), dialect) |
|
57 | ||
58 |
if self.skip_header: |
|
59 |
reader.next() |
|
60 | ||
61 |
titles = [t.strip() for t in self.columns_keynames.split(',')] |
|
62 |
indexes = [titles.index(t) for t in titles if t] |
|
63 |
caption = [titles[i] for i in indexes] |
|
64 |
data = [] |
|
65 | ||
66 |
for row in reader: |
|
67 |
if filter_criteria and not filter_row(row, titles, filter_criteria): |
|
68 |
continue |
|
69 |
try: |
|
70 |
line = [row[i] for i in indexes] |
|
71 |
data.append(dict(zip(caption, line))) |
|
72 |
except IndexError: |
|
73 |
continue |
|
74 | ||
75 |
return data |
passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html | ||
---|---|---|
1 |
{% extends "passerelle/manage.html" %} |
|
2 |
{% load i18n passerelle %} |
|
3 | ||
4 |
{% block more-user-links %} |
|
5 |
{{ block.super }} |
|
6 |
{% if object.id %} |
|
7 |
<a href="{% url 'csvdatasource-detail' slug=object.slug %}">{{ object.title }}</a> |
|
8 |
{% endif %} |
|
9 |
{% endblock %} |
|
10 | ||
11 | ||
12 |
{% block appbar %} |
|
13 |
<h2>{{ object.get_verbose_name }} - {{ object.title }}</h2> |
|
14 | ||
15 |
{% if perms.csvdatasource.change_csvdatasource %} |
|
16 |
<a rel="popup" class="button" href="{% url 'csvdatasource-edit' slug=object.slug %}">{% trans 'edit' %}</a> |
|
17 |
{% endif %} |
|
18 | ||
19 |
{% if perms.csvdatasource.delete_csvdatasource %} |
|
20 |
<a rel="popup" class="button" href="{% url 'csvdatasource-delete' slug=object.slug %}">{% trans 'delete' %}</a> |
|
21 |
{% endif %} |
|
22 |
{% endblock %} |
|
23 | ||
24 |
{% block content %} |
|
25 |
<p> |
|
26 |
{% blocktrans with file=object.csv_file %}File: {{ file }} {% endblocktrans %} |
|
27 |
</p> |
|
28 | ||
29 |
<div> |
|
30 |
<h3>{% trans 'Endpoints' %}</h3> |
|
31 |
<ul> |
|
32 |
<li>{% trans "Returning all file lines: "%} |
|
33 |
<a href="{% url "csvdatasource-data" slug=object.slug %}">{% url "csvdatasource-data" slug=object.slug %}</a> |
|
34 |
</li> |
|
35 |
<li>{% trans "Returning all lines containing 'abc' in 'text' column if defined : "%} |
|
36 |
<a href="{% url "csvdatasource-data" slug=object.slug %}?q=abc">{% url "csvdatasource-data" slug=object.slug %}?q=abc</a> |
|
37 |
</li> |
|
38 |
</ul> |
|
39 |
</div> |
|
40 | ||
41 | ||
42 |
{% if perms.base.view_accessright %} |
|
43 |
<div> |
|
44 |
<h3>{% trans "Security" %}</h3> |
|
45 | ||
46 |
<p> |
|
47 |
{% trans 'Access is limited to the following API users:' %} |
|
48 |
</p> |
|
49 | ||
50 |
{% access_rights_table resource=object permission='can_access' %} |
|
51 |
{% endif %} |
|
52 | ||
53 |
</div> |
|
54 | ||
55 |
{% endblock %} |
|
56 |
passerelle/apps/csvdatasource/urls.py | ||
---|---|---|
1 |
from django.conf.urls import patterns, include, url |
|
2 |
from django.contrib.auth.decorators import login_required |
|
3 | ||
4 |
from passerelle.urls_utils import decorated_includes, required, app_enabled |
|
5 | ||
6 |
from views import * |
|
7 | ||
8 |
public_urlpatterns = patterns('', |
|
9 |
url(r'^(?P<slug>[\w,-]+)/$', CsvDataSourceDetailView.as_view(), name='csvdatasource-detail'), |
|
10 |
url(r'^(?P<slug>[\w,-]+)/data$', CsvDataView.as_view(), name='csvdatasource-data'), |
|
11 |
) |
|
12 | ||
13 |
management_urlpatterns = patterns('', |
|
14 |
url(r'^add$', CsvDataSourceCreateView.as_view(), name='csvdatasource-add'), |
|
15 |
url(r'^(?P<slug>[\w,-]+)/edit$', CsvDataSourceUpdateView.as_view(), name='csvdatasource-edit'), |
|
16 |
url(r'^(?P<slug>[\w,-]+)/delete$', CsvDataSourceDeleteView.as_view(), name='csvdatasource-delete'), |
|
17 |
) |
|
18 | ||
19 | ||
20 |
urlpatterns = required( |
|
21 |
app_enabled('csvdatasource'), |
|
22 |
patterns('', |
|
23 |
url(r'^csvdatasource/', include(public_urlpatterns)), |
|
24 |
url(r'^manage/csvdatasource/', |
|
25 |
decorated_includes(login_required, include(management_urlpatterns))), |
|
26 |
) |
|
27 |
) |
passerelle/apps/csvdatasource/views.py | ||
---|---|---|
1 |
import json |
|
2 | ||
3 |
from django.core.urlresolvers import reverse |
|
4 |
from django.views.generic.edit import CreateView, UpdateView, DeleteView |
|
5 |
from django.views.generic.detail import DetailView, SingleObjectMixin |
|
6 |
from django.views.generic.base import View |
|
7 | ||
8 |
from passerelle import utils |
|
9 | ||
10 |
from .models import CsvDataSource |
|
11 |
from .forms import CsvDataSourceForm |
|
12 | ||
13 | ||
14 |
class CsvDataSourceDetailView(DetailView): |
|
15 |
model = CsvDataSource |
|
16 | ||
17 | ||
18 |
class CsvDataSourceCreateView(CreateView): |
|
19 |
model = CsvDataSource |
|
20 |
template_name = 'passerelle/manage/service_form.html' |
|
21 |
form_class = CsvDataSourceForm |
|
22 | ||
23 | ||
24 |
class CsvDataSourceUpdateView(UpdateView): |
|
25 |
model = CsvDataSource |
|
26 |
template_name = 'passerelle/manage/service_form.html' |
|
27 |
form_class = CsvDataSourceForm |
|
28 | ||
29 | ||
30 |
class CsvDataSourceDeleteView(DeleteView): |
|
31 |
model = CsvDataSource |
|
32 |
template_name = 'passerelle/manage/service_confirm_delete.html' |
|
33 | ||
34 |
def get_success_url(self): |
|
35 |
return reverse('manage-home') |
|
36 | ||
37 | ||
38 |
class CsvDataView(View, SingleObjectMixin): |
|
39 |
model = CsvDataSource |
|
40 | ||
41 |
@utils.protected_api('can_access') |
|
42 |
@utils.to_json('api') |
|
43 |
def get(self, request, *args, **kwargs): |
|
44 |
obj = self.get_object() |
|
45 |
return obj.get_data(request.GET.get('q')) |
passerelle/settings.py | ||
---|---|---|
108 | 108 |
'concerto', |
109 | 109 |
'bdp', |
110 | 110 |
'base_adresse', |
111 |
'csvdatasource', |
|
111 | 112 |
# backoffice templates and static |
112 | 113 |
'gadjo', |
113 | 114 |
) |
passerelle/static/css/style.css | ||
---|---|---|
32 | 32 | |
33 | 33 |
li.gis a { background-image: url(icons/icon-gis.png); } |
34 | 34 |
li.gis a:hover { background-image: url(icons/icon-gis-hover.png); } |
35 | ||
36 |
li.grid a { background-image: url(icons/icon-grid.png); } |
|
37 |
li.grid a:hover { background-image: url(icons/icon-grid-hover.png); } |
tests/test_csv_datasource.py | ||
---|---|---|
1 |
import pytest |
|
2 |
from StringIO import StringIO |
|
3 | ||
4 |
from django.core.files import File |
|
5 | ||
6 |
data = """121;69981;DELANOUE;Eliot;H |
|
7 |
525;6;DANIEL WILLIAMS;Shanone;F |
|
8 |
253;67742;MARTIN;Sandra;F |
|
9 |
511;38142;MARCHETTI;Lucie;F |
|
10 |
235;22;MARTIN;Sandra;F |
|
11 |
620;52156;ARNAUD;Mathis;H |
|
12 |
902;36;BRIGAND;Coline;F |
|
13 |
2179;48548;THEBAULT;Salima;F |
|
14 |
3420;46;WILSON-LUZAYADIO;Anaelle;F |
|
15 |
1421;55486;WONE;Fadouma;F |
|
16 |
2841;51;FIDJI;Zakia;F |
|
17 |
2431;59;BELICARD;Sacha;H |
|
18 |
4273;60;GOUBERT;Adrien;H |
|
19 |
4049;64;MOVSESSIAN;Dimitri;H |
|
20 |
4605;67;ABDOU BACAR;Kyle;H |
|
21 |
4135;22231;SAVERIAS;Marius;H |
|
22 |
4809;75;COROLLER;Maelys;F |
|
23 |
5427;117;KANTE;Aliou;H |
|
24 |
7017;118;ANGELOTTI;Esther;F |
|
25 |
116642;118;ZAHMOUM;Yaniss;H |
|
26 |
""" |
|
27 | ||
28 |
from csvdatasource.models import CsvDataSource |
|
29 | ||
30 |
pytestmark = pytest.mark.django_db |
|
31 | ||
32 |
def test_unfiltered_data(): |
|
33 |
csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), |
|
34 |
columns_keynames='field,,another_field,') |
|
35 |
result = csv.get_data() |
|
36 |
for item in result: |
|
37 |
assert 'field' in item |
|
38 |
assert 'another_field' in item |
|
39 | ||
40 |
def test_good_filter_data(): |
|
41 |
filter_criteria = 'ak' |
|
42 |
csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), |
|
43 |
columns_keynames=',id,,text,') |
|
44 |
result = csv.get_data(filter_criteria) |
|
45 |
assert len(result) |
|
46 |
for item in result: |
|
47 |
assert 'id' in item |
|
48 |
assert 'text' in item |
|
49 |
assert filter_criteria in item['text'] |
|
50 | ||
51 |
def test_bad_filter_data(): |
|
52 |
filter_criteria = 'bad' |
|
53 |
csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), |
|
54 |
columns_keynames=',id,,text,') |
|
55 |
result = csv.get_data(filter_criteria) |
|
56 |
assert len(result) == 0 |
|
57 | ||
58 |
def test_useless_filter_data(): |
|
59 |
csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), |
|
60 |
columns_keynames='id,,nom,prenom,sexe') |
|
61 |
result = csv.get_data('Ali') |
|
62 |
assert len(result) == 20 |
|
63 | ||
64 |
def test_columns_keynames_with_spaces(): |
|
65 |
csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), |
|
66 |
columns_keynames='id , , nom,text , ') |
|
67 |
result = csv.get_data('Yaniss') |
|
68 |
assert len(result) == 1 |
|
69 | ||
70 |
def test_skipped_header_data(): |
|
71 |
csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), |
|
72 |
columns_keynames=',id,,text,', |
|
73 |
skip_header=True) |
|
74 |
result = csv.get_data('Eliot') |
|
75 |
assert len(result) == 0 |
|
76 | ||
77 |
def test_data(): |
|
78 |
csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'), |
|
79 |
columns_keynames='fam,id,, text,sexe ') |
|
80 |
result = csv.get_data('Sacha') |
|
81 |
assert result[0] == {'id': '59', 'text': 'Sacha', |
|
82 |
'fam': '2431', 'sexe': 'H'} |
|
0 |
- |