0001-csv-files-data-source-5896.patch
passerelle/contrib/csvdatasource/__init__.py | ||
---|---|---|
1 |
# passerelle.contrib.csvdatasource |
|
2 |
# Copyright (C) 2015 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software: you can redistribute it and/or modify it |
|
5 |
# under the terms of the GNU Affero General Public License as published |
|
6 |
# by the Free Software Foundation, either version 3 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU Affero General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU Affero General Public License |
|
15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
import django.apps |
|
18 | ||
19 |
class AppConfig(django.apps.AppConfig): |
|
20 |
name = 'passerelle.contrib.csvdatasource' |
|
21 |
label = 'csvdatasource' |
|
22 | ||
23 |
def get_after_urls(self): |
|
24 |
from . import urls |
|
25 |
return urls.urlpatterns |
|
26 | ||
27 |
default_app_config = 'passerelle.contrib.csvdatasource.AppConfig' |
passerelle/contrib/csvdatasource/admin.py | ||
---|---|---|
1 |
from django.contrib import admin |
|
2 | ||
3 |
# Register your models here. |
passerelle/contrib/csvdatasource/forms.py | ||
---|---|---|
1 |
from django.utils.text import slugify |
|
2 |
from django import forms |
|
3 | ||
4 |
from .models import CsvDataSource |
|
5 | ||
6 | ||
7 |
class CsvDataSourceForm(forms.ModelForm): |
|
8 | ||
9 |
class Meta: |
|
10 |
model = CsvDataSource |
|
11 |
exclude = ('slug', 'users') |
|
12 | ||
13 |
def save(self, commit=True): |
|
14 |
if not self.instance.slug: |
|
15 |
self.instance.slug = slugify(self.instance.title) |
|
16 |
return super(CsvDataSourceForm, self).save(commit=commit) |
passerelle/contrib/csvdatasource/migrations/0001_initial.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
from __future__ import unicode_literals |
|
3 | ||
4 |
from django.db import models, migrations |
|
5 | ||
6 | ||
7 |
class Migration(migrations.Migration): |
|
8 | ||
9 |
dependencies = [ |
|
10 |
('base', '0001_initial'), |
|
11 |
] |
|
12 | ||
13 |
operations = [ |
|
14 |
migrations.CreateModel( |
|
15 |
name='CsvDataSource', |
|
16 |
fields=[ |
|
17 |
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), |
|
18 |
('title', models.CharField(max_length=50)), |
|
19 |
('slug', models.SlugField()), |
|
20 |
('description', models.TextField()), |
|
21 |
('csv_file', models.FileField(upload_to=b'csv')), |
|
22 |
('key_column', models.IntegerField(default=0)), |
|
23 |
('text_column', models.IntegerField(default=1)), |
|
24 |
('cache_duration', models.IntegerField(default=600, verbose_name='Cache duration in seconds')), |
|
25 |
('users', models.ManyToManyField(to='base.ApiUser', blank=True)), |
|
26 |
], |
|
27 |
options={ |
|
28 |
'verbose_name': 'CSV File', |
|
29 |
}, |
|
30 |
bases=(models.Model,), |
|
31 |
), |
|
32 |
] |
passerelle/contrib/csvdatasource/models.py | ||
---|---|---|
1 |
import re |
|
2 |
import csv |
|
3 |
import logging |
|
4 | ||
5 |
from django.db import models |
|
6 |
from django.core.cache import cache |
|
7 |
from django.utils.translation import ugettext_lazy as _ |
|
8 |
from django.core.urlresolvers import reverse |
|
9 | ||
10 |
from passerelle.base.models import BaseResource |
|
11 | ||
12 |
logger = logging.getLogger(__name__) |
|
13 | ||
14 |
_CACHE_SENTINEL = object() |
|
15 | ||
16 |
class CsvError(Exception): |
|
17 |
pass |
|
18 | ||
19 | ||
20 |
class CsvDataSource(BaseResource): |
|
21 |
csv_file = models.FileField(upload_to='csv') |
|
22 |
key_column = models.IntegerField(default=0) |
|
23 |
text_column = models.IntegerField(default=1) |
|
24 |
cache_duration = models.IntegerField(default=600, |
|
25 |
verbose_name=_('Cache duration in seconds')) |
|
26 | ||
27 |
category = _('Data Sources') |
|
28 | ||
29 |
class Meta: |
|
30 |
verbose_name = 'CSV File' |
|
31 | ||
32 |
@classmethod |
|
33 |
def get_verbose_name(cls): |
|
34 |
return cls._meta.verbose_name |
|
35 | ||
36 |
@classmethod |
|
37 |
def get_icon_class(cls): |
|
38 |
return 'grid' |
|
39 | ||
40 |
@classmethod |
|
41 |
def is_enabled(cls): |
|
42 |
return True |
|
43 | ||
44 |
@classmethod |
|
45 |
def get_add_url(cls): |
|
46 |
return reverse('csvdatasource-add') |
|
47 | ||
48 |
def get_edit_url(self): |
|
49 |
return reverse('csvdatasource-edit', kwargs={'slug': self.slug}) |
|
50 | ||
51 |
def get_delete_url(self): |
|
52 |
return reverse('csvdatasource-delete', kwargs={'slug': self.slug}) |
|
53 | ||
54 |
def get_absolute_url(self): |
|
55 |
return reverse('csvdatasource-detail', kwargs={'slug': self.slug}) |
|
56 | ||
57 |
def has_cache(self): |
|
58 |
self.__content = cache.get(self.slug, _CACHE_SENTINEL) |
|
59 |
return self.__content is not _CACHE_SENTINEL |
|
60 | ||
61 |
def set_cache(self, data): |
|
62 |
cache.set(self.slug, data, self.cache_duration) |
|
63 | ||
64 |
@property |
|
65 |
def content(self): |
|
66 |
if self.has_cache(): |
|
67 |
return cache.get(self.slug, _CACHE_SENTINEL) |
|
68 |
self.__content = self.csv_file.read() |
|
69 |
self.set_cache(self.__content) |
|
70 |
return self.__content |
|
71 | ||
72 |
def get_data(self, id_row=None, columns=None, filter_criteria=None, |
|
73 |
captions=None): |
|
74 | ||
75 |
def get_text(row, columns): |
|
76 |
d = '' |
|
77 |
for col in columns: |
|
78 |
try: |
|
79 |
d += row[int(col)] + ' ' |
|
80 |
except ValueError: |
|
81 |
raise CsvError(_('Wrong column format: %s') % col) |
|
82 |
except IndexError: |
|
83 |
raise CsvError(_('Unknown column: %s') % col) |
|
84 |
return d.strip() |
|
85 | ||
86 |
def get_id(row, column=0): |
|
87 |
try: |
|
88 |
if column: |
|
89 |
return unicode(row[int(column)], 'utf-8') |
|
90 |
else: |
|
91 |
return unicode(row[int(self.key_column)], 'utf-8') |
|
92 |
except IndexError: |
|
93 |
raise CsvError(_('Unexisting columns for id')) |
|
94 |
except ValueError: |
|
95 |
raise CsvError(_('Wrong id index: %s') % column) |
|
96 | ||
97 |
def filter_row(row, columns, filter_criteria): |
|
98 |
for col in columns: |
|
99 |
if filter_criteria in unicode(row[int(col)], 'utf-8'): |
|
100 |
return True |
|
101 | ||
102 |
data = [] |
|
103 |
dialect = csv.Sniffer().sniff(self.content[:1024]) |
|
104 |
reader = csv.reader(self.content.splitlines(), dialect) |
|
105 | ||
106 |
for row in reader: |
|
107 |
if filter_criteria and not filter_row(row, columns, filter_criteria): |
|
108 |
continue |
|
109 |
r = {'id': get_id(row, id_row), 'text': get_text(row, columns)} |
|
110 |
if captions: |
|
111 |
for caption in captions: |
|
112 |
index, title = caption.split(':') |
|
113 |
r[title] = get_text(row, [index]) |
|
114 |
data.append(r) |
|
115 | ||
116 |
data.sort(lambda x,y: cmp(x['text'], y['text'])) |
|
117 |
return data |
passerelle/contrib/csvdatasource/templates/csvdatasource/csvdatasource_detail.html | ||
---|---|---|
1 |
{% extends "passerelle/base.html" %} |
|
2 |
{% load i18n passerelle %} |
|
3 | ||
4 | ||
5 | ||
6 |
{% block appbar %} |
|
7 |
<h2>CSV - {{ object.title }}</h2> |
|
8 | ||
9 |
{% if perms.csvdatasource.change_csvdatasource %} |
|
10 |
<a rel="popup" class="button" href="{% url 'csvdatasource-edit' slug=object.slug %}">{% trans 'edit' %}</a> |
|
11 |
{% endif %} |
|
12 | ||
13 |
{% if perms.csvdatasource.delete_csvdatasource %} |
|
14 |
<a rel="popup" class="button" href="{% url 'csvdatasource-delete' slug=object.slug %}">{% trans 'delete' %}</a> |
|
15 |
{% endif %} |
|
16 |
{% endblock %} |
|
17 | ||
18 | ||
19 | ||
20 |
{% block content %} |
|
21 |
<p> |
|
22 |
Data file : {{ object.csv_file.name }} |
|
23 |
</p> |
|
24 | ||
25 |
<div> |
|
26 |
<h3>{% trans 'Endpoints' %}</h3> |
|
27 |
<ul> |
|
28 |
<li>{% trans "Returning all lines containing 'abc': "%} |
|
29 |
<a href="{% url "csvdatasource-data" slug=object.slug %}?q=abc">{% url "csvdatasource-data" slug=object.slug %}?q=abc</a> |
|
30 |
</li> |
|
31 |
<li>{% trans "Specify 'id' and 'text' values indexes :" %} |
|
32 |
<a href="{% url "csvdatasource-data" slug=object.slug %}?q=abc&id_col=2&text_col=5">{% url "csvdatasource-data" slug=object.slug %}?q=abc&id_row=2&text_row=5</a> |
|
33 |
</li> |
|
34 |
</ul> |
|
35 |
</div> |
|
36 | ||
37 | ||
38 |
{% if perms.base.view_accessright %} |
|
39 |
<div> |
|
40 |
<h3>{% trans "Security" %}</h3> |
|
41 | ||
42 |
<p> |
|
43 |
{% trans 'Accessing the listings is open, but posting requests is limited to the following API users:' %} |
|
44 |
</p> |
|
45 | ||
46 |
{% access_rights_table resource=object permission='can_post_request' %} |
|
47 |
{% endif %} |
|
48 | ||
49 |
</div> |
|
50 | ||
51 |
{% endblock %} |
|
52 |
passerelle/contrib/csvdatasource/tests.py | ||
---|---|---|
1 |
from django.test import TestCase |
|
2 | ||
3 |
# Create your tests here. |
passerelle/contrib/csvdatasource/urls.py | ||
---|---|---|
1 |
from django.conf.urls import patterns, include, url |
|
2 |
from django.contrib.auth.decorators import login_required |
|
3 | ||
4 |
from passerelle.urls_utils import decorated_includes, required, app_enabled |
|
5 | ||
6 |
from views import * |
|
7 | ||
8 |
public_urlpatterns = patterns('', |
|
9 |
url(r'^(?P<slug>[\w,-]+)/$', CsvDataSourceDetailView.as_view(), name='csvdatasource-detail'), |
|
10 |
url(r'^(?P<slug>[\w,-]+)/data$', CsvDataView.as_view(), name='csvdatasource-data'), |
|
11 |
) |
|
12 | ||
13 |
management_urlpatterns = patterns('', |
|
14 |
url(r'^add$', CsvDataSourceCreateView.as_view(), name='csvdatasource-add'), |
|
15 |
url(r'^(?P<slug>[\w,-]+)/edit$', CsvDataSourceUpdateView.as_view(), name='csvdatasource-edit'), |
|
16 |
url(r'^(?P<slug>[\w,-]+)/delete$', CsvDataSourceDeleteView.as_view(), name='csvdatasource-delete'), |
|
17 |
) |
|
18 | ||
19 | ||
20 |
urlpatterns = required( |
|
21 |
app_enabled('passerelle.contrib.csvdatasource'), |
|
22 |
patterns('', |
|
23 |
url(r'^csvdatasource/', include(public_urlpatterns)), |
|
24 |
url(r'^manage/csvdatasource/', |
|
25 |
decorated_includes(login_required, include(management_urlpatterns))), |
|
26 |
) |
|
27 |
) |
passerelle/contrib/csvdatasource/views.py | ||
---|---|---|
1 |
import json |
|
2 | ||
3 |
from django.core.urlresolvers import reverse |
|
4 |
from django.views.generic.edit import CreateView, UpdateView, DeleteView |
|
5 |
from django.views.generic.detail import DetailView, SingleObjectMixin |
|
6 |
from django.views.generic.base import View |
|
7 | ||
8 |
from passerelle.base.views import ResourceView |
|
9 |
from passerelle.utils import to_json |
|
10 | ||
11 |
from .models import CsvDataSource |
|
12 |
from .forms import CsvDataSourceForm |
|
13 | ||
14 | ||
15 |
class CsvDataSourceDetailView(DetailView): |
|
16 |
model = CsvDataSource |
|
17 | ||
18 | ||
19 |
class CsvDataSourceCreateView(CreateView): |
|
20 |
model = CsvDataSource |
|
21 |
template_name = 'passerelle/manage/service_form.html' |
|
22 |
form_class = CsvDataSourceForm |
|
23 | ||
24 | ||
25 |
class CsvDataSourceUpdateView(UpdateView): |
|
26 |
model = CsvDataSource |
|
27 |
template_name = 'passerelle/manage/service_form.html' |
|
28 |
form_class = CsvDataSourceForm |
|
29 | ||
30 | ||
31 |
class CsvDataSourceDeleteView(DeleteView): |
|
32 |
model = CsvDataSource |
|
33 |
template_name = 'passerelle/manage/service_confirm_delete.html' |
|
34 | ||
35 |
def get_success_url(self): |
|
36 |
return reverse('manage-home') |
|
37 | ||
38 | ||
39 |
class CsvDataView(View, SingleObjectMixin): |
|
40 |
model = CsvDataSource |
|
41 | ||
42 |
@to_json('api') |
|
43 |
def get(self, request, *args, **kwargs): |
|
44 |
obj = self.get_object() |
|
45 |
filter_criteria = request.GET.get('q') |
|
46 |
id_col = request.GET.get('id_col') |
|
47 |
columns = [request.GET.get('text_col', obj.text_column)] |
|
48 |
text_cols = request.GET.get('text_cols') |
|
49 |
cols_captions = request.GET.get('cols_captions') |
|
50 |
if text_cols: |
|
51 |
text_cols = text_cols.split(',') |
|
52 |
columns = set(columns) | set(text_cols) |
|
53 |
if cols_captions: |
|
54 |
cols_captions = cols_captions.split(',') |
|
55 |
return obj.get_data(id_col, columns, filter_criteria, cols_captions) |
passerelle/settings.py | ||
---|---|---|
108 | 108 |
'concerto', |
109 | 109 |
'bdp', |
110 | 110 |
'base_adresse', |
111 |
# datasources |
|
112 |
'passerelle.contrib.csvdatasource', |
|
111 | 113 |
# backoffice templates and static |
112 | 114 |
'gadjo', |
113 | 115 |
) |
passerelle/static/css/style.css | ||
---|---|---|
32 | 32 | |
33 | 33 |
li.gis a { background-image: url(icons/icon-gis.png); } |
34 | 34 |
li.gis a:hover { background-image: url(icons/icon-gis-hover.png); } |
35 | ||
36 |
li.grid a { background-image: url(icons/icon-grid.png); } |
|
37 |
li.grid a:hover { background-image: url(icons/icon-grid-hover.png); } |
|
35 |
- |