From 8e37c1e5c5ea936fd9028ab43cf7ae38aad95fbc Mon Sep 17 00:00:00 2001 From: Serghei Mihai Date: Wed, 26 Aug 2015 15:39:01 +0200 Subject: [PATCH] csv files datasource (#5896) --- passerelle/apps/csvdatasource/__init__.py | 27 ++++++ passerelle/apps/csvdatasource/admin.py | 3 + passerelle/apps/csvdatasource/forms.py | 16 ++++ .../csvdatasource/locale/fr/LC_MESSAGES/django.mo | Bin 0 -> 1441 bytes .../csvdatasource/locale/fr/LC_MESSAGES/django.po | 92 ++++++++++++++++++ .../apps/csvdatasource/migrations/0001_initial.py | 32 +++++++ passerelle/apps/csvdatasource/models.py | 105 +++++++++++++++++++++ .../csvdatasource/csvdatasource_detail.html | 61 ++++++++++++ passerelle/apps/csvdatasource/tests.py | 3 + passerelle/apps/csvdatasource/urls.py | 27 ++++++ passerelle/apps/csvdatasource/views.py | 45 +++++++++ passerelle/settings.py | 1 + passerelle/static/css/style.css | 3 + 13 files changed, 415 insertions(+) create mode 100644 passerelle/apps/csvdatasource/__init__.py create mode 100644 passerelle/apps/csvdatasource/admin.py create mode 100644 passerelle/apps/csvdatasource/forms.py create mode 100644 passerelle/apps/csvdatasource/locale/fr/LC_MESSAGES/django.mo create mode 100644 passerelle/apps/csvdatasource/locale/fr/LC_MESSAGES/django.po create mode 100644 passerelle/apps/csvdatasource/migrations/0001_initial.py create mode 100644 passerelle/apps/csvdatasource/models.py create mode 100644 passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html create mode 100644 passerelle/apps/csvdatasource/tests.py create mode 100644 passerelle/apps/csvdatasource/urls.py create mode 100644 passerelle/apps/csvdatasource/views.py diff --git a/passerelle/apps/csvdatasource/__init__.py b/passerelle/apps/csvdatasource/__init__.py new file mode 100644 index 0000000..6481473 --- /dev/null +++ b/passerelle/apps/csvdatasource/__init__.py @@ -0,0 +1,27 @@ +# passerelle.apps.csvdatasource +# Copyright (C) 2015 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import django.apps + + +class AppConfig(django.apps.AppConfig): + name = 'csvdatasource' + + def get_after_urls(self): + from . import urls + return urls.urlpatterns + +default_app_config = 'csvdatasource.AppConfig' diff --git a/passerelle/apps/csvdatasource/admin.py b/passerelle/apps/csvdatasource/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/passerelle/apps/csvdatasource/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/passerelle/apps/csvdatasource/forms.py b/passerelle/apps/csvdatasource/forms.py new file mode 100644 index 0000000..5b2d30a --- /dev/null +++ b/passerelle/apps/csvdatasource/forms.py @@ -0,0 +1,16 @@ +from django.utils.text import slugify +from django import forms + +from .models import CsvDataSource + + +class CsvDataSourceForm(forms.ModelForm): + + class Meta: + model = CsvDataSource + exclude = ('slug', 'users') + + def save(self, commit=True): + if not self.instance.slug: + self.instance.slug = slugify(self.instance.title) + return super(CsvDataSourceForm, self).save(commit=commit) diff --git a/passerelle/apps/csvdatasource/locale/fr/LC_MESSAGES/django.mo b/passerelle/apps/csvdatasource/locale/fr/LC_MESSAGES/django.mo new file mode 100644 index 0000000000000000000000000000000000000000..37af3b0ca2db53a04263a0d1c6b456e93ad0e496 GIT binary patch literal 1441 zcmb7@U5^_z6ow6ykA{y5kdVrkj#et!6`gDnwP-cF8|{$Hwh@!fD#(WmQ;|rc<=SSrE#JiQQ< zRfaoRXp)+&NDG}=HsFraw5UufEc4osu5IVwjwLoOy__pb3so&NIq4K+6R{|qAg`jc zNrKo_MoT>>uGI3gl3Li1{`FV=6`N_tWvy@WhiTK7({$on*Ieh3Ib|Zp78zj=qzI`& zbinDti2QKt+BVNJBdp!ATfKh-2fMrn9ph!Gq_E@)GMiFaCX{f3iA*|%lv`gePt|!) zAM4)rRoq?2tyXJ`^1>{*W2dHgBvKa)v*279i>`!p8udSmZpCySPo~51Nn;|)!nj~$ z=Q0aEt>!kE6)604JPZ0p)SVB|ZgjZY?%WL8dqMXTYVUIOycRZ2dSP{ zd@h2S;ERxw=;YR!FZm!zj+AT_X1>v&)X{JhZ_d|gwHtlhmC#7{xJ2Fll6Fgl%iE{) zf*UK`;o0ms*xTrQS-CJltkWWcB|_SJC|%=J!Fwt=MmJ##UDlyJ?C#U@=i#=d9(8v1 z8!%ydA%!8BmklcO^ochvrL}{_+PHAx(zGy&rL52zI&a{5?Z(4wCLNqYMeSrQ6vPN| z^H_ofQSGBTL%zzJN$a?F-nw`eTEIq?!x}iBl$>LguVq^!Sz)H%VK7sY6pm1H?|8u`>*8cyx62$+mlh&uQRaqLj5QZ&qKRFuBysCCNo&=tP U5f4`F>EKN~^Pzk1x_=bxFNa5l761SM literal 0 HcmV?d00001 diff --git a/passerelle/apps/csvdatasource/locale/fr/LC_MESSAGES/django.po b/passerelle/apps/csvdatasource/locale/fr/LC_MESSAGES/django.po new file mode 100644 index 0000000..a798eaf --- /dev/null +++ b/passerelle/apps/csvdatasource/locale/fr/LC_MESSAGES/django.po @@ -0,0 +1,92 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2015-08-26 08:25-0500\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" + +#: models.py:19 +msgid "CSV File" +msgstr "Fichier CSV" + +#: models.py:20 +msgid "Remote file URL" +msgstr "Adresse du fichier distant" + +#: models.py:22 +msgid "Optional column titles" +msgstr "Titres optionnels des colonnes" + +#: models.py:23 +msgid "in \"[column1_title],[column2_title],...\" format" +msgstr "au format \"[titre_colonne1],[titre_colonne2],...\"" + +#: models.py:25 +msgid "Cache duration in seconds" +msgstr "Durée du cache en secondes" + +#: models.py:26 +msgid "applies to remote file" +msgstr "s'applique au fichier distant" + +#: models.py:29 +msgid "Data Sources" +msgstr "Source des données" + +#: templates/csvdatasource/csvdatasource_detail.html:16 +msgid "edit" +msgstr "modifier" + +#: templates/csvdatasource/csvdatasource_detail.html:20 +msgid "delete" +msgstr "supprimer" + +#: templates/csvdatasource/csvdatasource_detail.html:26 +#, python-format +msgid "File: %(file)s " +msgstr "Fichier: %(file)s" + +#: templates/csvdatasource/csvdatasource_detail.html:28 +#, python-format +msgid "" +"URL: \n" +" %(address)s\n" +" " +msgstr "" +"Adresse: \n" +" %(address)s\n" +" " +#: templates/csvdatasource/csvdatasource_detail.html:35 +msgid "Endpoints" +msgstr "" + +#: templates/csvdatasource/csvdatasource_detail.html:37 +msgid "Returning all file lines: " +msgstr "Retourne toutes les lignes du fichier: " + +#: templates/csvdatasource/csvdatasource_detail.html:40 +msgid "Returning all lines containing 'abc' in 'text' column if defined : " +msgstr "" +"Retourne toutes les lignes du fichier contenant 'abc' dans la colonne 'text' " +"si définie :" + +#: templates/csvdatasource/csvdatasource_detail.html:49 +msgid "Security" +msgstr "" + +#: templates/csvdatasource/csvdatasource_detail.html:52 +msgid "Access is limited to the following API users:" +msgstr "" diff --git a/passerelle/apps/csvdatasource/migrations/0001_initial.py b/passerelle/apps/csvdatasource/migrations/0001_initial.py new file mode 100644 index 0000000..e3ed3ca --- /dev/null +++ b/passerelle/apps/csvdatasource/migrations/0001_initial.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='CsvDataSource', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('title', models.CharField(max_length=50)), + ('slug', models.SlugField()), + ('description', models.TextField()), + ('local_csv_file', models.FileField(upload_to=b'csv', verbose_name='CSV File', blank=True)), + ('remote_csv_file', models.URLField(verbose_name='Remote file URL', blank=True)), + ('columns_titles', models.CharField(help_text='in "[column1_title],[column2_title],..." format', max_length=256, verbose_name='Optional column titles', blank=True)), + ('cache_duration', models.IntegerField(default=600, help_text='applies to remote file', verbose_name='Cache duration in seconds')), + ('users', models.ManyToManyField(to='base.ApiUser', blank=True)), + ], + options={ + 'verbose_name': 'CSV File', + }, + bases=(models.Model,), + ), + ] diff --git a/passerelle/apps/csvdatasource/models.py b/passerelle/apps/csvdatasource/models.py new file mode 100644 index 0000000..8933fef --- /dev/null +++ b/passerelle/apps/csvdatasource/models.py @@ -0,0 +1,105 @@ +import re +import csv +import logging +import requests + +from django.db import models +from django.core.cache import cache +from django.utils.translation import ugettext_lazy as _ +from django.core.urlresolvers import reverse + +from passerelle.base.models import BaseResource + +logger = logging.getLogger(__name__) + +_CACHE_SENTINEL = object() + + +class CsvDataSource(BaseResource): + local_csv_file = models.FileField(_('CSV File'), upload_to='csv', blank=True) + remote_csv_file = models.URLField(_('Remote file URL'), blank=True) + columns_titles = models.CharField(max_length=256, blank=True, + verbose_name=_('Optional column titles'), + help_text=_('in "[column1_title],[column2_title],..." format')) + cache_duration = models.IntegerField(default=600, + verbose_name=_('Cache duration in seconds'), + help_text=_('applies to remote file')) + + + category = _('Data Sources') + + class Meta: + verbose_name = 'CSV File' + + @classmethod + def get_verbose_name(cls): + return cls._meta.verbose_name + + @classmethod + def get_icon_class(cls): + return 'grid' + + @classmethod + def is_enabled(cls): + return True + + @classmethod + def get_add_url(cls): + return reverse('csvdatasource-add') + + def get_absolute_url(self): + return reverse('csvdatasource-detail', kwargs={'slug': self.slug}) + + def has_cache(self): + return cache.get(self.slug, _CACHE_SENTINEL) is not _CACHE_SENTINEL + + def set_cache(self, data): + cache.set(self.slug, data, self.cache_duration) + + @property + def content(self): + if self.local_csv_file: + logger.debug('returning data from local csv file') + return self.local_csv_file.read() + + if self.remote_csv_file: + if self.has_cache(): + logger.debug('returning cache content for remote file') + return cache.get(self.slug, _CACHE_SENTINEL) + logger.debug('getting data from url %s', self.remote_csv_file) + r = requests.get(self.remote_csv_file) + if r.ok: + logger.debug('data successfully downloaded from %s', + self.remote_csv_file) + self.set_cache(r.content) + return r.content + + def get_data(self, filter_criteria=None): + + def filter_row(row, filter_criteria): + if 'text' in self.columns_titles: + col = self.columns_titles.index('text') + if filter_criteria not in unicode(row[col], 'utf-8'): + return False + return True + + data = [] + content = self.content + if not content: + return None + + dialect = csv.Sniffer().sniff(content[:1024]) + reader = csv.reader(content.splitlines(), dialect) + + if self.columns_titles: + self.columns_titles = self.columns_titles.split(',') + else: + self.columns_titles = reader.next() + + for row in reader: + if filter_criteria and not filter_row(row, filter_criteria): + continue + line = dict(zip(self.columns_titles, row)) + data.append(line) + + return data diff --git a/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html b/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html new file mode 100644 index 0000000..a3d0b10 --- /dev/null +++ b/passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html @@ -0,0 +1,61 @@ +{% extends "passerelle/base.html" %} +{% load i18n passerelle %} + +{% block more-user-links %} +{{ block.super }} +{% if object.id %} +{{ object.title }} +{% endif %} +{% endblock %} + + +{% block appbar %} +

CSV - {{ object.title }}

+ +{% if perms.csvdatasource.change_csvdatasource %} +{% trans 'edit' %} +{% endif %} + +{% if perms.csvdatasource.delete_csvdatasource %} +{% trans 'delete' %} +{% endif %} +{% endblock %} + +{% block content %} +

+ {% if object.local_csv_file.name %} {% blocktrans with file=object.local_csv_file.name %}File: {{ file }} {% endblocktrans %} + {% elif object.remote_csv_file %} + {% blocktrans with address=object.remote_csv_file %}URL: + {{ address }} + {% endblocktrans %} + {% endif %} +

+ +
+

{% trans 'Endpoints' %}

+ +
+ + +{% if perms.base.view_accessright %} +
+

{% trans "Security" %}

+ +

+ {% trans 'Access is limited to the following API users:' %} +

+ +{% access_rights_table resource=object permission='can_access' %} +{% endif %} + +
+ +{% endblock %} + diff --git a/passerelle/apps/csvdatasource/tests.py b/passerelle/apps/csvdatasource/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/passerelle/apps/csvdatasource/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/passerelle/apps/csvdatasource/urls.py b/passerelle/apps/csvdatasource/urls.py new file mode 100644 index 0000000..1bcd8cf --- /dev/null +++ b/passerelle/apps/csvdatasource/urls.py @@ -0,0 +1,27 @@ +from django.conf.urls import patterns, include, url +from django.contrib.auth.decorators import login_required + +from passerelle.urls_utils import decorated_includes, required, app_enabled + +from views import * + +public_urlpatterns = patterns('', + url(r'^(?P[\w,-]+)/$', CsvDataSourceDetailView.as_view(), name='csvdatasource-detail'), + url(r'^(?P[\w,-]+)/data$', CsvDataView.as_view(), name='csvdatasource-data'), +) + +management_urlpatterns = patterns('', + url(r'^add$', CsvDataSourceCreateView.as_view(), name='csvdatasource-add'), + url(r'^(?P[\w,-]+)/edit$', CsvDataSourceUpdateView.as_view(), name='csvdatasource-edit'), + url(r'^(?P[\w,-]+)/delete$', CsvDataSourceDeleteView.as_view(), name='csvdatasource-delete'), +) + + +urlpatterns = required( + app_enabled('csvdatasource'), + patterns('', + url(r'^csvdatasource/', include(public_urlpatterns)), + url(r'^manage/csvdatasource/', + decorated_includes(login_required, include(management_urlpatterns))), + ) +) diff --git a/passerelle/apps/csvdatasource/views.py b/passerelle/apps/csvdatasource/views.py new file mode 100644 index 0000000..9d4a083 --- /dev/null +++ b/passerelle/apps/csvdatasource/views.py @@ -0,0 +1,45 @@ +import json + +from django.core.urlresolvers import reverse +from django.views.generic.edit import CreateView, UpdateView, DeleteView +from django.views.generic.detail import DetailView, SingleObjectMixin +from django.views.generic.base import View + +from passerelle import utils + +from .models import CsvDataSource +from .forms import CsvDataSourceForm + + +class CsvDataSourceDetailView(DetailView): + model = CsvDataSource + + +class CsvDataSourceCreateView(CreateView): + model = CsvDataSource + template_name = 'passerelle/manage/service_form.html' + form_class = CsvDataSourceForm + + +class CsvDataSourceUpdateView(UpdateView): + model = CsvDataSource + template_name = 'passerelle/manage/service_form.html' + form_class = CsvDataSourceForm + + +class CsvDataSourceDeleteView(DeleteView): + model = CsvDataSource + template_name = 'passerelle/manage/service_confirm_delete.html' + + def get_success_url(self): + return reverse('manage-home') + + +class CsvDataView(View, SingleObjectMixin): + model = CsvDataSource + + @utils.protected_api('can_access') + @utils.to_json('api') + def get(self, request, *args, **kwargs): + obj = self.get_object() + return obj.get_data(request.GET.get('q')) diff --git a/passerelle/settings.py b/passerelle/settings.py index 2e75677..4aa00db 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -108,6 +108,7 @@ INSTALLED_APPS = ( 'concerto', 'bdp', 'base_adresse', + 'csvdatasource', # backoffice templates and static 'gadjo', ) diff --git a/passerelle/static/css/style.css b/passerelle/static/css/style.css index 436cc1c..3ceb90b 100644 --- a/passerelle/static/css/style.css +++ b/passerelle/static/css/style.css @@ -32,3 +32,6 @@ li.bdp a:hover { background-image: url(icons/icon-bdp-hover.png); } li.gis a { background-image: url(icons/icon-gis.png); } li.gis a:hover { background-image: url(icons/icon-gis-hover.png); } + +li.grid a { background-image: url(icons/icon-grid.png); } +li.grid a:hover { background-image: url(icons/icon-grid-hover.png); } -- 2.5.0