Produits Entr'ouvert » Passerelle

# passerelle.contrib.csvdatasource

# Copyright (C) 2015  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import django.apps

class AppConfig(django.apps.AppConfig):

    name = 'csvdatasource'

    def get_after_urls(self):

        from . import urls

        return urls.urlpatterns

default_app_config = 'csvdatasource.AppConfig'

from django.contrib import admin

# Register your models here.

from django.utils.text import slugify

from django import forms

from .models import CsvDataSource

class CsvDataSourceForm(forms.ModelForm):

    class Meta:

        model = CsvDataSource

        exclude = ('slug', 'users')

    def save(self, commit=True):

        if not self.instance.slug:

            self.instance.slug = slugify(self.instance.title)

        return super(CsvDataSourceForm, self).save(commit=commit)

# SOME DESCRIPTIVE TITLE.

# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER

# This file is distributed under the same license as the PACKAGE package.

# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.

#

#, fuzzy

msgid ""

msgstr ""

"Project-Id-Version: PACKAGE VERSION\n"

"Report-Msgid-Bugs-To: \n"

"POT-Creation-Date: 2015-07-13 03:36-0500\n"

"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"

"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"

"Language-Team: LANGUAGE <LL@li.org>\n"

"Language: \n"

"MIME-Version: 1.0\n"

"Content-Type: text/plain; charset=UTF-8\n"

"Content-Transfer-Encoding: 8bit\n"

"Plural-Forms: nplurals=2; plural=(n > 1);\n"

#: models.py:23

msgid "Optional column titles"

msgstr "Titres optionnels des colonnes"

#: models.py:24

msgid "in \"[column1_title],[column2_title],...\" format"

msgstr "au format \"[titre_colonne1],[titre_colonne2],...\""

#: models.py:26

msgid "Cache duration in seconds"

msgstr "Durée du cache en secondes"

#: models.py:29

msgid "Data Sources"

msgstr "Source des données"

#: templates/csvdatasource/csvdatasource_detail.html:10

msgid "edit"

msgstr "modifier"

#: templates/csvdatasource/csvdatasource_detail.html:14

msgid "delete"

msgstr "supprimer"

#: templates/csvdatasource/csvdatasource_detail.html:26

msgid "Endpoints"

msgstr ""

#: templates/csvdatasource/csvdatasource_detail.html:28

msgid "Returning all file lines: "

msgstr "Retourne toutes les lignes du fichier: "

#: templates/csvdatasource/csvdatasource_detail.html:31

msgid "Returning all lines containing 'abc' in 'text' column if defined : "

msgstr ""

"Retourne toutes les lignes du fichier contenant 'abc' dans la colonne 'text' "

"si définie :"

#: templates/csvdatasource/csvdatasource_detail.html:40

msgid "Security"

msgstr ""

#: templates/csvdatasource/csvdatasource_detail.html:43

msgid ""

"Accessing the listings is open, but posting requests is limited to the "

"following API users:"

msgstr ""

# -*- coding: utf-8 -*-

from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [

        ('base', '0001_initial'),

    ]

    operations = [

        migrations.CreateModel(

            name='CsvDataSource',

            fields=[

                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),

                ('title', models.CharField(max_length=50)),

                ('slug', models.SlugField()),

                ('description', models.TextField()),

                ('csv_file', models.FileField(upload_to=b'csv')),

                ('columns_titles', models.CharField(help_text='in "[column1_title],[column2_title],..." format', max_length=256, verbose_name='Optional column titles', blank=True)),

                ('cache_duration', models.IntegerField(default=600, verbose_name='Cache duration in seconds')),

                ('users', models.ManyToManyField(to='base.ApiUser', blank=True)),

            ],

            options={

                'verbose_name': 'CSV File',

            },

            bases=(models.Model,),

        ),

    ]

import re

import csv

import logging

from django.db import models

from django.core.cache import cache

from django.utils.translation import ugettext_lazy as _

from django.core.urlresolvers import reverse

from passerelle.base.models import BaseResource

logger = logging.getLogger(__name__)

_CACHE_SENTINEL = object()

class CsvError(Exception):

    pass

class CsvDataSource(BaseResource):

    csv_file = models.FileField(upload_to='csv')

    columns_titles = models.CharField(max_length=256, blank=True,

                            verbose_name=_('Optional column titles'),

                            help_text=_('in "[column1_title],[column2_title],..." format'))

    cache_duration = models.IntegerField(default=600,

                            verbose_name=_('Cache duration in seconds'))

    category = _('Data Sources')

    class Meta:

        verbose_name = 'CSV File'

    @classmethod

    def get_verbose_name(cls):

        return cls._meta.verbose_name

    @classmethod

    def get_icon_class(cls):

        return 'grid'

    @classmethod

    def is_enabled(cls):

        return True

    @classmethod

    def get_add_url(cls):

        return reverse('csvdatasource-add')

    def get_edit_url(self):

        return reverse('csvdatasource-edit', kwargs={'slug': self.slug})

    def get_delete_url(self):

        return reverse('csvdatasource-delete', kwargs={'slug': self.slug})

    def get_absolute_url(self):

        return reverse('csvdatasource-detail', kwargs={'slug': self.slug})

    def has_cache(self):

        self.__content = cache.get(self.slug, _CACHE_SENTINEL)

        return self.__content is not _CACHE_SENTINEL

    def set_cache(self, data):

        cache.set(self.slug, data, self.cache_duration)

    @property

    def content(self):

        if self.has_cache():

            return cache.get(self.slug, _CACHE_SENTINEL)

        self.__content = self.csv_file.read()

        self.set_cache(self.__content)

        return self.__content

    def get_data(self, filter_criteria=None):

        def filter_row(row, filter_criteria):

            if 'text' in self.columns_titles:

                col = self.columns_titles.index('text')

                if filter_criteria not in unicode(row[col], 'utf-8'):

                    return False

            return True

        data = []

        dialect = csv.Sniffer().sniff(self.content[:1024])

        reader = csv.reader(self.content.splitlines(), dialect)

        if self.columns_titles:

            self.columns_titles = self.columns_titles.split(',')

        else:

            self.columns_titles = reader.next()

        for row in reader:

            if filter_criteria and not filter_row(row, filter_criteria):

                continue

            line = dict(zip(self.columns_titles, row))

            data.append(line)

        return data

{% extends "passerelle/base.html" %}

{% load i18n passerelle %}

{% block appbar %}

<h2>CSV - {{ object.title }}</h2>

{% if perms.csvdatasource.change_csvdatasource %}

<a rel="popup" class="button" href="{% url 'csvdatasource-edit' slug=object.slug %}">{% trans 'edit' %}</a>

{% endif %}

{% if perms.csvdatasource.delete_csvdatasource %}

<a rel="popup" class="button" href="{% url 'csvdatasource-delete' slug=object.slug %}">{% trans 'delete' %}</a>

{% endif %}

{% endblock %}

{% block content %}

<p>

Data file : {{ object.csv_file.name }}

</p>

<div>

<h3>{% trans 'Endpoints' %}</h3>

<ul>

  <li>{% trans "Returning all file lines: "%}

    <a href="{% url "csvdatasource-data" slug=object.slug %}">{%  url "csvdatasource-data" slug=object.slug %}</a>

  </li>

  <li>{% trans "Returning all lines containing 'abc' in 'text' column if defined : "%}

    <a href="{% url "csvdatasource-data" slug=object.slug %}?q=abc">{%  url "csvdatasource-data" slug=object.slug %}?q=abc</a>

  </li>

</ul>

</div>

{% if perms.base.view_accessright %}

<div>

<h3>{% trans "Security" %}</h3>

<p>

{% trans 'Accessing the listings is open, but posting requests is limited to the following API users:' %}

</p>

{% access_rights_table resource=object permission='can_post_request' %}

{% endif %}

</div>

{% endblock %}

from django.test import TestCase

# Create your tests here.

from django.conf.urls import patterns, include, url

from django.contrib.auth.decorators import login_required

from passerelle.urls_utils import decorated_includes, required, app_enabled

from views import *

public_urlpatterns = patterns('',

    url(r'^(?P<slug>[\w,-]+)/$', CsvDataSourceDetailView.as_view(), name='csvdatasource-detail'),

    url(r'^(?P<slug>[\w,-]+)/data$', CsvDataView.as_view(), name='csvdatasource-data'),

)

management_urlpatterns = patterns('',

    url(r'^add$', CsvDataSourceCreateView.as_view(), name='csvdatasource-add'),

    url(r'^(?P<slug>[\w,-]+)/edit$', CsvDataSourceUpdateView.as_view(), name='csvdatasource-edit'),

    url(r'^(?P<slug>[\w,-]+)/delete$', CsvDataSourceDeleteView.as_view(), name='csvdatasource-delete'),

)

urlpatterns = required(

    app_enabled('csvdatasource'),

    patterns('',

        url(r'^csvdatasource/', include(public_urlpatterns)),

        url(r'^manage/csvdatasource/',

            decorated_includes(login_required, include(management_urlpatterns))),

    )

)

import json

from django.core.urlresolvers import reverse

from django.views.generic.edit import CreateView, UpdateView, DeleteView

from django.views.generic.detail import DetailView, SingleObjectMixin

from django.views.generic.base import View

from passerelle.base.views import ResourceView

from passerelle.utils import to_json

from .models import CsvDataSource

from .forms import CsvDataSourceForm

class CsvDataSourceDetailView(DetailView):

    model = CsvDataSource

class CsvDataSourceCreateView(CreateView):

    model = CsvDataSource

    template_name = 'passerelle/manage/service_form.html'

    form_class = CsvDataSourceForm

class CsvDataSourceUpdateView(UpdateView):

    model = CsvDataSource

    template_name = 'passerelle/manage/service_form.html'

    form_class = CsvDataSourceForm

class CsvDataSourceDeleteView(DeleteView):

    model = CsvDataSource

    template_name = 'passerelle/manage/service_confirm_delete.html'

    def get_success_url(self):

        return reverse('manage-home')

class CsvDataView(View, SingleObjectMixin):

    model = CsvDataSource

    @to_json('api')

    def get(self, request, *args, **kwargs):

        obj = self.get_object()

        return obj.get_data(request.GET.get('q'))

    'csvdatasource',

li.grid a { background-image: url(icons/icon-grid.png); }

li.grid a:hover { background-image: url(icons/icon-grid-hover.png); }