Produits Entr'ouvert » Passerelle

# passerelle.apps.csvdatasource

# Copyright (C) 2015  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import django.apps

class AppConfig(django.apps.AppConfig):

    name = 'csvdatasource'

    def get_after_urls(self):

        from . import urls

        return urls.urlpatterns

default_app_config = 'csvdatasource.AppConfig'

from django.contrib import admin

# Register your models here.

from django.utils.text import slugify

from django import forms

from .models import CsvDataSource

class CsvDataSourceForm(forms.ModelForm):

    class Meta:

        model = CsvDataSource

        exclude = ('slug', 'users')

    def save(self, commit=True):

        if not self.instance.slug:

            self.instance.slug = slugify(self.instance.title)

        return super(CsvDataSourceForm, self).save(commit=commit)

# SOME DESCRIPTIVE TITLE.

# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER

# This file is distributed under the same license as the PACKAGE package.

# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.

#

#, fuzzy

msgid ""

msgstr ""

"Project-Id-Version: PACKAGE VERSION\n"

"Report-Msgid-Bugs-To: \n"

"POT-Creation-Date: 2015-08-26 08:25-0500\n"

"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"

"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"

"Language-Team: LANGUAGE <LL@li.org>\n"

"Language: \n"

"MIME-Version: 1.0\n"

"Content-Type: text/plain; charset=UTF-8\n"

"Content-Transfer-Encoding: 8bit\n"

"Plural-Forms: nplurals=2; plural=(n > 1);\n"

#: models.py:19

msgid "CSV File"

msgstr "Fichier CSV"

#: models.py:20

msgid "Remote file URL"

msgstr "Adresse du fichier distant"

#: models.py:22

msgid "Optional column titles"

msgstr "Titres optionnels des colonnes"

#: models.py:23

msgid "in \"[column1_title],[column2_title],...\" format"

msgstr "au format \"[titre_colonne1],[titre_colonne2],...\""

#: models.py:25

msgid "Cache duration in seconds"

msgstr "Durée du cache en secondes"

#: models.py:26

msgid "applies to remote file"

msgstr "s'applique au fichier distant"

#: models.py:29

msgid "Data Sources"

msgstr "Source des données"

#: templates/csvdatasource/csvdatasource_detail.html:16

msgid "edit"

msgstr "modifier"

#: templates/csvdatasource/csvdatasource_detail.html:20

msgid "delete"

msgstr "supprimer"

#: templates/csvdatasource/csvdatasource_detail.html:26

#, python-format

msgid "File: %(file)s "

msgstr "Fichier: %(file)s"

#: templates/csvdatasource/csvdatasource_detail.html:28

#, python-format

msgid ""

"URL: \n"

"  <a href=\"%(address)s\">%(address)s</a>\n"

"  "

msgstr ""

"Adresse: \n"

"  <a href=\"%(address)s\">%(address)s</a>\n"

"  "

#: templates/csvdatasource/csvdatasource_detail.html:35

msgid "Endpoints"

msgstr ""

#: templates/csvdatasource/csvdatasource_detail.html:37

msgid "Returning all file lines: "

msgstr "Retourne toutes les lignes du fichier: "

#: templates/csvdatasource/csvdatasource_detail.html:40

msgid "Returning all lines containing 'abc' in 'text' column if defined : "

msgstr ""

"Retourne toutes les lignes du fichier contenant 'abc' dans la colonne 'text' "

"si définie :"

#: templates/csvdatasource/csvdatasource_detail.html:49

msgid "Security"

msgstr ""

#: templates/csvdatasource/csvdatasource_detail.html:52

msgid "Access is limited to the following API users:"

msgstr ""

# -*- coding: utf-8 -*-

from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [

        ('base', '0001_initial'),

    ]

    operations = [

        migrations.CreateModel(

            name='CsvDataSource',

            fields=[

                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),

                ('title', models.CharField(max_length=50)),

                ('slug', models.SlugField()),

                ('description', models.TextField()),

                ('local_csv_file', models.FileField(upload_to=b'csv', verbose_name='CSV File', blank=True)),

                ('remote_csv_file', models.URLField(verbose_name='Remote file URL', blank=True)),

                ('columns_titles', models.CharField(help_text='in "[column1_title],[column2_title],..." format', max_length=256, verbose_name='Optional column titles', blank=True)),

                ('cache_duration', models.IntegerField(default=600, help_text='applies to remote file', verbose_name='Cache duration in seconds')),

                ('users', models.ManyToManyField(to='base.ApiUser', blank=True)),

            ],

            options={

                'verbose_name': 'CSV File',

            },

            bases=(models.Model,),

        ),

    ]

import re

import csv

import logging

import requests

from django.db import models

from django.core.cache import cache

from django.utils.translation import ugettext_lazy as _

from django.core.urlresolvers import reverse

from passerelle.base.models import BaseResource

logger = logging.getLogger(__name__)

_CACHE_SENTINEL = object()

class CsvDataSource(BaseResource):

    local_csv_file = models.FileField(_('CSV File'), upload_to='csv', blank=True)

    remote_csv_file = models.URLField(_('Remote file URL'), blank=True)

    columns_titles = models.CharField(max_length=256, blank=True,

                            verbose_name=_('Optional column titles'),

                            help_text=_('in "[column1_title],[column2_title],..." format'))

    cache_duration = models.IntegerField(default=600,

                                         verbose_name=_('Cache duration in seconds'),

                                         help_text=_('applies to remote file'))

    category = _('Data Sources')

    class Meta:

        verbose_name = 'CSV File'

    @classmethod

    def get_verbose_name(cls):

        return cls._meta.verbose_name

    @classmethod

    def get_icon_class(cls):

        return 'grid'

    @classmethod

    def is_enabled(cls):

        return True

    @classmethod

    def get_add_url(cls):

        return reverse('csvdatasource-add')

    def get_absolute_url(self):

        return reverse('csvdatasource-detail', kwargs={'slug': self.slug})

    def has_cache(self):

        return cache.get(self.slug, _CACHE_SENTINEL) is not _CACHE_SENTINEL

    def set_cache(self, data):

        cache.set(self.slug, data, self.cache_duration)

    @property

    def content(self):

        if self.local_csv_file:

            logger.debug('returning data from local csv file')

            return self.local_csv_file.read()

        if self.remote_csv_file:

            if self.has_cache():

                logger.debug('returning cache content for remote file')

                return cache.get(self.slug, _CACHE_SENTINEL)

            logger.debug('getting data from url %s', self.remote_csv_file)

            r = requests.get(self.remote_csv_file)

            if r.ok:

                logger.debug('data successfully downloaded from %s',

                            self.remote_csv_file)

                self.set_cache(r.content)

                return r.content

    def get_data(self, filter_criteria=None):

        def filter_row(row, filter_criteria):

            if 'text' in self.columns_titles:

                col = self.columns_titles.index('text')

                if filter_criteria not in unicode(row[col], 'utf-8'):

                    return False

            return True

        data = []

        content = self.content

        if not content:

            return None

        dialect = csv.Sniffer().sniff(content[:1024])

        reader = csv.reader(content.splitlines(), dialect)

        if self.columns_titles:

            self.columns_titles = self.columns_titles.split(',')

        else:

            self.columns_titles = reader.next()

        for row in reader:

            if filter_criteria and not filter_row(row, filter_criteria):

                continue

            line = dict(zip(self.columns_titles, row))

            data.append(line)

        return data

{% extends "passerelle/base.html" %}

{% load i18n passerelle %}

{% block more-user-links %}

{{ block.super }}

{% if object.id %}

<a href="{% url 'csvdatasource-detail' slug=object.slug %}">{{ object.title }}</a>

{% endif %}

{% endblock %}

{% block appbar %}

<h2>CSV - {{ object.title }}</h2>

{% if perms.csvdatasource.change_csvdatasource %}

<a rel="popup" class="button" href="{% url 'csvdatasource-edit' slug=object.slug %}">{% trans 'edit' %}</a>

{% endif %}

{% if perms.csvdatasource.delete_csvdatasource %}

<a rel="popup" class="button" href="{% url 'csvdatasource-delete' slug=object.slug %}">{% trans 'delete' %}</a>

{% endif %}

{% endblock %}

{% block content %}

<p>

  {% if object.local_csv_file.name %} {% blocktrans with file=object.local_csv_file.name %}File: {{ file }} {% endblocktrans %}

  {% elif object.remote_csv_file %}

  {% blocktrans with address=object.remote_csv_file %}URL: 

  <a href="{{ address }}">{{ address }}</a>

  {% endblocktrans %}

  {% endif %}

</p>

<div>

<h3>{% trans 'Endpoints' %}</h3>

<ul>

  <li>{% trans "Returning all file lines: "%}

    <a href="{% url "csvdatasource-data" slug=object.slug %}">{%  url "csvdatasource-data" slug=object.slug %}</a>

  </li>

  <li>{% trans "Returning all lines containing 'abc' in 'text' column if defined : "%}

    <a href="{% url "csvdatasource-data" slug=object.slug %}?q=abc">{%  url "csvdatasource-data" slug=object.slug %}?q=abc</a>

  </li>

</ul>

</div>

{% if perms.base.view_accessright %}

<div>

<h3>{% trans "Security" %}</h3>

<p>

  {% trans 'Access is limited to the following API users:' %}

</p>

{% access_rights_table resource=object permission='can_access' %}

{% endif %}

</div>

{% endblock %}

from django.test import TestCase

# Create your tests here.

from django.conf.urls import patterns, include, url

from django.contrib.auth.decorators import login_required

from passerelle.urls_utils import decorated_includes, required, app_enabled

from views import *

public_urlpatterns = patterns('',

    url(r'^(?P<slug>[\w,-]+)/$', CsvDataSourceDetailView.as_view(), name='csvdatasource-detail'),

    url(r'^(?P<slug>[\w,-]+)/data$', CsvDataView.as_view(), name='csvdatasource-data'),

)

management_urlpatterns = patterns('',

    url(r'^add$', CsvDataSourceCreateView.as_view(), name='csvdatasource-add'),

    url(r'^(?P<slug>[\w,-]+)/edit$', CsvDataSourceUpdateView.as_view(), name='csvdatasource-edit'),

    url(r'^(?P<slug>[\w,-]+)/delete$', CsvDataSourceDeleteView.as_view(), name='csvdatasource-delete'),

)

urlpatterns = required(

    app_enabled('csvdatasource'),

    patterns('',

        url(r'^csvdatasource/', include(public_urlpatterns)),

        url(r'^manage/csvdatasource/',

            decorated_includes(login_required, include(management_urlpatterns))),

    )

)

import json

from django.core.urlresolvers import reverse

from django.views.generic.edit import CreateView, UpdateView, DeleteView

from django.views.generic.detail import DetailView, SingleObjectMixin

from django.views.generic.base import View

from passerelle import utils

from .models import CsvDataSource

from .forms import CsvDataSourceForm

class CsvDataSourceDetailView(DetailView):

    model = CsvDataSource

class CsvDataSourceCreateView(CreateView):

    model = CsvDataSource

    template_name = 'passerelle/manage/service_form.html'

    form_class = CsvDataSourceForm

class CsvDataSourceUpdateView(UpdateView):

    model = CsvDataSource

    template_name = 'passerelle/manage/service_form.html'

    form_class = CsvDataSourceForm

class CsvDataSourceDeleteView(DeleteView):

    model = CsvDataSource

    template_name = 'passerelle/manage/service_confirm_delete.html'

    def get_success_url(self):

        return reverse('manage-home')

class CsvDataView(View, SingleObjectMixin):

    model = CsvDataSource

    @utils.protected_api('can_access')

    @utils.to_json('api')

    def get(self, request, *args, **kwargs):

        obj = self.get_object()

        return obj.get_data(request.GET.get('q'))

    'csvdatasource',

li.grid a { background-image: url(icons/icon-grid.png); }

li.grid a:hover { background-image: url(icons/icon-grid-hover.png); }