Projet

Général

Profil

0001-csv-files-datasource-5896.patch

Serghei Mihai, 31 août 2015 12:13

Télécharger (20,1 ko)

Voir les différences:

Subject: [PATCH] csv files datasource (#5896)

 passerelle/apps/csvdatasource/__init__.py          | 27 ++++++++
 passerelle/apps/csvdatasource/forms.py             | 16 +++++
 .../apps/csvdatasource/migrations/0001_initial.py  | 30 +++++++++
 passerelle/apps/csvdatasource/models.py            | 76 ++++++++++++++++++++++
 .../csvdatasource/csvdatasource_detail.html        | 61 +++++++++++++++++
 passerelle/apps/csvdatasource/urls.py              | 27 ++++++++
 passerelle/apps/csvdatasource/views.py             | 45 +++++++++++++
 passerelle/locale/fr/LC_MESSAGES/django.po         | 68 +++++++++++++++++--
 passerelle/settings.py                             |  1 +
 passerelle/static/css/style.css                    |  3 +
 tests/test_csv_datasource.py                       | 63 ++++++++++++++++++
 11 files changed, 410 insertions(+), 7 deletions(-)
 create mode 100644 passerelle/apps/csvdatasource/__init__.py
 create mode 100644 passerelle/apps/csvdatasource/forms.py
 create mode 100644 passerelle/apps/csvdatasource/migrations/0001_initial.py
 create mode 100644 passerelle/apps/csvdatasource/models.py
 create mode 100644 passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html
 create mode 100644 passerelle/apps/csvdatasource/urls.py
 create mode 100644 passerelle/apps/csvdatasource/views.py
 create mode 100644 tests/test_csv_datasource.py
passerelle/apps/csvdatasource/__init__.py
1
# passerelle.apps.csvdatasource
2
# Copyright (C) 2015  Entr'ouvert
3
#
4
# This program is free software: you can redistribute it and/or modify it
5
# under the terms of the GNU Affero General Public License as published
6
# by the Free Software Foundation, either version 3 of the License, or
7
# (at your option) any later version.
8
#
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
# GNU Affero General Public License for more details.
13
#
14
# You should have received a copy of the GNU Affero General Public License
15
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16

  
17
import django.apps
18

  
19

  
20
class AppConfig(django.apps.AppConfig):
21
    name = 'csvdatasource'
22

  
23
    def get_after_urls(self):
24
        from . import urls
25
        return urls.urlpatterns
26

  
27
default_app_config = 'csvdatasource.AppConfig'
passerelle/apps/csvdatasource/forms.py
1
from django.utils.text import slugify
2
from django import forms
3

  
4
from .models import CsvDataSource
5

  
6

  
7
class CsvDataSourceForm(forms.ModelForm):
8

  
9
    class Meta:
10
        model = CsvDataSource
11
        exclude = ('slug', 'users')
12

  
13
    def save(self, commit=True):
14
        if not self.instance.slug:
15
            self.instance.slug = slugify(self.instance.title)
16
        return super(CsvDataSourceForm, self).save(commit=commit)
passerelle/apps/csvdatasource/migrations/0001_initial.py
1
# -*- coding: utf-8 -*-
2
from __future__ import unicode_literals
3

  
4
from django.db import models, migrations
5

  
6

  
7
class Migration(migrations.Migration):
8

  
9
    dependencies = [
10
        ('base', '0001_initial'),
11
    ]
12

  
13
    operations = [
14
        migrations.CreateModel(
15
            name='CsvDataSource',
16
            fields=[
17
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
18
                ('title', models.CharField(max_length=50)),
19
                ('slug', models.SlugField()),
20
                ('description', models.TextField()),
21
                ('csv_file', models.FileField(upload_to=b'csv', verbose_name='CSV File')),
22
                ('columns_titles', models.CharField(help_text='in "[column1_title],[column2_title],..." format', max_length=256, verbose_name='Column titles')),
23
                ('users', models.ManyToManyField(to='base.ApiUser', blank=True)),
24
            ],
25
            options={
26
                'verbose_name': 'CSV File',
27
            },
28
            bases=(models.Model,),
29
        ),
30
    ]
passerelle/apps/csvdatasource/models.py
1
import re
2
import csv
3

  
4
from django.db import models
5
from django.utils.translation import ugettext_lazy as _
6
from django.core.urlresolvers import reverse
7

  
8
from passerelle.base.models import BaseResource
9

  
10
_CACHE_SENTINEL = object()
11

  
12

  
13
class CsvDataSource(BaseResource):
14
    csv_file = models.FileField(_('CSV File'), upload_to='csv')
15
    columns_titles = models.CharField(max_length=256,
16
                            verbose_name=_('Column titles'),
17
                            help_text=_('in "[column1_title],[column2_title],..." format'))
18

  
19
    category = _('Data Sources')
20

  
21
    class Meta:
22
        verbose_name = 'CSV File'
23

  
24
    @classmethod
25
    def get_verbose_name(cls):
26
        return cls._meta.verbose_name
27

  
28
    @classmethod
29
    def get_icon_class(cls):
30
        return 'grid'
31

  
32
    @classmethod
33
    def is_enabled(cls):
34
        return True
35

  
36
    @classmethod
37
    def get_add_url(cls):
38
        return reverse('csvdatasource-add')
39

  
40
    def get_absolute_url(self):
41
        return reverse('csvdatasource-detail', kwargs={'slug': self.slug})
42

  
43
    def get_data(self, filter_criteria=None):
44

  
45
        def filter_row(row, titles, filter_criteria):
46
            if 'text' in titles:
47
                col = titles.index('text')
48
                if filter_criteria not in unicode(row[col], 'utf-8'):
49
                    return False
50
            return True
51

  
52
        content = self.csv_file.read()
53
        if not content:
54
            return None
55

  
56
        dialect = csv.Sniffer().sniff(content[:1024])
57
        reader = csv.reader(content.splitlines(), dialect)
58

  
59
        # skip header
60
        reader.next()
61

  
62
        titles = self.columns_titles.split(',')
63
        indexes = [titles.index(t) for t in titles if t.split()]
64
        caption = [titles[i] for i in indexes]
65
        data = []
66

  
67
        for row in reader:
68
            if filter_criteria and not filter_row(row, titles, filter_criteria):
69
                continue
70
            try:
71
                line = [row[i] for i in indexes]
72
                data.append(dict(zip(caption, line)))
73
            except IndexError:
74
                continue
75

  
76
        return data
passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html
1
{% extends "passerelle/base.html" %}
2
{% load i18n passerelle %}
3

  
4
{% block more-user-links %}
5
{{ block.super }}
6
{% if object.id %}
7
<a href="{% url 'csvdatasource-detail' slug=object.slug %}">{{ object.title }}</a>
8
{% endif %}
9
{% endblock %}
10

  
11

  
12
{% block appbar %}
13
<h2>CSV - {{ object.title }}</h2>
14

  
15
{% if perms.csvdatasource.change_csvdatasource %}
16
<a rel="popup" class="button" href="{% url 'csvdatasource-edit' slug=object.slug %}">{% trans 'edit' %}</a>
17
{% endif %}
18

  
19
{% if perms.csvdatasource.delete_csvdatasource %}
20
<a rel="popup" class="button" href="{% url 'csvdatasource-delete' slug=object.slug %}">{% trans 'delete' %}</a>
21
{% endif %}
22
{% endblock %}
23

  
24
{% block content %}
25
<p>
26
  {% if object.local_csv_file.name %} {% blocktrans with file=object.local_csv_file.name %}File: {{ file }} {% endblocktrans %}
27
  {% elif object.remote_csv_file %}
28
  {% blocktrans with address=object.remote_csv_file %}URL: 
29
  <a href="{{ address }}">{{ address }}</a>
30
  {% endblocktrans %}
31
  {% endif %}
32
</p>
33

  
34
<div>
35
<h3>{% trans 'Endpoints' %}</h3>
36
<ul>
37
  <li>{% trans "Returning all file lines: "%}
38
    <a href="{% url "csvdatasource-data" slug=object.slug %}">{%  url "csvdatasource-data" slug=object.slug %}</a>
39
  </li>
40
  <li>{% trans "Returning all lines containing 'abc' in 'text' column if defined : "%}
41
    <a href="{% url "csvdatasource-data" slug=object.slug %}?q=abc">{%  url "csvdatasource-data" slug=object.slug %}?q=abc</a>
42
  </li>
43
</ul>
44
</div>
45

  
46

  
47
{% if perms.base.view_accessright %}
48
<div>
49
<h3>{% trans "Security" %}</h3>
50

  
51
<p>
52
  {% trans 'Access is limited to the following API users:' %}
53
</p>
54

  
55
{% access_rights_table resource=object permission='can_access' %}
56
{% endif %}
57

  
58
</div>
59

  
60
{% endblock %}
61

  
passerelle/apps/csvdatasource/urls.py
1
from django.conf.urls import patterns, include, url
2
from django.contrib.auth.decorators import login_required
3

  
4
from passerelle.urls_utils import decorated_includes, required, app_enabled
5

  
6
from views import *
7

  
8
public_urlpatterns = patterns('',
9
    url(r'^(?P<slug>[\w,-]+)/$', CsvDataSourceDetailView.as_view(), name='csvdatasource-detail'),
10
    url(r'^(?P<slug>[\w,-]+)/data$', CsvDataView.as_view(), name='csvdatasource-data'),
11
)
12

  
13
management_urlpatterns = patterns('',
14
    url(r'^add$', CsvDataSourceCreateView.as_view(), name='csvdatasource-add'),
15
    url(r'^(?P<slug>[\w,-]+)/edit$', CsvDataSourceUpdateView.as_view(), name='csvdatasource-edit'),
16
    url(r'^(?P<slug>[\w,-]+)/delete$', CsvDataSourceDeleteView.as_view(), name='csvdatasource-delete'),
17
)
18

  
19

  
20
urlpatterns = required(
21
    app_enabled('csvdatasource'),
22
    patterns('',
23
        url(r'^csvdatasource/', include(public_urlpatterns)),
24
        url(r'^manage/csvdatasource/',
25
            decorated_includes(login_required, include(management_urlpatterns))),
26
    )
27
)
passerelle/apps/csvdatasource/views.py
1
import json
2

  
3
from django.core.urlresolvers import reverse
4
from django.views.generic.edit import CreateView, UpdateView, DeleteView
5
from django.views.generic.detail import DetailView, SingleObjectMixin
6
from django.views.generic.base import View
7

  
8
from passerelle import utils
9

  
10
from .models import CsvDataSource
11
from .forms import CsvDataSourceForm
12

  
13

  
14
class CsvDataSourceDetailView(DetailView):
15
    model = CsvDataSource
16

  
17

  
18
class CsvDataSourceCreateView(CreateView):
19
    model = CsvDataSource
20
    template_name = 'passerelle/manage/service_form.html'
21
    form_class = CsvDataSourceForm
22

  
23

  
24
class CsvDataSourceUpdateView(UpdateView):
25
    model = CsvDataSource
26
    template_name = 'passerelle/manage/service_form.html'
27
    form_class = CsvDataSourceForm
28

  
29

  
30
class CsvDataSourceDeleteView(DeleteView):
31
    model = CsvDataSource
32
    template_name = 'passerelle/manage/service_confirm_delete.html'
33

  
34
    def get_success_url(self):
35
        return reverse('manage-home')
36

  
37

  
38
class CsvDataView(View, SingleObjectMixin):
39
    model = CsvDataSource
40

  
41
    @utils.protected_api('can_access')
42
    @utils.to_json('api')
43
    def get(self, request, *args, **kwargs):
44
        obj = self.get_object()
45
        return obj.get_data(request.GET.get('q'))
passerelle/locale/fr/LC_MESSAGES/django.po
8 8
msgstr ""
9 9
"Project-Id-Version: Passerelle 0\n"
10 10
"Report-Msgid-Bugs-To: \n"
11
"POT-Creation-Date: 2015-08-26 10:13-0500\n"
11
"POT-Creation-Date: 2015-08-26 10:20-0500\n"
12 12
"PO-Revision-Date: 2014-07-21 08:17-0500\n"
13 13
"Last-Translator: Frederic Peters <fpeters@entrouvert.com>\n"
14 14
"Language: fr\n"
......
38 38
#: passerelle/apps/bdp/templates/bdp/bdp_detail.html:7
39 39
#: passerelle/apps/clicrdv/templates/clicrdv/clicrdv_detail.html:7
40 40
#: passerelle/apps/concerto/templates/concerto/concerto_detail.html:7
41
#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:16
41 42
#: passerelle/apps/gdc/templates/gdc/gdc_detail.html:7
42 43
#: passerelle/apps/pastell/templates/pastell/pastell_detail.html:7
43 44
#: passerelle/apps/pastell/templates/pastell/pastell_detail.html:37
......
51 52
#: passerelle/apps/bdp/templates/bdp/bdp_detail.html:10
52 53
#: passerelle/apps/clicrdv/templates/clicrdv/clicrdv_detail.html:10
53 54
#: passerelle/apps/concerto/templates/concerto/concerto_detail.html:10
55
#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:20
54 56
#: passerelle/apps/gdc/templates/gdc/gdc_detail.html:10
55 57
#: passerelle/apps/pastell/templates/pastell/pastell_detail.html:10
56 58
#: passerelle/contrib/maarch/templates/passerelle/contrib/maarch/detail.html:17
......
64 66
#: passerelle/apps/choosit/templates/choosit/choosit_register_detail.html:11
65 67
#: passerelle/apps/clicrdv/templates/clicrdv/clicrdv_detail.html:21
66 68
#: passerelle/apps/concerto/templates/concerto/concerto_detail.html:17
69
#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:35
67 70
#: passerelle/apps/gdc/templates/gdc/gdc_detail.html:30
68 71
#: passerelle/contrib/maarch/templates/passerelle/contrib/maarch/detail.html:24
69 72
#: passerelle/contrib/teamnet_axel/templates/passerelle/contrib/teamnet_axel/detail.html:24
......
93 96
#: passerelle/apps/choosit/templates/choosit/choosit_register_detail.html:20
94 97
#: passerelle/apps/clicrdv/templates/clicrdv/clicrdv_detail.html:42
95 98
#: passerelle/apps/concerto/templates/concerto/concerto_detail.html:53
99
#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:49
96 100
#: passerelle/apps/gdc/templates/gdc/gdc_detail.html:45
97 101
#: passerelle/apps/pastell/templates/pastell/pastell_detail.html:43
98 102
#: passerelle/contrib/maarch/templates/passerelle/contrib/maarch/detail.html:43
......
284 288
msgid "Invoice update:"
285 289
msgstr ""
286 290

  
291
#: passerelle/apps/csvdatasource/models.py:19
292
msgid "CSV File"
293
msgstr "Fichier CSV"
294

  
295
#: passerelle/apps/csvdatasource/models.py:20
296
msgid "Remote file URL"
297
msgstr "Adresse du fichier distant"
298

  
299
#: passerelle/apps/csvdatasource/models.py:22
300
msgid "Optional column titles"
301
msgstr "Titres des colonnes optionnels"
302

  
303
#: passerelle/apps/csvdatasource/models.py:23
304
msgid "in \"[column1_title],[column2_title],...\" format"
305
msgstr "au format \"[titre_colonne1],[titre_colonne2],...\""
306

  
307
#: passerelle/apps/csvdatasource/models.py:25
308
msgid "Cache duration in seconds"
309
msgstr "Durée du cache en secondes"
310

  
311
#: passerelle/apps/csvdatasource/models.py:26
312
msgid "applies to remote file"
313
msgstr "s'applique au fichier distant"
314

  
315
#: passerelle/apps/csvdatasource/models.py:29
316
msgid "Data Sources"
317
msgstr "Sources des données"
318

  
319
#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:26
320
#, python-format
321
msgid "File: %(file)s "
322
msgstr "Fichier: %(file)s"
323

  
324
#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:28
325
#, python-format
326
msgid ""
327
"URL: \n"
328
"  <a href=\"%(address)s\">%(address)s</a>\n"
329
"  "
330
msgstr ""
331

  
332
#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:37
333
msgid "Returning all file lines: "
334
msgstr "Retourne toutes les lignes du fichier :"
335

  
336
#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:40
337
msgid "Returning all lines containing 'abc' in 'text' column if defined : "
338
msgstr "Retourne toutes les ligns contenant 'abc' dans la colonne 'text', si définie :"
339

  
340
#: passerelle/apps/csvdatasource/templates/csvdatasource/csvdatasource_detail.html:52
341
#: passerelle/contrib/maarch/templates/passerelle/contrib/maarch/detail.html:46
342
#: passerelle/contrib/teamnet_axel/templates/passerelle/contrib/teamnet_axel/detail.html:53
343
msgid "Access is limited to the following API users:"
344
msgstr "L'accès est limité aux utilisateurs d'API suivants :"
345

  
287 346
#: passerelle/apps/gdc/models.py:11
288 347
msgid "GDC Web Service URL"
289 348
msgstr "URL du webservice GDC"
......
510 569
msgid "Get a contact:"
511 570
msgstr ""
512 571

  
513
#: passerelle/contrib/maarch/templates/passerelle/contrib/maarch/detail.html:46
514
#: passerelle/contrib/teamnet_axel/templates/passerelle/contrib/teamnet_axel/detail.html:53
515
msgid "Access is limited to the following API users:"
516
msgstr "L'accès est limité aux utilisateurs d'API suivants :"
517

  
518 572
#: passerelle/contrib/teamnet_axel/models.py:38
519 573
msgid "Teamnet Axel WSDL URL"
520 574
msgstr ""
......
677 731

  
678 732
#: passerelle/views.py:48
679 733
msgid "Connectors"
680
msgstr "Connecteurs"
734
msgstr "Connecteurs"
passerelle/settings.py
108 108
    'concerto',
109 109
    'bdp',
110 110
    'base_adresse',
111
    'csvdatasource',
111 112
    # backoffice templates and static
112 113
    'gadjo',
113 114
)
passerelle/static/css/style.css
32 32

  
33 33
li.gis a { background-image: url(icons/icon-gis.png); }
34 34
li.gis a:hover { background-image: url(icons/icon-gis-hover.png); }
35

  
36
li.grid a { background-image: url(icons/icon-grid.png); }
37
li.grid a:hover { background-image: url(icons/icon-grid-hover.png); }
tests/test_csv_datasource.py
1
import pytest
2
from StringIO import StringIO
3

  
4
from django.core.files import File
5

  
6
data = """
7
121;69,981;DELANOUE;Eliot;H
8
525;6;DANIEL WILLIAMS;Shanone;F
9
253;67,742;MARTIN;Sandra;F
10
511;38,142;MARCHETTI;Lucie;F
11
235;22;MARTIN;Sandra;F
12
620;52,156;ARNAUD;Mathis;H
13
902;36;BRIGAND;Coline;F
14
2179;48,548;THEBAULT;Salima;F
15
3420;46;WILSON-LUZAYADIO;Anaelle;F
16
1421;55,486;WONE;Fadouma;F
17
2841;51;FIDJI;Zakia;F
18
2431;59;BELICARD;Sacha;H
19
4273;60;GOUBERT;Adrien;H
20
4049;64;MOVSESSIAN;Dimitri;H
21
4605;67;ABDOU BACAR;Kyle;H
22
4135;22,231;SAVERIAS;Marius;H
23
4809;75;COROLLER;Maelys;F
24
5427;117;KANTE;Aliou;H
25
7017;118;ANGELOTTI;Esther;F
26
116642;118;ZAHMOUM;Yaniss;H
27
"""
28

  
29
from csvdatasource.models import CsvDataSource
30

  
31
pytestmark = pytest.mark.django_db
32

  
33
def test_unfiltered_data():
34
    csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'),
35
                                       columns_titles='field,,another_field,')
36
    result = csv.get_data()
37
    for item in result:
38
        assert 'field' in item
39
        assert 'another_field' in item
40

  
41
def test_good_filter_data():
42
    filter_criteria = 'ak'
43
    csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'),
44
                                       columns_titles=',id,,text,')
45
    result = csv.get_data(filter_criteria)
46
    assert len(result)
47
    for item in result:
48
        assert 'id' in item
49
        assert 'text' in item
50
        assert filter_criteria in item['text']
51

  
52
def test_bad_filter_data():
53
    filter_criteria = 'bad'
54
    csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'),
55
                                       columns_titles=',id,,text,')
56
    result = csv.get_data(filter_criteria)
57
    assert len(result) == 0
58

  
59
def test_useless_filter_data():
60
    csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), 'data.csv'),
61
                                       columns_titles='id,,nom, prenom, sexe')
62
    result = csv.get_data('Ali')
63
    assert len(result) == 20
0
-