Projet

Général

Profil

0003-base-add-import-export-UI-15269.patch

Valentin Deniaud, 19 décembre 2019 16:33

Télécharger (12,4 ko)

Voir les différences: 

Subject: [PATCH 3/3] base: add import/export UI (#15269)

Site export as well as connector export.

 passerelle/base/forms.py                      |  6 ++
 passerelle/base/urls.py                       |  8 +-
 passerelle/base/views.py                      | 36 ++++++++-
 passerelle/templates/passerelle/manage.html   |  5 ++
 .../passerelle/manage/service_view.html       |  4 +
 passerelle/urls.py                            |  8 +-
 passerelle/views.py                           |  8 ++
 tests/test_manager.py                         | 81 +++++++++++++++++++
 8 files changed, 149 insertions(+), 7 deletions(-)
passerelle/base/forms.py
1 1 
from django import forms
2 
from django.utils.translation import ugettext_lazy as _
2 3 

  		




  3
  4
  
    
from .models import ApiUser, AccessRight, AvailabilityParameters

  		




  4
  5
  
    

  		




  ......




  27
  28
  
    
        widgets = {

  		




  28
  29
  
    
            'notification_delays': forms.TextInput,

  		




  29
  30
  
    
        }

  		




  
  31
  
    

  		




  
  32
  
    

  		




  
  33
  
    
class ImportSiteForm(forms.Form):

  		




  
  34
  
    
    site_json = forms.FileField(label=_('Site Export File'))

  		




  
  35
  
    
    import_users = forms.BooleanField(label=_('Import users and access rights'), required=False)

  		












  

    
      passerelle/base/urls.py
    
  





  2
  2
  
    

  		




  3
  3
  
    
from .views import ApiUserCreateView, ApiUserUpdateView, ApiUserDeleteView, \

  		




  4
  4
  
    
        ApiUserListView, AccessRightDeleteView, AccessRightCreateView, \

  		




  5
  
  
    
        LoggingParametersUpdateView, ManageAvailabilityView

  		




  
  5
  
    
        LoggingParametersUpdateView, ManageAvailabilityView, ImportSiteView, \

  		




  
  6
  
    
        ExportSiteView

  		




  6
  7
  
    

  		




  7
  8
  
    
access_urlpatterns = [

  		




  8
  9
  
    
    url(r'^$', ApiUserListView.as_view(), name='apiuser-list'),

  		




  ......




  20
  21
  
    
        ManageAvailabilityView.as_view(), name='manage-availability')

  		




  21
  22
  
    

  		




  22
  23
  
    
]

  		




  
  24
  
    

  		




  
  25
  
    
import_export_urlpatterns = [

  		




  
  26
  
    
    url(r'^import$', ImportSiteView.as_view(), name='import-site'),

  		




  
  27
  
    
    url(r'^export$', ExportSiteView.as_view(), name='export-site'),

  		




  
  28
  
    
]

  		












  

    
      passerelle/base/views.py
    
  





  15
  15
  
    
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

  		




  16
  16
  
    

  		




  17
  17
  
    
import datetime

  		




  
  18
  
    
import json

  		




  18
  19
  
    

  		




  19
  20
  
    
from dateutil import parser as date_parser

  		




  20
  21
  
    

  		




  ......




  25
  26
  
    
from django.db.models import Q

  		




  26
  27
  
    
from django.forms import models as model_forms

  		




  27
  28
  
    
from django.views.generic import (

  		




  28
  
  
    
    DetailView, ListView, CreateView, UpdateView, DeleteView, FormView)

  		




  29
  
  
    
from django.http import Http404

  		




  
  29
  
    
    View, DetailView, ListView, CreateView, UpdateView, DeleteView, FormView)

  		




  
  30
  
    
from django.http import Http404, HttpResponse

  		




  30
  31
  
    
from django.utils.timezone import make_aware

  		




  31
  32
  
    
from django.utils.translation import ugettext_lazy as _

  		




  32
  33
  
    

  		




  33
  34
  
    
from .models import ApiUser, AccessRight, LoggingParameters, ResourceStatus, Job

  		




  34
  
  
    
from .forms import ApiUserForm, AccessRightForm, AvailabilityParametersForm

  		




  
  35
  
    
from .forms import ApiUserForm, AccessRightForm, AvailabilityParametersForm, ImportSiteForm

  		




  35
  36
  
    
from ..views import GenericConnectorMixin

  		




  36
  
  
    
from ..utils import get_trusted_services

  		




  
  37
  
    
from ..utils import get_trusted_services, import_site, export_site

  		




  37
  38
  
    

  		




  38
  39
  
    

  		




  39
  40
  
    
class ResourceView(DetailView):

  		




  ......




  277
  278
  
    
        except Job.DoesNotExist:

  		




  278
  279
  
    
            raise Http404()

  		




  279
  280
  
    
        return context

  		




  
  281
  
    

  		




  
  282
  
    

  		




  
  283
  
    
class ImportSiteView(FormView):

  		




  
  284
  
    
    template_name = 'passerelle/manage/import_site.html'

  		




  
  285
  
    
    form_class = ImportSiteForm

  		




  
  286
  
    

  		




  
  287
  
    
    def get_success_url(self):

  		




  
  288
  
    
        return reverse('manage-home')

  		




  
  289
  
    

  		




  
  290
  
    
    def form_valid(self, form):

  		




  
  291
  
    
        try:

  		




  
  292
  
    
            site_json = json.load(self.request.FILES['site_json'])

  		




  
  293
  
    
        except ValueError:

  		




  
  294
  
    
            form.add_error('site_json', _('File is not in the expected JSON format.'))

  		




  
  295
  
    
            return self.form_invalid(form)

  		




  
  296
  
    

  		




  
  297
  
    
        results = import_site(site_json, overwrite=True,

  		




  
  298
  
    
                              import_users=form.cleaned_data['import_users'])

  		




  
  299
  
    
        return super(ImportSiteView, self).form_valid(form)

  		




  
  300
  
    

  		




  
  301
  
    

  		




  
  302
  
    
class ExportSiteView(View):

  		




  
  303
  
    

  		




  
  304
  
    
    def get(self, request, *args, **kwargs):

  		




  
  305
  
    
        response = HttpResponse(content_type='application/json')

  		




  
  306
  
    
        json.dump(export_site(), response, indent=2)

  		




  
  307
  
    
        return response

  		












  

    
      passerelle/templates/passerelle/manage.html
    
  





  4
  4
  
    
{% block appbar %}

  		




  5
  5
  
    
  <h2>{% trans 'Web Services' %}</h2>

  		




  6
  6
  
    
  <span class="actions">

  		




  
  7
  
    
  <a class="extra-actions-menu-opener"></a>

  		




  7
  8
  
    
  <a href="{% url 'apiuser-list' %}">{% trans 'Access Management' %}</a>

  		




  8
  9
  
    
  <a href="{% url 'add-connector' %}">{% trans 'Add Connector' %}</a>

  		




  9
  10
  
    
  </span>

  		




  
  11
  
    
  <ul class="extra-actions-menu">

  		




  
  12
  
    
    <li><a href="{% url 'import-site' %}" rel="popup">{% trans 'Import' %}</a></li>

  		




  
  13
  
    
    <li><a download href="{% url 'export-site' %}">{% trans 'Export' %}</a></li>

  		




  
  14
  
    
  </ul>

  		




  10
  15
  
    
{% endblock %}

  		




  11
  16
  
    

  		




  12
  17
  
    
{% block content %}

  		












  

    
      passerelle/templates/passerelle/manage/service_view.html
    
  





  15
  15
  
    
    {% endwith %}

  		




  16
  16
  
    
</h2>

  		




  17
  17
  
    
<span class="actions">

  		




  
  18
  
    
<a class="extra-actions-menu-opener"></a>

  		




  18
  19
  
    
{% if object|can_edit:request.user and has_check_status %}

  		




  19
  20
  
    
<a rel="popup" href="{% url 'manage-availability' resource_type=object|resource_type resource_pk=object.id %}">{% trans 'availability check parameters' %}</a>

  		




  20
  21
  
    
{% endif %}

  		




  ......




  28
  29
  
    
<a rel="popup" href="{{ object.get_delete_url }}">{% trans 'delete' %}</a>

  		




  29
  30
  
    
{% endif %}

  		




  30
  31
  
    
</span>

  		




  
  32
  
    
<ul class="extra-actions-menu">

  		




  
  33
  
    
  <li><a download href="{% url 'export-connector' connector=object.get_connector_slug slug=object.slug %}">{% trans 'Export' %}</a></li>

  		




  
  34
  
    
</ul>

  		




  31
  35
  
    
{% endblock %}

  		




  32
  36
  
    

  		




  33
  37
  
    
{% block content %}

  		












  

    
      passerelle/urls.py
    
  





  9
  9
  
    
from .views import (HomePageView, ManageView, ManageAddView,

  		




  10
  10
  
    
        GenericCreateConnectorView, GenericDeleteConnectorView,

  		




  11
  11
  
    
        GenericEditConnectorView, GenericEndpointView, GenericConnectorView,

  		




  12
  
  
    
        GenericViewLogsConnectorView, GenericLogView,

  		




  
  12
  
    
        GenericViewLogsConnectorView, GenericLogView, GenericExportConnectorView,

  		




  13
  13
  
    
        login, logout, menu_json)

  		




  14
  14
  
    
from .base.views import GenericViewJobsConnectorView, GenericJobView

  		




  15
  15
  
    
from .urls_utils import decorated_includes, required, app_enabled, manager_required

  		




  16
  
  
    
from .base.urls import access_urlpatterns

  		




  
  16
  
    
from .base.urls import access_urlpatterns, import_export_urlpatterns

  		




  17
  17
  
    
from .plugins import register_apps_urls

  		




  18
  18
  
    

  		




  19
  19
  
    
from passerelle.apps.pastell import urls as pastell_urls

  		




  ......




  34
  34
  
    

  		




  35
  35
  
    
    url(r'^manage/access/',

  		




  36
  36
  
    
        decorated_includes(manager_required, include(access_urlpatterns))),

  		




  
  37
  
    
    url(r'^manage/',

  		




  
  38
  
    
        decorated_includes(manager_required, include(import_export_urlpatterns))),

  		




  37
  39
  
    
]

  		




  38
  40
  
    

  		




  39
  41
  
    
urlpatterns += required(

  		




  ......




  73
  75
  
    
                GenericViewJobsConnectorView.as_view(), name='view-jobs-connector'),

  		




  74
  76
  
    
            url(r'^(?P<slug>[\w,-]+)/jobs/(?P<job_pk>\d+)/$',

  		




  75
  77
  
    
                GenericJobView.as_view(), name='view-job'),

  		




  
  78
  
    
            url(r'^(?P<slug>[\w,-]+)/export$',

  		




  
  79
  
    
                GenericExportConnectorView.as_view(), name='export-connector'),

  		




  76
  80
  
    
        ])))

  		




  77
  81
  
    
]

  		




  78
  82
  
    

  		












  

    
      passerelle/views.py
    
  





  477
  477
  
    

  		




  478
  478
  
    
    def delete(self, request, *args, **kwargs):

  		




  479
  479
  
    
        return self.get(request, *args, **kwargs)

  		




  
  480
  
    

  		




  
  481
  
    

  		




  
  482
  
    
class GenericExportConnectorView(GenericConnectorMixin, DetailView):

  		




  
  483
  
    

  		




  
  484
  
    
    def get(self, request, *args, **kwargs):

  		




  
  485
  
    
        response = HttpResponse(content_type='application/json')

  		




  
  486
  
    
        json.dump({'resources': [self.get_object().export_json()]}, response, indent=2)

  		




  
  487
  
    
        return response

  		












  

    
      tests/test_manager.py
    
  





  2
  2
  
    
import re

  		




  3
  3
  
    
from StringIO import StringIO

  		




  4
  4
  
    

  		




  
  5
  
    
from webtest import Upload

  		




  
  6
  
    

  		




  5
  7
  
    
from django.contrib.auth.models import User

  		




  6
  8
  
    
from django.contrib.contenttypes.models import ContentType

  		




  7
  9
  
    
from django.core.files import File

  		




  ......




  336
  338
  
    
    base_url = re.findall(r'data-job-base-url="(.*)"', resp.text)[0]

  		




  337
  339
  
    
    resp = app.get(base_url + job_pk + '/')

  		




  338
  340
  
    
    resp = app.get(base_url + '12345' + '/', status=404)

  		




  
  341
  
    

  		




  
  342
  
    

  		




  
  343
  
    
def test_manager_import_export(app, admin_user):

  		




  
  344
  
    
    data = StringIO('1;Foo\n2;Bar\n3;Baz')

  		




  
  345
  
    
    csv = CsvDataSource.objects.create(csv_file=File(data, 't.csv'),

  		




  
  346
  
    
           columns_keynames='id, text', slug='test', title='a title', description='a description')

  		




  
  347
  
    
    csv2 = CsvDataSource.objects.create(csv_file=File(data, 't.csv'),

  		




  
  348
  
    
           columns_keynames='id, text', slug='test2', title='a title', description='a description')

  		




  
  349
  
    
    api = ApiUser.objects.create(username='public',

  		




  
  350
  
    
                    fullname='public',

  		




  
  351
  
    
                    description='access for all',

  		




  
  352
  
    
                    keytype='', key='')

  		




  
  353
  
    
    obj_type = ContentType.objects.get_for_model(csv)

  		




  
  354
  
    
    AccessRight.objects.create(codename='can_access',

  		




  
  355
  
    
                    apiuser=api,

  		




  
  356
  
    
                    resource_type=obj_type,

  		




  
  357
  
    
                    resource_pk=csv.pk,

  		




  
  358
  
    
    )

  		




  
  359
  
    

  		




  
  360
  
    
    # export site

  		




  
  361
  
    
    app = login(app)

  		




  
  362
  
    
    resp = app.get('/manage/')

  		




  
  363
  
    
    resp = resp.click('Export')

  		




  
  364
  
    
    assert resp.headers['content-type'] == 'application/json'

  		




  
  365
  
    
    site_export = resp.text

  		




  
  366
  
    

  		




  
  367
  
    
    # invalid json

  		




  
  368
  
    
    resp = app.get('/manage/', status=200)

  		




  
  369
  
    
    resp = resp.click('Import')

  		




  
  370
  
    
    resp.form['site_json'] = Upload('export.json', b'garbage', 'application/json')

  		




  
  371
  
    
    resp = resp.form.submit()

  		




  
  372
  
    
    assert 'File is not in the expected JSON format.' in resp.text

  		




  
  373
  
    

  		




  
  374
  
    
    # empty json

  		




  
  375
  
    
    resp = app.get('/manage/', status=200)

  		




  
  376
  
    
    resp = resp.click('Import')

  		




  
  377
  
    
    resp.form['site_json'] = Upload('export.json', b'{}', 'application/json')

  		




  
  378
  
    
    resp = resp.form.submit().follow()

  		




  
  379
  
    
    assert CsvDataSource.objects.count() == 2

  		




  
  380
  
    

  		




  
  381
  
    
    # import site

  		




  
  382
  
    
    CsvDataSource.objects.all().delete()

  		




  
  383
  
    
    resp = app.get('/manage/', status=200)

  		




  
  384
  
    
    resp = resp.click('Import')

  		




  
  385
  
    
    resp.form['site_json'] = Upload('export.json', site_export.encode('utf-8'), 'application/json')

  		




  
  386
  
    
    resp = resp.form.submit().follow()

  		




  
  387
  
    
    assert CsvDataSource.objects.count() == 2

  		




  
  388
  
    

  		




  
  389
  
    
    # export connector

  		




  
  390
  
    
    resp = app.get('/%s/%s/' % (csv.get_connector_slug(), csv.slug), status=200)

  		




  
  391
  
    
    resp = resp.click('Export')

  		




  
  392
  
    
    assert resp.headers['content-type'] == 'application/json'

  		




  
  393
  
    
    connector_export = resp.text

  		




  
  394
  
    

  		




  
  395
  
    
    # import connector

  		




  
  396
  
    
    csv.delete()

  		




  
  397
  
    
    resp = app.get('/manage/', status=200)

  		




  
  398
  
    
    resp = resp.click('Import')

  		




  
  399
  
    
    resp.form['site_json'] = Upload('export.json', connector_export.encode('utf-8'),

  		




  
  400
  
    
                                     'application/json')

  		




  
  401
  
    
    resp = resp.form.submit().follow()

  		




  
  402
  
    
    assert CsvDataSource.objects.count() == 2

  		




  
  403
  
    
    assert CsvDataSource.objects.filter(slug='test').exists()

  		




  
  404
  
    

  		




  
  405
  
    
    # import users

  		




  
  406
  
    
    ApiUser.objects.all().delete()

  		




  
  407
  
    
    AccessRight.objects.all().delete()

  		




  
  408
  
    
    resp = app.get('/manage/', status=200)

  		




  
  409
  
    
    resp = resp.click('Import')

  		




  
  410
  
    
    resp.form['site_json'] = Upload('export.json', site_export.encode('utf-8'), 'application/json')

  		




  
  411
  
    
    resp = resp.form.submit().follow()

  		




  
  412
  
    
    assert not ApiUser.objects.exists()

  		




  
  413
  
    
    assert not AccessRight.objects.exists()

  		




  
  414
  
    
    resp = resp.click('Import')

  		




  
  415
  
    
    resp.form['import_users'] = True

  		




  
  416
  
    
    resp.form['site_json'] = Upload('export.json', site_export.encode('utf-8'), 'application/json')

  		




  
  417
  
    
    resp = resp.form.submit().follow()

  		




  
  418
  
    
    assert ApiUser.objects.filter(username='public').exists()

  		




  
  419
  
    
    assert AccessRight.objects.filter(codename='can_access').exists()

  		




  339
  
  
    
-