0002-implement-JSON-import-export-fixes-13887.patch
passerelle/apps/choosit/models.py | ||
---|---|---|
196 | 196 |
reg = ChoositRegisterWS(self.url, self.key) |
197 | 197 |
ws = reg.update(subscriptions, user) |
198 | 198 |
return {"message": ws['status']} |
199 | ||
200 |
def export_json(self): |
|
201 |
raise NotImplementedError |
passerelle/apps/csvdatasource/models.py | ||
---|---|---|
83 | 83 |
return [] |
84 | 84 |
return getattr(self, attribute).strip().splitlines() |
85 | 85 | |
86 |
def export_json(self): |
|
87 |
return { |
|
88 |
'slug': self.slug, |
|
89 |
'label': self.label, |
|
90 |
'description': self.description, |
|
91 |
'filters': self.filters, |
|
92 |
'projections': self.projections, |
|
93 |
'order': self.order, |
|
94 |
'distinct': self.distinct, |
|
95 |
'structure': self.structure, |
|
96 |
} |
|
97 | ||
98 |
@classmethod |
|
99 |
def import_json(cls, d): |
|
100 |
return cls(**d) |
|
101 | ||
86 | 102 | |
87 | 103 |
class CsvDataSource(BaseResource): |
88 | 104 |
csv_file = models.FileField(_('CSV File'), upload_to='csv') |
... | ... | |
359 | 375 |
if len(data[0]) != 1: |
360 | 376 |
raise APIError('more or less than one column', data=data) |
361 | 377 |
return data[0].values()[0] |
378 | ||
379 |
def export_json(self): |
|
380 |
d = super(CsvDataSource, self).export_json() |
|
381 |
d['queries'] = [query.export_json() for query in Query.objects.filter(resource=self)] |
|
382 |
return d |
|
383 | ||
384 |
@classmethod |
|
385 |
def import_json_real(cls, d, **kwargs): |
|
386 |
queries = d.pop('queries', []) |
|
387 |
instance = super(CsvDataSource, cls).import_json_real(d, **kwargs) |
|
388 |
new = [] |
|
389 |
for query in queries: |
|
390 |
q = Query.import_json(query) |
|
391 |
q.resource = instance |
|
392 |
new.append(q) |
|
393 |
Query.objects.bulk_create(new) |
|
394 |
return instance |
passerelle/base/management/commands/export-site.py | ||
---|---|---|
1 |
import json |
|
2 |
import sys |
|
3 | ||
4 |
from django.core.management.base import BaseCommand |
|
5 | ||
6 |
from passerelle.utils import export_site |
|
7 | ||
8 | ||
9 |
class Command(BaseCommand): |
|
10 |
args = '' |
|
11 |
help = 'Export the site' |
|
12 | ||
13 |
def handle(self, *args, **options): |
|
14 |
json.dump(export_site(), sys.stdout, indent=4) |
passerelle/base/management/commands/import-site.py | ||
---|---|---|
1 |
import json |
|
2 | ||
3 |
from django.core.management.base import BaseCommand |
|
4 | ||
5 |
from passerelle.utils import import_site |
|
6 | ||
7 | ||
8 |
class Command(BaseCommand): |
|
9 |
args = '<filename>' |
|
10 |
help = 'Import an exported site' |
|
11 | ||
12 |
def handle(self, filename, **options): |
|
13 |
import_site(json.load(open(filename))) |
passerelle/base/models.py | ||
---|---|---|
1 | 1 |
import logging |
2 |
import os |
|
3 |
import base64 |
|
2 | 4 | |
5 |
from django.apps import apps |
|
3 | 6 |
from django.conf import settings |
4 | 7 |
from django.core.exceptions import ValidationError, ObjectDoesNotExist |
5 | 8 |
from django.core.urlresolvers import reverse |
6 |
from django.db import models |
|
9 |
from django.db import models, transaction
|
|
7 | 10 |
from django.db.models import Q |
8 | 11 |
from django.utils.translation import ugettext_lazy as _ |
9 | 12 |
from django.utils.text import slugify |
13 |
from django.core.files.base import ContentFile |
|
10 | 14 | |
11 | 15 |
from django.contrib.contenttypes.models import ContentType |
12 | 16 |
from django.contrib.contenttypes import fields |
13 | 17 | |
18 |
from jsonfield import JSONField |
|
19 | ||
14 | 20 |
from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager |
15 | 21 | |
16 | 22 |
import passerelle |
... | ... | |
44 | 50 |
if self.keytype and not self.key: |
45 | 51 |
raise ValidationError(_('Key can not be empty for type %s.') % self.keytype) |
46 | 52 | |
53 |
def export_json(self): |
|
54 |
return { |
|
55 |
'username': self.username, |
|
56 |
'fullname': self.fullname, |
|
57 |
'description': self.description, |
|
58 |
'keytype': self.keytype, |
|
59 |
'key': self.key, |
|
60 |
'ipsource': self.ipsource, |
|
61 |
} |
|
62 | ||
63 |
@classmethod |
|
64 |
def import_json(self, d): |
|
65 |
return self.objects.get_or_create(username=d['username'], |
|
66 |
defaults=d) |
|
67 | ||
47 | 68 | |
48 | 69 |
class TemplateVar(models.Model): |
49 | 70 |
name = models.CharField(max_length=64) |
... | ... | |
155 | 176 |
return [(x, getattr(self, x.name, None)) for x in self._meta.fields if x.name not in ( |
156 | 177 |
'id', 'title', 'slug', 'description', 'log_level', 'users')] |
157 | 178 | |
179 |
def export_json(self): |
|
180 |
d = { |
|
181 |
'resource_type': '%s.%s' % (self.__class__._meta.app_label, |
|
182 |
self.__class__._meta.model_name), |
|
183 |
'title': self.title, |
|
184 |
'slug': self.slug, |
|
185 |
'description': self.description, |
|
186 |
# FIXME: Should we dump users ? I think it's dead. |
|
187 |
'log_level': self.log_level, |
|
188 |
'access_rights': [] |
|
189 |
} |
|
190 |
resource_type = ContentType.objects.get_for_model(self) |
|
191 |
for ar in AccessRight.objects.filter(resource_type=resource_type, |
|
192 |
resource_pk=self.pk).select_related(): |
|
193 |
d['access_rights'].append({ |
|
194 |
'codename': ar.codename, |
|
195 |
'apiuser': ar.apiuser.username, |
|
196 |
}) |
|
197 |
for field, model in self.__class__._meta.get_concrete_fields_with_model(): |
|
198 |
if field.name == 'id': |
|
199 |
continue |
|
200 |
if isinstance(field, (models.TextField, models.CharField, models.SlugField, |
|
201 |
models.URLField, models.BooleanField, models.IntegerField, |
|
202 |
models.CommaSeparatedIntegerField, models.EmailField, |
|
203 |
models.IntegerField, models.PositiveIntegerField, JSONField)): |
|
204 |
d[field.name] = getattr(self, field.attname) |
|
205 |
elif isinstance(field, models.FileField): |
|
206 |
value = getattr(self, field.attname) |
|
207 |
d[field.name] = { |
|
208 |
'name': os.path.basename(value.name), |
|
209 |
'content': base64.b64encode(value.read()), |
|
210 |
} |
|
211 |
else: |
|
212 |
raise Exception('export_json: field %s of ressource class %s is unsupported' % ( |
|
213 |
field, self.__class__)) |
|
214 |
return d |
|
215 | ||
216 |
@staticmethod |
|
217 |
def import_json(d): |
|
218 |
d = d.copy() |
|
219 |
app_label, model_name = d['resource_type'].split('.') |
|
220 |
model = apps.get_model(app_label, model_name) |
|
221 |
try: |
|
222 |
return model.objects.get(slug=d['slug']) |
|
223 |
except model.DoesNotExist: |
|
224 |
pass |
|
225 |
with transaction.atomic(): |
|
226 |
# prevent semi-creation of ressources |
|
227 |
instance = model.import_json_real(d) |
|
228 |
resource_type = ContentType.objects.get_for_model(instance) |
|
229 |
# We can only connect AccessRight objects to the new Resource after its creation |
|
230 |
for ar in d['access_rights']: |
|
231 |
apiuser = ApiUser.objects.get(username=ar['apiuser']) |
|
232 |
AccessRight.objects.get_or_create( |
|
233 |
codename=ar['codename'], |
|
234 |
resource_type=resource_type, |
|
235 |
resource_pk=instance.pk, |
|
236 |
apiuser=apiuser) |
|
237 |
return instance |
|
238 | ||
239 |
@classmethod |
|
240 |
def import_json_real(cls, d, **kwargs): |
|
241 |
kwargs = { |
|
242 |
'title': d['title'], |
|
243 |
'slug': d['slug'], |
|
244 |
'description': d['description'], |
|
245 |
'log_level': d['log_level'], |
|
246 |
} |
|
247 |
kwargs.update(kwargs) |
|
248 |
instance = cls(**kwargs) |
|
249 |
for field, model in cls._meta.get_concrete_fields_with_model(): |
|
250 |
if field.name == 'id': |
|
251 |
continue |
|
252 |
if isinstance(field, (models.TextField, models.CharField, models.SlugField, |
|
253 |
models.URLField, models.BooleanField, models.IntegerField, |
|
254 |
models.CommaSeparatedIntegerField, models.EmailField, |
|
255 |
models.IntegerField, models.PositiveIntegerField, JSONField)): |
|
256 |
setattr(instance, field.attname, d[field.name]) |
|
257 |
elif isinstance(field, models.FileField): |
|
258 |
getattr(instance, field.attname).save( |
|
259 |
d[field.name]['name'], |
|
260 |
ContentFile(base64.b64decode(d[field.name]['content'])), |
|
261 |
save=False) |
|
262 |
else: |
|
263 |
raise Exception('import_json_real: field %s of ressource class ' |
|
264 |
'%s is unsupported' % (field, cls)) |
|
265 | ||
266 |
instance.save() |
|
267 |
return instance |
|
268 | ||
158 | 269 | |
159 | 270 |
class AccessRight(models.Model): |
160 | 271 |
codename = models.CharField(max_length=100, verbose_name='codename') |
passerelle/repost/models.py | ||
---|---|---|
20 | 20 |
out_data = json.loads(p.read()) |
21 | 21 |
p.close() |
22 | 22 |
return out_data |
23 | ||
24 |
def export_json(self): |
|
25 |
raise NotImplementedError |
passerelle/utils/__init__.py | ||
---|---|---|
13 | 13 |
from django.utils.decorators import available_attrs |
14 | 14 |
from django.views.generic.detail import SingleObjectMixin |
15 | 15 |
from django.contrib.contenttypes.models import ContentType |
16 |
from django.db import transaction |
|
16 | 17 | |
17 | 18 |
from passerelle.base.context_processors import template_vars |
18 |
from passerelle.base.models import ApiUser, AccessRight |
|
19 |
from passerelle.base.models import ApiUser, AccessRight, BaseResource
|
|
19 | 20 |
from passerelle.base.signature import check_query, check_url |
20 | 21 | |
21 | 22 |
from .jsonresponse import to_json |
... | ... | |
173 | 174 |
extra={'requests_response_content': content}) |
174 | 175 | |
175 | 176 |
return response |
177 | ||
178 | ||
179 |
def export_site(): |
|
180 |
d = {} |
|
181 |
d['apiusers'] = [apiuser.export_json() for apiuser in ApiUser.objects.all()] |
|
182 |
d['resources'] = resources = [] |
|
183 |
for subclass in BaseResource.__subclasses__(): |
|
184 |
if subclass._meta.abstract: |
|
185 |
continue |
|
186 |
for resource in subclass.objects.all(): |
|
187 |
try: |
|
188 |
resources.append(resource.export_json()) |
|
189 |
except NotImplementedError: |
|
190 |
break |
|
191 |
return d |
|
192 | ||
193 | ||
194 |
def import_site(d): |
|
195 |
d = d.copy() |
|
196 | ||
197 |
with transaction.atomic(): |
|
198 |
for apiuser in d['apiusers']: |
|
199 |
ApiUser.import_json(apiuser) |
|
200 | ||
201 |
for resource in d['resources']: |
|
202 |
BaseResource.import_json(resource) |
tests/test_import_export.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
import sys |
|
3 |
import json |
|
4 |
import os |
|
5 |
import pytest |
|
6 |
from StringIO import StringIO |
|
7 |
import tempfile |
|
8 | ||
9 |
from django.core.wsgi import get_wsgi_application |
|
10 |
from webtest import TestApp |
|
11 |
from django.contrib.auth.models import User |
|
12 |
from django.core.files import File |
|
13 |
from django.core.urlresolvers import reverse |
|
14 |
from django.core.management import call_command |
|
15 |
from django.contrib.contenttypes.models import ContentType |
|
16 |
from django.test import Client |
|
17 | ||
18 |
from passerelle.base.models import ApiUser, AccessRight |
|
19 |
from test_manager import login, admin_user |
|
20 |
from passerelle.utils import import_site, export_site |
|
21 | ||
22 |
data = """121;69981;DELANOUE;Eliot;H |
|
23 |
525;6;DANIEL WILLIAMS;Shanone;F |
|
24 |
253;67742;MARTIN;Sandra;F |
|
25 |
511;38142;MARCHETTI;Lucie;F |
|
26 |
235;22;MARTIN;Sandra;F |
|
27 |
620;52156;ARNAUD;Mathis;H |
|
28 |
902;36;BRIGAND;Coline;F |
|
29 |
2179;48548;THEBAULT;Salima;F |
|
30 |
3420;46;WILSON-LUZAYADIO;Anaëlle;F |
|
31 |
1421;55486;WONE;Fadouma;F |
|
32 |
2841;51;FIDJI;Zakia;F |
|
33 |
2431;59;BELICARD;Sacha;H |
|
34 |
4273;60;GOUBERT;Adrien;H |
|
35 |
4049;64;MOVSESSIAN;Dimitri;H |
|
36 |
4605;67;ABDOU BACAR;Kyle;H |
|
37 |
4135;22231;SAVERIAS;Marius;H |
|
38 |
4809;75;COROLLER;Maelys;F |
|
39 |
5427;117;KANTE;Aliou;H |
|
40 |
116642;118;ZAHMOUM;Yaniss;H |
|
41 |
216352;38;Dupont;Benoît;H |
|
42 |
""" |
|
43 | ||
44 |
data_bom = data.decode('utf-8').encode('utf-8-sig') |
|
45 | ||
46 |
from csvdatasource.models import CsvDataSource, Query |
|
47 | ||
48 |
pytestmark = pytest.mark.django_db |
|
49 | ||
50 |
TEST_BASE_DIR = os.path.join(os.path.dirname(__file__), 'data', 'csvdatasource') |
|
51 | ||
52 | ||
53 |
def get_file_content(filename): |
|
54 |
return file(os.path.join(TEST_BASE_DIR, filename)).read() |
|
55 | ||
56 | ||
57 |
@pytest.fixture |
|
58 |
def setup(): |
|
59 | ||
60 |
def maker(columns_keynames='fam,id,lname,fname,sex', filename='data.csv', sheet_name='Feuille2', |
|
61 |
data=''): |
|
62 |
api = ApiUser.objects.create(username='all', keytype='', key='') |
|
63 |
csv = CsvDataSource.objects.create(csv_file=File(StringIO(data), filename), |
|
64 |
sheet_name=sheet_name, columns_keynames=columns_keynames, |
|
65 |
slug='test', title='a title', |
|
66 |
description='a description') |
|
67 |
obj_type = ContentType.objects.get_for_model(csv) |
|
68 |
AccessRight.objects.create(codename='can_access', apiuser=api, resource_type=obj_type, |
|
69 |
resource_pk=csv.pk) |
|
70 |
url = reverse('csvdatasource-data', kwargs={'slug': csv.slug}) |
|
71 |
return csv, url |
|
72 | ||
73 |
return maker |
|
74 | ||
75 | ||
76 |
def parse_response(response): |
|
77 |
return json.loads(response.content)['data'] |
|
78 | ||
79 | ||
80 |
@pytest.fixture |
|
81 |
def client(): |
|
82 |
return Client() |
|
83 | ||
84 | ||
85 |
@pytest.fixture(params=['data.csv', 'data.ods', 'data.xls', 'data.xlsx']) |
|
86 |
def filetype(request): |
|
87 |
return request.param |
|
88 | ||
89 | ||
90 |
def test_export_csvdatasource(app, setup, filetype): |
|
91 |
def clear(): |
|
92 |
Query.objects.all().delete() |
|
93 |
CsvDataSource.objects.all().delete() |
|
94 |
ApiUser.objects.all().delete() |
|
95 | ||
96 |
csvdata, url = setup('id,whatever,nom,prenom,sexe', filename=filetype, |
|
97 |
data=get_file_content(filetype)) |
|
98 |
query = Query(slug='query-1_', resource=csvdata, structure='array') |
|
99 |
query.projections = '\n'.join(['id:int(id)', 'prenom:prenom']) |
|
100 |
query.save() |
|
101 | ||
102 |
first = export_site() |
|
103 |
clear() |
|
104 |
import_site(first) |
|
105 |
second = export_site() |
|
106 | ||
107 |
# we don't care about file names |
|
108 |
first['resources'][0]['csv_file']['name'] = 'whocares' |
|
109 |
second['resources'][0]['csv_file']['name'] = 'whocares' |
|
110 |
assert first == second |
|
111 | ||
112 |
old_stdout = sys.stdout |
|
113 |
output = sys.stdout = StringIO() |
|
114 |
call_command('export-site') |
|
115 |
sys.stdout = old_stdout |
|
116 | ||
117 |
output = output.getvalue() |
|
118 |
third = json.loads(output) |
|
119 |
third['resources'][0]['csv_file']['name'] = 'whocares' |
|
120 |
assert first == third |
|
121 | ||
122 |
clear() |
|
123 |
with tempfile.NamedTemporaryFile() as f: |
|
124 |
f.write(output) |
|
125 |
f.flush() |
|
126 |
call_command('import-site', f.name) |
|
127 |
fourth = export_site() |
|
128 |
fourth['resources'][0]['csv_file']['name'] = 'whocares' |
|
129 |
assert first == fourth |
|
0 |
- |