1
|
# -*- coding: utf-8 -*-
|
2
|
#!/usr/bin/env python
|
3
|
|
4
|
import os
|
5
|
import csv
|
6
|
from collections import defaultdict
|
7
|
|
8
|
from datetime import datetime, date
|
9
|
|
10
|
import calebasse.settings
|
11
|
import django.core.management
|
12
|
|
13
|
django.core.management.setup_environ(calebasse.settings)
|
14
|
|
15
|
from django.db import transaction
|
16
|
from calebasse.agenda.models import EventWithAct
|
17
|
from calebasse.dossiers.models import PatientRecord
|
18
|
from calebasse.personnes.models import Worker
|
19
|
from calebasse.ressources.models import Service
|
20
|
from calebasse.ressources.models import ActType
|
21
|
from calebasse.actes.models import Act, ActValidationState
|
22
|
|
23
|
from import_dossiers import map_cs
|
24
|
|
25
|
# Configuration
|
26
|
db_path = "./scripts/20130104-213225"
|
27
|
|
28
|
dbs = ["F_ST_ETIENNE_SESSAD_TED", "F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD"]
|
29
|
|
30
|
def _to_datetime(str_date):
|
31
|
if not str_date:
|
32
|
return None
|
33
|
return datetime.strptime(str_date[:19], "%Y-%m-%d %H:%M:%S")
|
34
|
|
35
|
def _to_date(str_date):
|
36
|
dt = _to_datetime(str_date)
|
37
|
return dt and dt.date()
|
38
|
|
39
|
def _to_time(str_date):
|
40
|
dt = _to_datetime(str_date)
|
41
|
return dt and dt.time()
|
42
|
|
43
|
def _to_duration(str_date):
|
44
|
dt = _to_datetime(str_date)
|
45
|
if dt is None:
|
46
|
return None
|
47
|
return dt - datetime(1900, 01, 01, 0, 0)
|
48
|
|
49
|
def _to_int(str_int):
|
50
|
if not str_int:
|
51
|
return None
|
52
|
return int(str_int)
|
53
|
|
54
|
def _get_dict(cols, line):
|
55
|
""""""
|
56
|
res = {}
|
57
|
for i, data in enumerate(line):
|
58
|
res[cols[i]] = data.decode('utf-8')
|
59
|
return res
|
60
|
|
61
|
def batch_delete(qs, limit):
|
62
|
count = qs.count()
|
63
|
i = 0
|
64
|
while i < count:
|
65
|
ids = qs[i:i+limit].values_list('pk', flat=True)
|
66
|
qs.filter(pk__in=ids).delete()
|
67
|
i += limit
|
68
|
|
69
|
|
70
|
PERIOD_FAURE_NOUS = {1 : 1,
|
71
|
2 : 2,
|
72
|
3 : 3,
|
73
|
4 : 4,
|
74
|
5: 6,
|
75
|
6: 7,
|
76
|
7: 8,
|
77
|
8: 9,
|
78
|
9: None,
|
79
|
10: 10,
|
80
|
12: 11,
|
81
|
13: 12,
|
82
|
}
|
83
|
|
84
|
JOURS = {1: 'lundi',
|
85
|
2: 'mardi',
|
86
|
3: 'mercredi',
|
87
|
4: 'jeudi',
|
88
|
5: 'vendredi'
|
89
|
}
|
90
|
|
91
|
dic_worker = {}
|
92
|
|
93
|
def load_csv2(db, name, offset=0, limit=9999999, id_column=0):
|
94
|
csvfile = open(os.path.join(db_path, db, name + '.csv'), 'rb')
|
95
|
csvlines = csv.reader(csvfile, delimiter=';', quotechar='|')
|
96
|
cols = csvlines.next()
|
97
|
yield cols
|
98
|
i = 0
|
99
|
for line in csvlines:
|
100
|
if not (offset <= int(line[id_column]) < offset+limit):
|
101
|
continue
|
102
|
yield _get_dict(cols, line)
|
103
|
i += 1
|
104
|
csvfile.close()
|
105
|
|
106
|
def load_csv(db, name, offset=0, limit=9999999, id_column=0):
|
107
|
records = []
|
108
|
idx = {}
|
109
|
|
110
|
csvfile = open(os.path.join(db_path, db, name + '.csv'), 'rb')
|
111
|
csvlines = csv.reader(csvfile, delimiter=';', quotechar='|')
|
112
|
cols = csvlines.next()
|
113
|
i = 0
|
114
|
for line in csvlines:
|
115
|
if not (offset <= int(line[id_column]) < offset+limit):
|
116
|
continue
|
117
|
data = _get_dict(cols, line)
|
118
|
records.append(data)
|
119
|
idx[data['id']] = i
|
120
|
i += 1
|
121
|
csvfile.close()
|
122
|
return records, idx, cols
|
123
|
|
124
|
def add_invalid(d, reason):
|
125
|
d.setdefault('invalid', '')
|
126
|
if d['invalid']:
|
127
|
d['invalid'] += ', '
|
128
|
d['invalid'] += reason
|
129
|
|
130
|
log = open('import_actes_ids.log', 'a')
|
131
|
|
132
|
@transaction.commit_on_success
|
133
|
def main():
|
134
|
""" """
|
135
|
|
136
|
workers = Worker.objects.all()
|
137
|
for db in dbs:
|
138
|
workers_idx = {}
|
139
|
act_types_idx = {}
|
140
|
if "F_ST_ETIENNE_CMPP" == db:
|
141
|
service = Service.objects.get(name="CMPP")
|
142
|
elif "F_ST_ETIENNE_CAMSP" == db:
|
143
|
service = Service.objects.get(name="CAMSP")
|
144
|
elif "F_ST_ETIENNE_SESSAD_TED" == db:
|
145
|
service = Service.objects.get(name="SESSAD TED")
|
146
|
elif "F_ST_ETIENNE_SESSAD" == db:
|
147
|
service = Service.objects.get(name="SESSAD DYS")
|
148
|
|
149
|
print '===', service.name, '==='
|
150
|
print datetime.now()
|
151
|
print >>log, datetime.now(), '===', service.name, '==='
|
152
|
|
153
|
# load workers mapping
|
154
|
for i, worker in enumerate(workers):
|
155
|
if service.name == 'CMPP':
|
156
|
j = worker.old_cmpp_id
|
157
|
elif service.name == 'CAMSP':
|
158
|
j = worker.old_camsp_id
|
159
|
elif service.name == 'SESSAD DYS':
|
160
|
j = worker.old_sessad_dys_id
|
161
|
elif service.name == 'SESSAD TED':
|
162
|
j = worker.old_sessad_ted_id
|
163
|
else:
|
164
|
print "service inconnu!!!"
|
165
|
exit(0)
|
166
|
if j:
|
167
|
workers_idx[j] = worker
|
168
|
# load act_type mapping
|
169
|
act_types = ActType.objects.for_service(service)
|
170
|
act_type_id_not_found = set()
|
171
|
for i, act_type in enumerate(act_types):
|
172
|
j = act_type.old_id
|
173
|
if j:
|
174
|
act_types_idx[j] = act_type
|
175
|
else:
|
176
|
act_type_id_not_found.add(act_type)
|
177
|
|
178
|
def set_act_type(row, not_found=None):
|
179
|
act_type_id = row['type_acte']
|
180
|
if act_type_id == '0':
|
181
|
add_invalid(row, 'no act_id=>not importable')
|
182
|
elif act_type_id in act_types_idx:
|
183
|
row['act_type'] = act_types_idx[act_type_id]
|
184
|
else:
|
185
|
add_invalid(row, 'act_type not found %s' % act_type_id)
|
186
|
if not_found:
|
187
|
not_found.add(act_type_id)
|
188
|
|
189
|
def handle_details2(data, idx, details, id_key):
|
190
|
for detail in details:
|
191
|
i = detail[id_key]
|
192
|
thera_id = detail['thera_id']
|
193
|
if i not in idx:
|
194
|
continue
|
195
|
row = data[idx[i]]
|
196
|
if thera_id in workers_idx:
|
197
|
workers = row.setdefault('workers', set())
|
198
|
workers.add(workers_idx[thera_id])
|
199
|
else:
|
200
|
add_invalid(row, 'unknown thera_id %s' % thera_id)
|
201
|
|
202
|
print "%s - Nombre de types d'actes : %d" % (service.name, len(act_types))
|
203
|
print "%s - Liste des types d'actes sans id : %s" % (service.name, str(act_type_id_not_found))
|
204
|
|
205
|
# loading dossiers idx
|
206
|
enfant_idx = {}
|
207
|
for enfant in PatientRecord.objects.filter(service=service):
|
208
|
enfant_idx[enfant.old_id] = enfant
|
209
|
|
210
|
def set_enfant(row, not_found=None):
|
211
|
# connect enfant
|
212
|
enfant_id = row['enfant_id']
|
213
|
if enfant_id == '0':
|
214
|
add_invalid(row, 'no enfant_id=>not an appointment')
|
215
|
row['event'] = True
|
216
|
elif enfant_id in enfant_idx:
|
217
|
row['enfant'] = enfant_idx[enfant_id]
|
218
|
else:
|
219
|
add_invalid(row, 'enfant_id not found %s' % enfant_id)
|
220
|
if not_found:
|
221
|
not_found.add(enfant_id)
|
222
|
|
223
|
# Clean act for this service
|
224
|
notfound = []
|
225
|
for year in range(1990, 2014):
|
226
|
# create index of already imported acts
|
227
|
act_idx = defaultdict(lambda: [])
|
228
|
for act in Act.objects.filter(date__year=year, patient__service=service):
|
229
|
key = (act.date, act.time, act.patient.old_id, act.act_type.old_id)
|
230
|
act_idx[key].append((act.id, bool(act.old_id)))
|
231
|
|
232
|
rows = load_csv2(db, 'actes')
|
233
|
cols = rows.next()
|
234
|
total = 0
|
235
|
for row in rows:
|
236
|
row.setdefault('invalid', '')
|
237
|
row.setdefault('workers', set())
|
238
|
row['date'] = _to_date(row['date_acte'])
|
239
|
if row['date'].year == year:
|
240
|
continue
|
241
|
row['time'] = _to_time(row['heure'])
|
242
|
row['duration'] = _to_duration(row['duree'])
|
243
|
row['is_billed'] = row['marque'] == '1'
|
244
|
row['validation_locked'] = row['date'] < date(2013, 1, 3)
|
245
|
set_enfant(row)
|
246
|
set_act_type(row)
|
247
|
row['state'] = map_cs[service.name].get(row['cs'], 'VALIDE')
|
248
|
key = (row['date'], row['time'], row['enfant_id'], row['type_acte'])
|
249
|
if key in act_idx:
|
250
|
acts = act_idx[key]
|
251
|
assert len(acts) != 0
|
252
|
if len(acts) > 1:
|
253
|
print >>log, datetime.now(), 'found more than one act for', row, ':', acts
|
254
|
continue
|
255
|
else:
|
256
|
act_id, has_old_id = acts[0]
|
257
|
if not has_old_id:
|
258
|
Act.objects.filter(id=act_id).update(old_id=row['id'])
|
259
|
total += 1
|
260
|
print >>log, datetime.now(), 'imported old_id', row['id'], 'to Act(id=', act_id, ')'
|
261
|
print >>log, datetime.now(), 'Fixed', total, 'acts for', year
|
262
|
print >>log, datetime.now(), 'Found', len(notfound), 'acts not imported'
|
263
|
|
264
|
cols.extend(['workers','invalid'])
|
265
|
actes_details = load_csv2(db, 'details_actes')
|
266
|
actes_details.next()
|
267
|
notfound_idx = dict((i, row['id']) for i, row in enumerate(notfound))
|
268
|
handle_details2(notfound, notfound_idx, actes_details, 'acte_id')
|
269
|
for row in notfound:
|
270
|
if row['invalid']:
|
271
|
print >>log, datetime.now(), 'row invalid', row
|
272
|
continue
|
273
|
act = Act.objects.create(
|
274
|
old_id=row['id'],
|
275
|
date=row['date'],
|
276
|
time=row['time'],
|
277
|
_duration=row['duration'].seconds // 60,
|
278
|
is_billed=row['is_billed'],
|
279
|
act_type=row['act_type'],
|
280
|
patient=row['enfant'],
|
281
|
validation_locked=row['validation_locked'],
|
282
|
parent_event=row['parent_event'])
|
283
|
act.doctors = row['workers']
|
284
|
ActValidationState.objects.create(act=act, state_name=row['state'], previous_state=None)
|
285
|
raise Exception('donothing')
|
286
|
|
287
|
if __name__ == "__main__":
|
288
|
main()
|