Revision bfe4851b
Added by Benjamin Dauvergne (en congé) about 12 years ago
scripts/import_actes_ids.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
#!/usr/bin/env python |
|
3 |
|
|
4 |
import os |
|
5 |
import csv |
|
6 |
from collections import defaultdict |
|
7 |
|
|
8 |
from datetime import datetime, date |
|
9 |
|
|
10 |
import calebasse.settings |
|
11 |
import django.core.management |
|
12 |
|
|
13 |
django.core.management.setup_environ(calebasse.settings) |
|
14 |
|
|
15 |
from django.db import transaction |
|
16 |
from calebasse.agenda.models import EventWithAct |
|
17 |
from calebasse.dossiers.models import PatientRecord |
|
18 |
from calebasse.personnes.models import Worker |
|
19 |
from calebasse.ressources.models import Service |
|
20 |
from calebasse.ressources.models import ActType |
|
21 |
from calebasse.actes.models import Act, ActValidationState |
|
22 |
|
|
23 |
from import_dossiers import map_cs |
|
24 |
|
|
25 |
# Configuration |
|
26 |
db_path = "./scripts/20130104-213225" |
|
27 |
|
|
28 |
dbs = ["F_ST_ETIENNE_SESSAD_TED", "F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD"] |
|
29 |
|
|
30 |
def _to_datetime(str_date): |
|
31 |
if not str_date: |
|
32 |
return None |
|
33 |
return datetime.strptime(str_date[:19], "%Y-%m-%d %H:%M:%S") |
|
34 |
|
|
35 |
def _to_date(str_date): |
|
36 |
dt = _to_datetime(str_date) |
|
37 |
return dt and dt.date() |
|
38 |
|
|
39 |
def _to_time(str_date): |
|
40 |
dt = _to_datetime(str_date) |
|
41 |
return dt and dt.time() |
|
42 |
|
|
43 |
def _to_duration(str_date): |
|
44 |
dt = _to_datetime(str_date) |
|
45 |
if dt is None: |
|
46 |
return None |
|
47 |
return dt - datetime(1900, 01, 01, 0, 0) |
|
48 |
|
|
49 |
def _to_int(str_int): |
|
50 |
if not str_int: |
|
51 |
return None |
|
52 |
return int(str_int) |
|
53 |
|
|
54 |
def _get_dict(cols, line): |
|
55 |
"""""" |
|
56 |
res = {} |
|
57 |
for i, data in enumerate(line): |
|
58 |
res[cols[i]] = data.decode('utf-8') |
|
59 |
return res |
|
60 |
|
|
61 |
def batch_delete(qs, limit): |
|
62 |
count = qs.count() |
|
63 |
i = 0 |
|
64 |
while i < count: |
|
65 |
ids = qs[i:i+limit].values_list('pk', flat=True) |
|
66 |
qs.filter(pk__in=ids).delete() |
|
67 |
i += limit |
|
68 |
|
|
69 |
|
|
70 |
PERIOD_FAURE_NOUS = {1 : 1, |
|
71 |
2 : 2, |
|
72 |
3 : 3, |
|
73 |
4 : 4, |
|
74 |
5: 6, |
|
75 |
6: 7, |
|
76 |
7: 8, |
|
77 |
8: 9, |
|
78 |
9: None, |
|
79 |
10: 10, |
|
80 |
12: 11, |
|
81 |
13: 12, |
|
82 |
} |
|
83 |
|
|
84 |
JOURS = {1: 'lundi', |
|
85 |
2: 'mardi', |
|
86 |
3: 'mercredi', |
|
87 |
4: 'jeudi', |
|
88 |
5: 'vendredi' |
|
89 |
} |
|
90 |
|
|
91 |
dic_worker = {} |
|
92 |
|
|
93 |
def load_csv2(db, name, offset=0, limit=9999999, id_column=0): |
|
94 |
csvfile = open(os.path.join(db_path, db, name + '.csv'), 'rb') |
|
95 |
csvlines = csv.reader(csvfile, delimiter=';', quotechar='|') |
|
96 |
cols = csvlines.next() |
|
97 |
yield cols |
|
98 |
i = 0 |
|
99 |
for line in csvlines: |
|
100 |
if not (offset <= int(line[id_column]) < offset+limit): |
|
101 |
continue |
|
102 |
yield _get_dict(cols, line) |
|
103 |
i += 1 |
|
104 |
csvfile.close() |
|
105 |
|
|
106 |
def load_csv(db, name, offset=0, limit=9999999, id_column=0): |
|
107 |
records = [] |
|
108 |
idx = {} |
|
109 |
|
|
110 |
csvfile = open(os.path.join(db_path, db, name + '.csv'), 'rb') |
|
111 |
csvlines = csv.reader(csvfile, delimiter=';', quotechar='|') |
|
112 |
cols = csvlines.next() |
|
113 |
i = 0 |
|
114 |
for line in csvlines: |
|
115 |
if not (offset <= int(line[id_column]) < offset+limit): |
|
116 |
continue |
|
117 |
data = _get_dict(cols, line) |
|
118 |
records.append(data) |
|
119 |
idx[data['id']] = i |
|
120 |
i += 1 |
|
121 |
csvfile.close() |
|
122 |
return records, idx, cols |
|
123 |
|
|
124 |
def add_invalid(d, reason): |
|
125 |
d.setdefault('invalid', '') |
|
126 |
if d['invalid']: |
|
127 |
d['invalid'] += ', ' |
|
128 |
d['invalid'] += reason |
|
129 |
|
|
130 |
log = open('import_actes_ids.log', 'a') |
|
131 |
|
|
132 |
@transaction.commit_on_success |
|
133 |
def main(): |
|
134 |
""" """ |
|
135 |
|
|
136 |
workers = Worker.objects.all() |
|
137 |
for db in dbs: |
|
138 |
workers_idx = {} |
|
139 |
act_types_idx = {} |
|
140 |
if "F_ST_ETIENNE_CMPP" == db: |
|
141 |
service = Service.objects.get(name="CMPP") |
|
142 |
elif "F_ST_ETIENNE_CAMSP" == db: |
|
143 |
service = Service.objects.get(name="CAMSP") |
|
144 |
elif "F_ST_ETIENNE_SESSAD_TED" == db: |
|
145 |
service = Service.objects.get(name="SESSAD TED") |
|
146 |
elif "F_ST_ETIENNE_SESSAD" == db: |
|
147 |
service = Service.objects.get(name="SESSAD DYS") |
|
148 |
|
|
149 |
print '===', service.name, '===' |
|
150 |
print datetime.now() |
|
151 |
print >>log, datetime.now(), '===', service.name, '===' |
|
152 |
|
|
153 |
# load workers mapping |
|
154 |
for i, worker in enumerate(workers): |
|
155 |
if service.name == 'CMPP': |
|
156 |
j = worker.old_cmpp_id |
|
157 |
elif service.name == 'CAMSP': |
|
158 |
j = worker.old_camsp_id |
|
159 |
elif service.name == 'SESSAD DYS': |
|
160 |
j = worker.old_sessad_dys_id |
|
161 |
elif service.name == 'SESSAD TED': |
|
162 |
j = worker.old_sessad_ted_id |
|
163 |
else: |
|
164 |
print "service inconnu!!!" |
|
165 |
exit(0) |
|
166 |
if j: |
|
167 |
workers_idx[j] = worker |
|
168 |
# load act_type mapping |
|
169 |
act_types = ActType.objects.for_service(service) |
|
170 |
act_type_id_not_found = set() |
|
171 |
for i, act_type in enumerate(act_types): |
|
172 |
j = act_type.old_id |
|
173 |
if j: |
|
174 |
act_types_idx[j] = act_type |
|
175 |
else: |
|
176 |
act_type_id_not_found.add(act_type) |
|
177 |
|
|
178 |
def set_act_type(row, not_found=None): |
|
179 |
act_type_id = row['type_acte'] |
|
180 |
if act_type_id == '0': |
|
181 |
add_invalid(row, 'no act_id=>not importable') |
|
182 |
elif act_type_id in act_types_idx: |
|
183 |
row['act_type'] = act_types_idx[act_type_id] |
|
184 |
else: |
|
185 |
add_invalid(row, 'act_type not found %s' % act_type_id) |
|
186 |
if not_found: |
|
187 |
not_found.add(act_type_id) |
|
188 |
|
|
189 |
def handle_details2(data, idx, details, id_key): |
|
190 |
for detail in details: |
|
191 |
i = detail[id_key] |
|
192 |
thera_id = detail['thera_id'] |
|
193 |
if i not in idx: |
|
194 |
continue |
|
195 |
row = data[idx[i]] |
|
196 |
if thera_id in workers_idx: |
|
197 |
workers = row.setdefault('workers', set()) |
|
198 |
workers.add(workers_idx[thera_id]) |
|
199 |
else: |
|
200 |
add_invalid(row, 'unknown thera_id %s' % thera_id) |
|
201 |
|
|
202 |
print "%s - Nombre de types d'actes : %d" % (service.name, len(act_types)) |
|
203 |
print "%s - Liste des types d'actes sans id : %s" % (service.name, str(act_type_id_not_found)) |
|
204 |
|
|
205 |
# loading dossiers idx |
|
206 |
enfant_idx = {} |
|
207 |
for enfant in PatientRecord.objects.filter(service=service): |
|
208 |
enfant_idx[enfant.old_id] = enfant |
|
209 |
|
|
210 |
def set_enfant(row, not_found=None): |
|
211 |
# connect enfant |
|
212 |
enfant_id = row['enfant_id'] |
|
213 |
if enfant_id == '0': |
|
214 |
add_invalid(row, 'no enfant_id=>not an appointment') |
|
215 |
row['event'] = True |
|
216 |
elif enfant_id in enfant_idx: |
|
217 |
row['enfant'] = enfant_idx[enfant_id] |
|
218 |
else: |
|
219 |
add_invalid(row, 'enfant_id not found %s' % enfant_id) |
|
220 |
if not_found: |
|
221 |
not_found.add(enfant_id) |
|
222 |
|
|
223 |
# Clean act for this service |
|
224 |
notfound = [] |
|
225 |
for year in range(1990, 2014): |
|
226 |
# create index of already imported acts |
|
227 |
act_idx = defaultdict(lambda: []) |
|
228 |
for act in Act.objects.filter(date__year=year, patient__service=service): |
|
229 |
key = (act.date, act.time, act.patient.old_id, act.act_type.old_id) |
|
230 |
act_idx[key].append((act.id, bool(act.old_id))) |
|
231 |
|
|
232 |
rows = load_csv2(db, 'actes') |
|
233 |
cols = rows.next() |
|
234 |
total = 0 |
|
235 |
for row in rows: |
|
236 |
row.setdefault('invalid', '') |
|
237 |
row.setdefault('workers', set()) |
|
238 |
row['date'] = _to_date(row['date_acte']) |
|
239 |
if row['date'].year == year: |
|
240 |
continue |
|
241 |
row['time'] = _to_time(row['heure']) |
|
242 |
row['duration'] = _to_duration(row['duree']) |
|
243 |
row['is_billed'] = row['marque'] == '1' |
|
244 |
row['validation_locked'] = row['date'] < date(2013, 1, 3) |
|
245 |
set_enfant(row) |
|
246 |
set_act_type(row) |
|
247 |
row['state'] = map_cs[service.name].get(row['cs'], 'VALIDE') |
|
248 |
key = (row['date'], row['time'], row['enfant_id'], row['type_acte']) |
|
249 |
if key in act_idx: |
|
250 |
acts = act_idx[key] |
|
251 |
assert len(acts) != 0 |
|
252 |
if len(acts) > 1: |
|
253 |
print >>log, datetime.now(), 'found more than one act for', row, ':', acts |
|
254 |
continue |
|
255 |
else: |
|
256 |
act_id, has_old_id = acts[0] |
|
257 |
if not has_old_id: |
|
258 |
Act.objects.filter(id=act_id).update(old_id=row['id']) |
|
259 |
total += 1 |
|
260 |
print >>log, datetime.now(), 'imported old_id', row['id'], 'to Act(id=', act_id, ')' |
|
261 |
print >>log, datetime.now(), 'Fixed', total, 'acts for', year |
|
262 |
print >>log, datetime.now(), 'Found', len(notfound), 'acts not imported' |
|
263 |
|
|
264 |
cols.extend(['workers','invalid']) |
|
265 |
actes_details = load_csv2(db, 'details_actes') |
|
266 |
actes_details.next() |
|
267 |
notfound_idx = dict((i, row['id']) for i, row in enumerate(notfound)) |
|
268 |
handle_details2(notfound, notfound_idx, actes_details, 'acte_id') |
|
269 |
for row in notfound: |
|
270 |
if row['invalid']: |
|
271 |
print >>log, datetime.now(), 'row invalid', row |
|
272 |
continue |
|
273 |
act = Act.objects.create( |
|
274 |
old_id=row['id'], |
|
275 |
date=row['date'], |
|
276 |
time=row['time'], |
|
277 |
_duration=row['duration'].seconds // 60, |
|
278 |
is_billed=row['is_billed'], |
|
279 |
act_type=row['act_type'], |
|
280 |
patient=row['enfant'], |
|
281 |
validation_locked=row['validation_locked'], |
|
282 |
parent_event=row['parent_event']) |
|
283 |
act.doctors = row['workers'] |
|
284 |
ActValidationState.objects.create(act=act, state_name=row['state'], previous_state=None) |
|
285 |
raise Exception('donothing') |
|
286 |
|
|
287 |
if __name__ == "__main__": |
|
288 |
main() |
Also available in: Unified diff
scripts: import_acts_id.py first version