Revision 50f8b1ae
Added by Benjamin Dauvergne over 12 years ago
| scripts/import_rs.py | ||
|---|---|---|
| 4 | 4 |
import os |
| 5 | 5 |
import csv |
| 6 | 6 |
|
| 7 |
from datetime import datetime, time, date
|
|
| 7 |
from datetime import datetime |
|
| 8 | 8 |
|
| 9 | 9 |
import calebasse.settings |
| 10 | 10 |
import django.core.management |
| 11 |
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned |
|
| 12 | 11 |
|
| 13 | 12 |
django.core.management.setup_environ(calebasse.settings) |
| 14 | 13 |
|
| 15 |
from django.contrib.auth.models import User |
|
| 16 |
|
|
| 17 |
from calebasse.agenda.models import Event, EventType |
|
| 18 |
from calebasse.dossiers.models import PatientRecord, Status, FileState |
|
| 19 |
from calebasse.ressources.models import Service |
|
| 20 |
from calebasse.personnes.models import Worker, Holiday, TimeTable, PERIODICITIES |
|
| 21 |
from calebasse.personnes.forms import PERIOD_LIST_TO_FIELDS |
|
| 22 |
from calebasse.ressources.models import WorkerType, HolidayType, ActType |
|
| 14 |
from django.db import transaction |
|
| 15 |
from calebasse.agenda.models import EventWithAct, Event |
|
| 16 |
from calebasse.dossiers.models import PatientRecord |
|
| 17 |
from calebasse.personnes.models import Worker |
|
| 18 |
from calebasse.ressources.models import Service, Room |
|
| 19 |
from calebasse.ressources.models import ActType |
|
| 23 | 20 |
|
| 24 | 21 |
# Configuration |
| 25 | 22 |
db_path = "./scripts/20121221-192258" |
| 26 | 23 |
|
| 27 |
dbs = ["F_ST_ETIENNE_SESSAD_TED", "F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD"] |
|
| 24 |
# dbs = ["F_ST_ETIENNE_SESSAD_TED", "F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD"] |
|
| 25 |
dbs = ["F_ST_ETIENNE_CAMSP"] |
|
| 28 | 26 |
|
| 29 | 27 |
def _to_datetime(str_date): |
| 30 | 28 |
if not str_date: |
| ... | ... | |
| 98 | 96 |
csvfile.close() |
| 99 | 97 |
return records, idx, cols |
| 100 | 98 |
|
| 99 |
def add_invalid(d, reason): |
|
| 100 |
d.setdefault('invalid', '')
|
|
| 101 |
if d['invalid']: |
|
| 102 |
d['invalid'] += ', ' |
|
| 103 |
d['invalid'] += reason |
|
| 104 |
|
|
| 101 | 105 |
def main(): |
| 102 | 106 |
""" """ |
| 103 | 107 |
|
| 104 | 108 |
workers = Worker.objects.all() |
| 105 |
invalid_rs_csv = open('./scripts/invalid.csv', 'wb+')
|
|
| 106 |
writer = csv.writer(invalid_rs_csv, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) |
|
| 109 |
invalid_rs_csv = open('./scripts/invalid_rs.csv', 'wb+')
|
|
| 110 |
invalid_rs_writer = csv.writer(invalid_rs_csv, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) |
|
| 111 |
invalid_rr_csv = open('./scripts/invalid_rr.csv', 'wb+')
|
|
| 112 |
invalid_rr_writer = csv.writer(invalid_rr_csv, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) |
|
| 107 | 113 |
for db in dbs: |
| 108 | 114 |
workers_idx = {}
|
| 109 | 115 |
act_types_idx = {}
|
| ... | ... | |
| 122 | 128 |
|
| 123 | 129 |
rs_data, rs_idx, rs_cols = load_csv(db, 'rs') |
| 124 | 130 |
rr_data, rr_idx, rr_cols = load_csv(db, 'rr') |
| 125 |
writer.writerow(map(lambda x: x.encode('utf-8'), rs_cols))
|
|
| 131 |
rs_cols.extend(['workers', 'invalid']) |
|
| 132 |
rr_cols.extend(['workers', 'invalid']) |
|
| 133 |
invalid_rs_writer.writerow(map(lambda x: x.encode('utf-8'), rs_cols))
|
|
| 134 |
invalid_rr_writer.writerow(map(lambda x: x.encode('utf-8'), rr_cols))
|
|
| 126 | 135 |
|
| 127 | 136 |
print "%s - Nombre de rdv : %d" % (service.name, len(rs_data)) |
| 128 | 137 |
print u"%s - Nombre de rdv récurrents : %d" % (service.name, len(rr_data)) |
| ... | ... | |
| 153 | 162 |
else: |
| 154 | 163 |
act_type_id_not_found.add(act_type) |
| 155 | 164 |
|
| 156 |
not_found = set() |
|
| 157 |
rs_not_found = set() |
|
| 158 |
for rs_detail in rs_details_data: |
|
| 159 |
rs_id = rs_detail['rs_id'] |
|
| 160 |
thera_id = rs_detail['thera_id'] |
|
| 161 |
if rs_id not in rs_idx: |
|
| 162 |
rs_not_found.add(rs_id) |
|
| 163 |
continue |
|
| 164 |
rs = rs_data[rs_idx[rs_id]] |
|
| 165 |
if thera_id in workers_idx: |
|
| 166 |
rs_workers = rs.setdefault('workers', set())
|
|
| 167 |
rs_workers.add(workers_idx[thera_id]) |
|
| 168 |
else: |
|
| 169 |
rs['valid'] = False |
|
| 170 |
not_found.add(thera_id) |
|
| 171 |
|
|
| 165 |
|
|
| 166 |
def handle_details(data, idx, details, id_key): |
|
| 167 |
not_found = set() |
|
| 168 |
id_not_found = set() |
|
| 169 |
for detail in details: |
|
| 170 |
i = detail[id_key] |
|
| 171 |
thera_id = detail['thera_id'] |
|
| 172 |
if i not in idx: |
|
| 173 |
id_not_found.add(i) |
|
| 174 |
continue |
|
| 175 |
row = data[idx[i]] |
|
| 176 |
if thera_id in workers_idx: |
|
| 177 |
workers = row.setdefault('workers', set())
|
|
| 178 |
workers.add(workers_idx[thera_id]) |
|
| 179 |
else: |
|
| 180 |
add_invalid(row, 'unknown thera_id %s' % thera_id) |
|
| 181 |
not_found.add(thera_id) |
|
| 172 | 182 |
|
| 173 |
print "%s - Liste des worker not found : %s" % (service.name, str(set(not_found))) |
|
| 174 |
print "%s - Liste des details pour des RS not found : %s" % (service.name, str(set(rs_not_found)))
|
|
| 183 |
print "%s - Liste des worker not found : %s" % (service.name, str(set(not_found)))
|
|
| 184 |
print "%s - Liste des details pour des RS/RR not found : %s" % (service.name, str(set(id_not_found)))
|
|
| 175 | 185 |
|
| 186 |
print ' - Traitement rs_detail....' |
|
| 187 |
handle_details(rs_data, rs_idx, rs_details_data, 'rs_id') |
|
| 188 |
print ' - Traitement rr_detail....' |
|
| 189 |
handle_details(rr_data, rr_idx, rr_details_data, 'rr_id') |
|
| 176 | 190 |
print "%s - Nombre de types d'actes : %d" % (service.name, len(act_types)) |
| 177 | 191 |
print "%s - Liste des types d'actes sans id : %s" % (service.name, str(act_type_id_not_found)) |
| 178 | 192 |
|
| ... | ... | |
| 185 | 199 |
rs_without_act_type = set() |
| 186 | 200 |
unknown_act_type_id = set() |
| 187 | 201 |
invalid_rs = set() |
| 202 |
seen_exceptions = dict() |
|
| 188 | 203 |
for rs in rs_data: |
| 189 |
rs['ok'] = True |
|
| 190 |
rs.setdefault('valid', True)
|
|
| 204 |
rs['exception'] = False |
|
| 205 |
rs['event'] = False |
|
| 206 |
rs['date'] = _to_date(rs['date_rdv']) |
|
| 191 | 207 |
# connect enfant |
| 192 | 208 |
enfant_id = rs['enfant_id'] |
| 193 | 209 |
if enfant_id == '0': |
| 194 |
rs['ok'] = False |
|
| 210 |
add_invalid(rs, 'no enfant_id=>not an appointment') |
|
| 211 |
rs['event'] = True |
|
| 195 | 212 |
elif enfant_id in enfant_idx: |
| 196 | 213 |
rs['enfant'] = enfant_idx[enfant_id] |
| 197 | 214 |
else: |
| 198 |
rs['ok'] = rs['valid'] = False
|
|
| 215 |
add_invalid(rs, 'enfant_id not found %s' % enfant_id)
|
|
| 199 | 216 |
enfant_not_found.add(enfant_id) |
| 200 | 217 |
# connect rr |
| 201 | 218 |
rr_id = rs['rr_ev_id'] |
| ... | ... | |
| 205 | 222 |
elif rr_id.startswith('rr_'):
|
| 206 | 223 |
_, rr_id = rr_id.split('_')
|
| 207 | 224 |
if rr_id in rr_idx: |
| 208 |
rs['rr'] = rr_data[rr_idx[rr_id]] |
|
| 225 |
if (rr_id, rs['date']) not in seen_exceptions: |
|
| 226 |
seen_exceptions[(rr_id, rs['date'])] = rs['id'] |
|
| 227 |
exceptions = rr_data[rr_idx[rr_id]].setdefault('exceptions', [])
|
|
| 228 |
exceptions.append(rs) |
|
| 229 |
rs['exception'] = True |
|
| 230 |
else: |
|
| 231 |
add_invalid(rs, 'already another exception on the same day: %s' % |
|
| 232 |
seen_exceptions[(rr_id, rs['date'])]) |
|
| 209 | 233 |
else: |
| 210 |
rs['ok'] = False
|
|
| 234 |
add_invalid(rs, 'rr_id not found %s' % rr_id)
|
|
| 211 | 235 |
rr_not_found.add(rr_id) |
| 212 | 236 |
else: |
| 213 |
print 'invalid rr_id', rr_id |
|
| 214 |
rs['date_rdv'] = _to_date(rs['date_rdv']) |
|
| 215 |
rs['heure'] = _to_time(rs['heure']) |
|
| 216 |
rs['duree'] = _to_duration(rs['duree']) |
|
| 237 |
add_invalid(rs, 'rr_id invalid %s' % rr_id) |
|
| 238 |
rs['time'] = _to_time(rs['heure']) |
|
| 239 |
rs['duration'] = _to_duration(rs['duree']) |
|
| 240 |
rs['start_datetime'] = datetime.combine(rs['date'], rs['time']) |
|
| 241 |
rs['end_datetime'] = rs['start_datetime'] + rs['duration'] |
|
| 217 | 242 |
act_type_id = rs['type_acte'] |
| 218 | 243 |
if act_type_id == '0' and rs['enfant_id'] == '0': |
| 219 |
rs['ok'] = False |
|
| 244 |
add_invalid(rs, 'no act_id=>not an appointment') |
|
| 245 |
rs['event'] = True |
|
| 220 | 246 |
elif act_type_id: |
| 221 | 247 |
if act_type_id in act_types_idx: |
| 222 | 248 |
rs['act_type'] = act_types_idx[act_type_id] |
| 223 | 249 |
else: |
| 224 |
rs['ok'] = rs['valid'] = False
|
|
| 250 |
add_invalid(rs, 'act_type not found %s' % act_type_id)
|
|
| 225 | 251 |
unknown_act_type_id.add(act_type_id) |
| 226 | 252 |
else: |
| 227 | 253 |
raise NotImplemented |
| 228 |
if not rs['valid']:
|
|
| 254 |
if rs.get('invalid') and not rs['event']:
|
|
| 229 | 255 |
invalid_rs.add(rs['id']) |
| 230 |
writer.writerow([ unicode(rs[col]).encode('utf-8') for col in rs_cols ])
|
|
| 256 |
invalid_rs_writer.writerow([ unicode(rs[col]).encode('utf-8') for col in rs_cols ])
|
|
| 231 | 257 |
|
| 232 | 258 |
|
| 233 | 259 |
print "%s - Liste des enfants not found : %s" % (service.name, str(enfant_not_found)) |
| ... | ... | |
| 236 | 262 |
print "%s - Liste des types d'actes inconnus : %s" % (service.name, str(unknown_act_type_id)) |
| 237 | 263 |
print "%s - Liste des RS invalides : %s" % (service.name, len(invalid_rs)) |
| 238 | 264 |
|
| 265 |
enfant_not_found = set() |
|
| 266 |
rr_not_found = set() |
|
| 267 |
rs_without_act_type = set() |
|
| 268 |
unknown_act_type_id = set() |
|
| 269 |
invalid_rs = set() |
|
| 270 |
for rr in rr_data: |
|
| 271 |
rr.setdefault('exceptions', [])
|
|
| 272 |
rr['start_date'] = _to_date(rr['date_debut']) |
|
| 273 |
rr['end_date'] = _to_date(rr['date_fin']) |
|
| 274 |
if rr['end_date'] and rr['start_date'] > rr['end_date']: |
|
| 275 |
add_invalid(rr, 'date_fin < date_debut') |
|
| 276 |
rr['time'] = _to_time(rr['heure']) |
|
| 277 |
rr['duration'] = _to_duration(rr['duree']) |
|
| 278 |
rr['start_datetime'] = datetime.combine(rr['start_date'], rr['time']) |
|
| 279 |
rr['end_datetime'] = rr['start_datetime'] + rr['duration'] |
|
| 280 |
# connect rythme |
|
| 281 |
rr['rythme'] = int(rr['rythme']) |
|
| 282 |
if PERIOD_FAURE_NOUS.get(rr['rythme']): |
|
| 283 |
rr['recurrence_periodicity'] = PERIOD_FAURE_NOUS[rr['rythme']] |
|
| 284 |
else: |
|
| 285 |
add_invalid(rr, 'rythme not found %s' % rr['rythme']) |
|
| 286 |
# connect enfant |
|
| 287 |
enfant_id = rr['enfant_id'] |
|
| 288 |
if enfant_id == '0': |
|
| 289 |
add_invalid(rr, 'not an appointment') |
|
| 290 |
elif enfant_id in enfant_idx: |
|
| 291 |
rr['enfant'] = enfant_idx[enfant_id] |
|
| 292 |
else: |
|
| 293 |
add_invalid(rr, 'enfant_id not found %s' % enfant_id) |
|
| 294 |
enfant_not_found.add(enfant_id) |
|
| 295 |
# connect act_type |
|
| 296 |
act_type_id = rr['type_acte'] |
|
| 297 |
if act_type_id == '0' and rr['enfant_id'] == '0': |
|
| 298 |
add_invalid(rr, 'not an appointment') |
|
| 299 |
elif act_type_id: |
|
| 300 |
if act_type_id in act_types_idx: |
|
| 301 |
rr['act_type'] = act_types_idx[act_type_id] |
|
| 302 |
else: |
|
| 303 |
add_invalid(rr, 'act_type not found %s' % act_type_id) |
|
| 304 |
unknown_act_type_id.add(act_type_id) |
|
| 305 |
else: |
|
| 306 |
raise NotImplemented |
|
| 307 |
if rr.get('invalid'):
|
|
| 308 |
invalid_rr_writer.writerow([ unicode(rr[col]).encode('utf-8') for col in rr_cols ])
|
|
| 309 |
|
|
| 310 |
# stats |
|
| 311 |
exceptions = 0 |
|
| 312 |
events = 0 |
|
| 313 |
single = 0 |
|
| 314 |
recurrent = 0 |
|
| 315 |
invalid_single = 0 |
|
| 316 |
invalid_recurrent = 0 |
|
| 317 |
for row in rs_data: |
|
| 318 |
if row['event']: |
|
| 319 |
events += 1 |
|
| 320 |
elif row.get('invalid'):
|
|
| 321 |
invalid_single += 1 |
|
| 322 |
elif row['exception']: |
|
| 323 |
exceptions += 1 |
|
| 324 |
else: |
|
| 325 |
single += 1 |
|
| 326 |
for row in rr_data: |
|
| 327 |
if row.get('invalid'):
|
|
| 328 |
invalid_recurrent += 1 |
|
| 329 |
else: |
|
| 330 |
recurrent += 1 |
|
| 331 |
print ' == Stat == ' |
|
| 332 |
print ' Évènements hors RDV', events |
|
| 333 |
print ' Rdv individuels', single |
|
| 334 |
print ' Rdv recurrents', recurrent |
|
| 335 |
print ' Exceptions', exceptions |
|
| 336 |
print ' Rdv recurrents invalides', invalid_recurrent |
|
| 337 |
print ' Rdv individuels invalides', invalid_single |
|
| 338 |
|
|
| 339 |
# create single rdv |
|
| 340 |
limit = 1000000 |
|
| 341 |
with transaction.commit_manually(): |
|
| 342 |
try: |
|
| 343 |
# single RS |
|
| 344 |
i = 0 |
|
| 345 |
rows = [] |
|
| 346 |
events = [] |
|
| 347 |
for row in rs_data[:limit]: |
|
| 348 |
if row['exception'] or row.get('invalid'):
|
|
| 349 |
continue |
|
| 350 |
i += 1 |
|
| 351 |
rows.append(row) |
|
| 352 |
events.append(EventWithAct.objects.create(patient=row['enfant'], |
|
| 353 |
start_datetime=row['start_datetime'], |
|
| 354 |
end_datetime=row['end_datetime'], |
|
| 355 |
act_type=row['act_type'], |
|
| 356 |
old_rs_id=row['id'], |
|
| 357 |
room=Room(id=1), |
|
| 358 |
title=row['libelle'], |
|
| 359 |
description=row['texte'])) |
|
| 360 |
print "Rdv creation %-6d\r" % i, |
|
| 361 |
|
|
| 362 |
def batch_bulk(model, rows, limit): |
|
| 363 |
i = 0 |
|
| 364 |
while rows[i:i+limit]: |
|
| 365 |
model.objects.bulk_create(rows[i:i+limit]) |
|
| 366 |
i += limit |
|
| 367 |
def service_and_workers(events, rows): |
|
| 368 |
services = [] |
|
| 369 |
ServiceThrough = EventWithAct.services.through |
|
| 370 |
for event in events: |
|
| 371 |
services.append(ServiceThrough( |
|
| 372 |
event_id=event.event_ptr_id, |
|
| 373 |
service_id=service.id)) |
|
| 374 |
batch_bulk(ServiceThrough, services, 100) |
|
| 375 |
ParticipantThrough = EventWithAct.participants.through |
|
| 376 |
participants = [] |
|
| 377 |
for row, event in zip(rows, events): |
|
| 378 |
for worker in row['workers']: |
|
| 379 |
participants.append( |
|
| 380 |
ParticipantThrough( |
|
| 381 |
event_id=event.event_ptr_id, |
|
| 382 |
people_id=worker.people_ptr_id)) |
|
| 383 |
batch_bulk(ParticipantThrough, participants, 100) |
|
| 384 |
print 'Created %s service links' % len(services) |
|
| 385 |
print 'Created %s participants links' % len(participants) |
|
| 386 |
service_and_workers(events, rows) |
|
| 387 |
# RR |
|
| 388 |
rows = [] |
|
| 389 |
events = [] |
|
| 390 |
i = 0 |
|
| 391 |
for row in rr_data[:limit]: |
|
| 392 |
if row.get('invalid'):
|
|
| 393 |
continue |
|
| 394 |
i += 1 |
|
| 395 |
rows.append(row) |
|
| 396 |
events.append(EventWithAct.objects.create( |
|
| 397 |
patient=row['enfant'], |
|
| 398 |
start_datetime=row['start_datetime'], |
|
| 399 |
end_datetime=row['end_datetime'], |
|
| 400 |
act_type=row['act_type'], |
|
| 401 |
old_rs_id=row['id'], |
|
| 402 |
room=Room(id=1), |
|
| 403 |
title=row['libelle'], |
|
| 404 |
description=row['texte'], |
|
| 405 |
recurrence_periodicity=row['recurrence_periodicity'], |
|
| 406 |
recurrence_end_date=row['end_date'])) |
|
| 407 |
print "Rdv recurrent creation %-6d\r" % i, |
|
| 408 |
|
|
| 409 |
service_and_workers(events, rows) |
|
| 410 |
# Exceptions |
|
| 411 |
excrows = [] |
|
| 412 |
excevents = [] |
|
| 413 |
i = 0 |
|
| 414 |
for rr, event in zip(rows, events): |
|
| 415 |
for row in rr['exceptions']: |
|
| 416 |
if row.get('invalid'):
|
|
| 417 |
print 'exception invalide' |
|
| 418 |
continue |
|
| 419 |
i += 1 |
|
| 420 |
excrows.append(row) |
|
| 421 |
excevents.append(EventWithAct.objects.create( |
|
| 422 |
patient=row['enfant'], |
|
| 423 |
start_datetime=row['start_datetime'], |
|
| 424 |
end_datetime=row['end_datetime'], |
|
| 425 |
act_type=row['act_type'], |
|
| 426 |
old_rs_id=row['id'], |
|
| 427 |
room=Room(id=1), |
|
| 428 |
title=row['libelle'], |
|
| 429 |
description=row['texte'], |
|
| 430 |
exception_to=event, |
|
| 431 |
exception_date=row['date'])) |
|
| 432 |
print "Exception creation %-6d\r" % i, |
|
| 433 |
|
|
| 434 |
service_and_workers(excevents, excrows) |
|
| 435 |
finally: |
|
| 436 |
transaction.rollback() |
|
| 437 |
|
|
| 438 |
|
|
| 439 |
|
|
| 440 |
|
|
| 239 | 441 |
invalid_rs_csv.close() |
| 442 |
invalid_rr_csv.close() |
|
| 240 | 443 |
|
| 241 | 444 |
if __name__ == "__main__": |
| 242 | 445 |
main() |
Also available in: Unified diff
scripts: import of meetings works