calebasse/scripts/import_some_acts.py @ 48091f78
| ee1df330 | Benjamin Dauvergne | # -*- coding: utf-8 -*-
|
|
#!/usr/bin/env python
|
|||
import os
|
|||
import csv
|
|||
from datetime import datetime, date
|
|||
import calebasse.settings
|
|||
import django.core.management
|
|||
django.core.management.setup_environ(calebasse.settings)
|
|||
from django.db import transaction
|
|||
from calebasse.dossiers.models import PatientRecord
|
|||
from calebasse.personnes.models import Worker
|
|||
from calebasse.ressources.models import Service
|
|||
from calebasse.ressources.models import ActType
|
|||
from calebasse.actes.models import Act, ActValidationState
|
|||
from import_dossiers import map_cs
|
|||
# Configuration
|
|||
db_path = "./scripts/20130104-213225"
|
|||
dbs = ["F_ST_ETIENNE_SESSAD_TED", "F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD"]
|
|||
# dbs = ["F_ST_ETIENNE_CAMSP"]
|
|||
def _to_datetime(str_date):
|
|||
if not str_date:
|
|||
return None
|
|||
return datetime.strptime(str_date[:19], "%Y-%m-%d %H:%M:%S")
|
|||
def _to_date(str_date):
|
|||
dt = _to_datetime(str_date)
|
|||
return dt and dt.date()
|
|||
def _to_time(str_date):
|
|||
dt = _to_datetime(str_date)
|
|||
return dt and dt.time()
|
|||
def _to_duration(str_date):
|
|||
dt = _to_datetime(str_date)
|
|||
if dt is None:
|
|||
return None
|
|||
return dt - datetime(1900, 01, 01, 0, 0)
|
|||
def _to_int(str_int):
|
|||
if not str_int:
|
|||
return None
|
|||
return int(str_int)
|
|||
def _get_dict(cols, line):
|
|||
""""""
|
|||
res = {}
|
|||
for i, data in enumerate(line):
|
|||
res[cols[i]] = data.decode('utf-8')
|
|||
return res
|
|||
def batch_delete(qs, limit):
|
|||
count = qs.count()
|
|||
i = 0
|
|||
while i < count:
|
|||
ids = qs[i:i+limit].values_list('pk', flat=True)
|
|||
qs.filter(pk__in=ids).delete()
|
|||
i += limit
|
|||
PERIOD_FAURE_NOUS = {1 : 1,
|
|||
2 : 2,
|
|||
3 : 3,
|
|||
4 : 4,
|
|||
5: 6,
|
|||
6: 7,
|
|||
7: 8,
|
|||
8: 9,
|
|||
9: None,
|
|||
10: 10,
|
|||
12: 11,
|
|||
13: 12,
|
|||
}
|
|||
JOURS = {1: 'lundi',
|
|||
2: 'mardi',
|
|||
3: 'mercredi',
|
|||
4: 'jeudi',
|
|||
5: 'vendredi'
|
|||
}
|
|||
dic_worker = {}
|
|||
def load_csv2(db, name, offset=0, limit=9999999, id_column=0):
|
|||
csvfile = open(os.path.join(db_path, db, name + '.csv'), 'rb')
|
|||
csvlines = csv.reader(csvfile, delimiter=';', quotechar='|')
|
|||
cols = csvlines.next()
|
|||
yield cols
|
|||
i = 0
|
|||
for line in csvlines:
|
|||
if not (offset <= int(line[id_column]) < offset+limit):
|
|||
continue
|
|||
yield _get_dict(cols, line)
|
|||
i += 1
|
|||
csvfile.close()
|
|||
def load_csv(db, name, offset=0, limit=9999999, id_column=0):
|
|||
records = []
|
|||
idx = {}
|
|||
csvfile = open(os.path.join(db_path, db, name + '.csv'), 'rb')
|
|||
csvlines = csv.reader(csvfile, delimiter=';', quotechar='|')
|
|||
cols = csvlines.next()
|
|||
i = 0
|
|||
for line in csvlines:
|
|||
if not (offset <= int(line[id_column]) < offset+limit):
|
|||
continue
|
|||
data = _get_dict(cols, line)
|
|||
records.append(data)
|
|||
idx[data['id']] = i
|
|||
i += 1
|
|||
csvfile.close()
|
|||
return records, idx, cols
|
|||
def add_invalid(d, reason):
|
|||
d.setdefault('invalid', '')
|
|||
if d['invalid']:
|
|||
d['invalid'] += ', '
|
|||
d['invalid'] += reason
|
|||
log = open('import_some_acts-%s.log' % datetime.now().isoformat(), 'w+')
|
|||
@transaction.commit_on_success
|
|||
def main():
|
|||
""" """
|
|||
workers = Worker.objects.all()
|
|||
for db in dbs:
|
|||
workers_idx = {}
|
|||
act_types_idx = {}
|
|||
if "F_ST_ETIENNE_CMPP" == db:
|
|||
service = Service.objects.get(name="CMPP")
|
|||
elif "F_ST_ETIENNE_CAMSP" == db:
|
|||
service = Service.objects.get(name="CAMSP")
|
|||
elif "F_ST_ETIENNE_SESSAD_TED" == db:
|
|||
service = Service.objects.get(name="SESSAD TED")
|
|||
elif "F_ST_ETIENNE_SESSAD" == db:
|
|||
service = Service.objects.get(name="SESSAD DYS")
|
|||
print '===', service.name, '==='
|
|||
print datetime.now()
|
|||
print >>log, datetime.now(), '===', service.name, '==='
|
|||
# load workers mapping
|
|||
worker_reverse_idx = {}
|
|||
for i, worker in enumerate(workers):
|
|||
if service.name == 'CMPP':
|
|||
j = worker.old_cmpp_id
|
|||
elif service.name == 'CAMSP':
|
|||
j = worker.old_camsp_id
|
|||
elif service.name == 'SESSAD DYS':
|
|||
j = worker.old_sessad_dys_id
|
|||
elif service.name == 'SESSAD TED':
|
|||
j = worker.old_sessad_ted_id
|
|||
else:
|
|||
print "service inconnu!!!"
|
|||
exit(0)
|
|||
if j:
|
|||
workers_idx[j] = worker
|
|||
worker_reverse_idx[worker] = j
|
|||
# load act_type mapping
|
|||
act_types = ActType.objects.for_service(service)
|
|||
act_type_id_not_found = set()
|
|||
for i, act_type in enumerate(act_types):
|
|||
j = act_type.old_id
|
|||
if j:
|
|||
act_types_idx[j] = act_type
|
|||
else:
|
|||
act_type_id_not_found.add(act_type)
|
|||
def set_act_type(row, not_found=None):
|
|||
act_type_id = row['type_acte']
|
|||
if act_type_id == '0':
|
|||
add_invalid(row, 'no act_id=>not importable')
|
|||
elif act_type_id in act_types_idx:
|
|||
row['act_type'] = act_types_idx[act_type_id]
|
|||
else:
|
|||
add_invalid(row, 'act_type not found %s' % act_type_id)
|
|||
if not_found:
|
|||
not_found.add(act_type_id)
|
|||
def handle_details2(data, idx, details, id_key):
|
|||
for detail in details:
|
|||
i = int(detail[id_key])
|
|||
thera_id = detail['thera_id']
|
|||
if i not in idx:
|
|||
continue
|
|||
row = data[idx[i]]
|
|||
if thera_id in workers_idx:
|
|||
ws = row.setdefault('workers', set())
|
|||
theras = row.setdefault('theras', set())
|
|||
ws.add(workers_idx[thera_id])
|
|||
theras.add(thera_id)
|
|||
else:
|
|||
add_invalid(row, 'unknown thera_id %s' % thera_id)
|
|||
print "%s - Nombre de types d'actes : %d" % (service.name, len(act_types))
|
|||
print "%s - Liste des types d'actes sans id : %s" % (service.name, str(act_type_id_not_found))
|
|||
# loading dossiers idx
|
|||
enfant_idx = {}
|
|||
for enfant in PatientRecord.objects.filter(service=service):
|
|||
enfant_idx[enfant.old_id] = enfant
|
|||
def set_enfant(row, not_found=None):
|
|||
# connect enfant
|
|||
enfant_id = row['enfant_id']
|
|||
if enfant_id == '0':
|
|||
add_invalid(row, 'no enfant_id=>not an appointment')
|
|||
row['event'] = True
|
|||
elif enfant_id in enfant_idx:
|
|||
row['enfant'] = enfant_idx[enfant_id]
|
|||
else:
|
|||
add_invalid(row, 'enfant_id not found %s' % enfant_id)
|
|||
if not_found:
|
|||
not_found.add(enfant_id)
|
|||
acts_ids = set([
|
|||
193815,
|
|||
173334,
|
|||
171872,
|
|||
193506,
|
|||
182039,
|
|||
166806,
|
|||
183054,
|
|||
181050,
|
|||
172301,
|
|||
193914,
|
|||
186876,
|
|||
161111,
|
|||
])
|
|||
rows = load_csv2(db, 'actes')
|
|||
rows.next()
|
|||
loaded_rows = []
|
|||
for row in rows:
|
|||
if int(row['id']) not in acts_ids:
|
|||
continue
|
|||
row.setdefault('invalid', '')
|
|||
row.setdefault('workers', set())
|
|||
row.setdefault('theras', set())
|
|||
row['date'] = _to_date(row['date_acte'])
|
|||
row['time'] = _to_time(row['heure'])
|
|||
row['duration'] = _to_duration(row['duree'])
|
|||
row['is_billed'] = row['marque'] == '1'
|
|||
row['validation_locked'] = row['date'] < date(2013, 1, 3)
|
|||
set_enfant(row)
|
|||
set_act_type(row)
|
|||
row['state'] = map_cs[service.name].get(row['cs'], 'VALIDE')
|
|||
loaded_rows.append(row)
|
|||
total = 0
|
|||
for row in loaded_rows:
|
|||
if row['invalid']:
|
|||
print >>log, datetime.now(), 'row invalid', row
|
|||
continue
|
|||
print >>log, datetime.now(), 'act', row['id'], 'imported'
|
|||
# act = Act.objects.create(
|
|||
# old_id=row['id'],
|
|||
# date=row['date'],
|
|||
# time=row['time'],
|
|||
# _duration=row['duration'].seconds // 60,
|
|||
# is_billed=row['is_billed'],
|
|||
# act_type=row['act_type'],
|
|||
# patient=row['enfant'],
|
|||
# validation_locked=row['validation_locked'])
|
|||
# act.doctors = row['workers']
|
|||
# ActValidationState.objects.create(act=act, state_name=row['state'], previous_state=None)
|
|||
total += 1
|
|||
print >>log, datetime.now(), 'created', total, 'new acts'
|
|||
raise Exception()
|
|||
if __name__ == "__main__":
|
|||
main()
|