Project

General

Profile

« Previous | Next » 

Revision bfe4851b

Added by Benjamin Dauvergne (en congé) about 12 years ago

scripts: import_acts_id.py first version

View differences:

scripts/import_actes_ids.py
1
# -*- coding: utf-8 -*-
2
#!/usr/bin/env python
3

  
4
import os
5
import csv
6
from collections import defaultdict
7

  
8
from datetime import datetime, date
9

  
10
import calebasse.settings
11
import django.core.management
12

  
13
django.core.management.setup_environ(calebasse.settings)
14

  
15
from django.db import transaction
16
from calebasse.agenda.models import EventWithAct
17
from calebasse.dossiers.models import PatientRecord
18
from calebasse.personnes.models import Worker
19
from calebasse.ressources.models import Service
20
from calebasse.ressources.models import ActType
21
from calebasse.actes.models import Act, ActValidationState
22

  
23
from import_dossiers import map_cs
24

  
25
# Configuration
26
db_path = "./scripts/20130104-213225"
27

  
28
dbs = ["F_ST_ETIENNE_SESSAD_TED", "F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD"]
29

  
30
def _to_datetime(str_date):
31
    if not str_date:
32
        return None
33
    return datetime.strptime(str_date[:19], "%Y-%m-%d %H:%M:%S")
34

  
35
def _to_date(str_date):
36
    dt = _to_datetime(str_date)
37
    return dt and dt.date()
38

  
39
def _to_time(str_date):
40
    dt = _to_datetime(str_date)
41
    return dt and dt.time()
42

  
43
def _to_duration(str_date):
44
    dt = _to_datetime(str_date)
45
    if dt is None:
46
        return None
47
    return dt - datetime(1900, 01, 01, 0, 0)
48

  
49
def _to_int(str_int):
50
    if not str_int:
51
        return None
52
    return int(str_int)
53

  
54
def _get_dict(cols, line):
55
    """"""
56
    res = {}
57
    for i, data in enumerate(line):
58
        res[cols[i]] = data.decode('utf-8')
59
    return res
60

  
61
def batch_delete(qs, limit):
62
    count = qs.count()
63
    i = 0
64
    while i < count:
65
        ids = qs[i:i+limit].values_list('pk', flat=True)
66
        qs.filter(pk__in=ids).delete()
67
        i += limit
68

  
69

  
70
PERIOD_FAURE_NOUS = {1 : 1,
71
2 : 2,
72
3 : 3,
73
4 : 4,
74
5: 6,
75
6: 7,
76
7: 8,
77
8: 9,
78
9: None,
79
10: 10,
80
12: 11,
81
13: 12,
82
}
83

  
84
JOURS = {1: 'lundi',
85
2: 'mardi',
86
3: 'mercredi',
87
4: 'jeudi',
88
5: 'vendredi'
89
}
90

  
91
dic_worker = {}
92

  
93
def load_csv2(db, name, offset=0, limit=9999999, id_column=0):
94
    csvfile = open(os.path.join(db_path, db, name + '.csv'), 'rb')
95
    csvlines = csv.reader(csvfile, delimiter=';', quotechar='|')
96
    cols = csvlines.next()
97
    yield cols
98
    i = 0
99
    for line in csvlines:
100
        if not (offset <= int(line[id_column]) < offset+limit):
101
            continue
102
        yield _get_dict(cols, line)
103
        i += 1
104
    csvfile.close()
105

  
106
def load_csv(db, name, offset=0, limit=9999999, id_column=0):
107
    records = []
108
    idx = {}
109

  
110
    csvfile = open(os.path.join(db_path, db, name + '.csv'), 'rb')
111
    csvlines = csv.reader(csvfile, delimiter=';', quotechar='|')
112
    cols = csvlines.next()
113
    i = 0
114
    for line in csvlines:
115
        if not (offset <= int(line[id_column]) < offset+limit):
116
            continue
117
        data = _get_dict(cols, line)
118
        records.append(data)
119
        idx[data['id']] = i
120
        i += 1
121
    csvfile.close()
122
    return records, idx, cols
123

  
124
def add_invalid(d, reason):
125
    d.setdefault('invalid', '')
126
    if d['invalid']:
127
        d['invalid'] += ', '
128
    d['invalid'] += reason
129

  
130
log = open('import_actes_ids.log', 'a')
131

  
132
@transaction.commit_on_success
133
def main():
134
    """ """
135

  
136
    workers = Worker.objects.all()
137
    for db in dbs:
138
        workers_idx = {}
139
        act_types_idx = {}
140
        if "F_ST_ETIENNE_CMPP" == db:
141
            service = Service.objects.get(name="CMPP")
142
        elif "F_ST_ETIENNE_CAMSP" == db:
143
            service = Service.objects.get(name="CAMSP")
144
        elif "F_ST_ETIENNE_SESSAD_TED" == db:
145
            service = Service.objects.get(name="SESSAD TED")
146
        elif "F_ST_ETIENNE_SESSAD" == db:
147
            service = Service.objects.get(name="SESSAD DYS")
148

  
149
        print '===', service.name, '==='
150
        print datetime.now()
151
        print >>log, datetime.now(), '===', service.name, '==='
152

  
153
        # load workers mapping
154
        for i, worker in enumerate(workers):
155
            if service.name == 'CMPP':
156
                j = worker.old_cmpp_id
157
            elif service.name == 'CAMSP':
158
                j = worker.old_camsp_id
159
            elif service.name == 'SESSAD DYS':
160
                j = worker.old_sessad_dys_id
161
            elif service.name == 'SESSAD TED':
162
                j = worker.old_sessad_ted_id
163
            else:
164
                print "service inconnu!!!"
165
                exit(0)
166
            if j:
167
                workers_idx[j] = worker
168
        # load act_type mapping
169
        act_types = ActType.objects.for_service(service)
170
        act_type_id_not_found = set()
171
        for i, act_type in enumerate(act_types):
172
            j = act_type.old_id
173
            if j:
174
                act_types_idx[j] = act_type
175
            else:
176
                act_type_id_not_found.add(act_type)
177

  
178
        def set_act_type(row, not_found=None):
179
            act_type_id = row['type_acte']
180
            if act_type_id == '0':
181
                add_invalid(row, 'no act_id=>not importable')
182
            elif act_type_id in act_types_idx:
183
                row['act_type'] = act_types_idx[act_type_id]
184
            else:
185
                add_invalid(row, 'act_type not found %s' % act_type_id)
186
                if not_found:
187
                    not_found.add(act_type_id)
188

  
189
        def handle_details2(data, idx, details, id_key):
190
            for detail in details:
191
                i = detail[id_key]
192
                thera_id = detail['thera_id']
193
                if i not in idx:
194
                    continue
195
                row = data[idx[i]]
196
                if thera_id in workers_idx:
197
                    workers = row.setdefault('workers', set())
198
                    workers.add(workers_idx[thera_id])
199
                else:
200
                    add_invalid(row, 'unknown thera_id %s' % thera_id)
201

  
202
        print "%s - Nombre de types d'actes : %d" % (service.name, len(act_types))
203
        print "%s - Liste des types d'actes sans id : %s" % (service.name, str(act_type_id_not_found))
204

  
205
        # loading dossiers idx
206
        enfant_idx = {}
207
        for enfant in PatientRecord.objects.filter(service=service):
208
            enfant_idx[enfant.old_id] = enfant
209

  
210
        def set_enfant(row, not_found=None):
211
            # connect enfant
212
            enfant_id = row['enfant_id']
213
            if enfant_id == '0':
214
                add_invalid(row, 'no enfant_id=>not an appointment')
215
                row['event'] = True
216
            elif enfant_id in enfant_idx:
217
                row['enfant'] = enfant_idx[enfant_id]
218
            else:
219
                add_invalid(row, 'enfant_id not found %s' % enfant_id)
220
                if not_found:
221
                    not_found.add(enfant_id)
222

  
223
        # Clean act for this service
224
        notfound = []
225
        for year in range(1990, 2014):
226
            # create index of already imported acts
227
            act_idx = defaultdict(lambda: [])
228
            for act in Act.objects.filter(date__year=year, patient__service=service):
229
                key = (act.date, act.time, act.patient.old_id, act.act_type.old_id)
230
                act_idx[key].append((act.id, bool(act.old_id)))
231

  
232
            rows = load_csv2(db, 'actes')
233
            cols = rows.next()
234
            total = 0
235
            for row in rows:
236
                row.setdefault('invalid', '')
237
                row.setdefault('workers', set())
238
                row['date'] = _to_date(row['date_acte'])
239
                if row['date'].year == year:
240
                    continue
241
                row['time'] = _to_time(row['heure'])
242
                row['duration'] = _to_duration(row['duree'])
243
                row['is_billed'] = row['marque'] == '1'
244
                row['validation_locked'] = row['date'] < date(2013, 1, 3)
245
                set_enfant(row)
246
                set_act_type(row)
247
                row['state'] = map_cs[service.name].get(row['cs'], 'VALIDE')
248
                key = (row['date'], row['time'], row['enfant_id'], row['type_acte'])
249
                if key in act_idx:
250
                    acts = act_idx[key]
251
                    assert len(acts) != 0
252
                    if len(acts) > 1:
253
                        print >>log, datetime.now(), 'found more than one act for', row, ':', acts
254
                        continue
255
                    else:
256
                        act_id, has_old_id = acts[0]
257
                        if not has_old_id:
258
                            Act.objects.filter(id=act_id).update(old_id=row['id'])
259
                            total += 1
260
                            print >>log, datetime.now(), 'imported old_id', row['id'], 'to Act(id=', act_id, ')'
261
            print >>log, datetime.now(), 'Fixed', total, 'acts for', year
262
        print >>log, datetime.now(), 'Found', len(notfound), 'acts not imported'
263

  
264
        cols.extend(['workers','invalid'])
265
        actes_details = load_csv2(db, 'details_actes')
266
        actes_details.next()
267
        notfound_idx = dict((i, row['id']) for i, row in enumerate(notfound))
268
        handle_details2(notfound, notfound_idx, actes_details, 'acte_id')
269
        for row in notfound:
270
            if row['invalid']:
271
                print >>log, datetime.now(), 'row invalid', row
272
                continue
273
            act = Act.objects.create(
274
                    old_id=row['id'],
275
                    date=row['date'],
276
                    time=row['time'],
277
                    _duration=row['duration'].seconds // 60,
278
                    is_billed=row['is_billed'],
279
                    act_type=row['act_type'],
280
                    patient=row['enfant'],
281
                    validation_locked=row['validation_locked'],
282
                    parent_event=row['parent_event'])
283
            act.doctors = row['workers']
284
            ActValidationState.objects.create(act=act, state_name=row['state'], previous_state=None)
285
    raise Exception('donothing')
286

  
287
if __name__ == "__main__":
288
    main()

Also available in: Unified diff