Revision a22b9e74
Added by Benjamin Dauvergne over 12 years ago
| scripts/import_ev_reunion.py | ||
|---|---|---|
| 10 | 10 |
import logging |
| 11 | 11 |
from datetime import datetime, timedelta |
| 12 | 12 |
|
| 13 |
log_file = "./scripts/import_ev_reunion.log" |
|
| 14 |
logging.basicConfig(filename=log_file,level=logging.DEBUG) |
|
| 15 |
|
|
| 13 | 16 |
import calebasse.settings |
| 14 | 17 |
import django.core.management |
| 15 |
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned |
|
| 16 | 18 |
|
| 17 | 19 |
django.core.management.setup_environ(calebasse.settings) |
| 18 | 20 |
|
| ... | ... | |
| 26 | 28 |
|
| 27 | 29 |
# Configuration |
| 28 | 30 |
db_path = "./scripts/20121221-192258" |
| 29 |
log_file = "./scripts/import_ev_reunion.log" |
|
| 30 | 31 |
|
| 31 | 32 |
dbs = ["F_ST_ETIENNE_SESSAD_TED", "F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD"] |
| 32 | 33 |
tables = ['rs', 'ev', 'details_rs', 'details_ev'] |
| ... | ... | |
| 59 | 60 |
res[cols[i]] = data.decode('utf-8')
|
| 60 | 61 |
return res |
| 61 | 62 |
|
| 63 |
import utils |
|
| 64 |
|
|
| 65 |
worker_indexes = utils.QuerysetIndex(Worker.objects.all(), |
|
| 66 |
'old_cmpp_id', 'old_camsp_id', 'old_sessad_dys_id', 'old_sessad_ted_id') |
|
| 67 |
|
|
| 62 | 68 |
def _get_therapists(line, table_name, service): |
| 69 |
global worker_indexes |
|
| 63 | 70 |
p_ids = [] |
| 64 | 71 |
for id, detail in tables_data['details_%s' % table_name].iteritems(): |
| 65 | 72 |
if detail['%s_id' % table_name] == line['id']: |
| ... | ... | |
| 68 | 75 |
for p_id in p_ids: |
| 69 | 76 |
try: |
| 70 | 77 |
if service.name == "CMPP": |
| 71 |
participants.append(Worker.objects.get(old_cmpp_id=int(p_id)))
|
|
| 78 |
worker = worker_indexes.by_old_cmpp_id[p_id]
|
|
| 72 | 79 |
elif service.name == "CAMSP": |
| 73 |
participants.append(Worker.objects.get(old_camsp_id=int(p_id)))
|
|
| 80 |
worker = worker_indexes.by_old_camsp_id[p_id]
|
|
| 74 | 81 |
elif service.name == "SESSAD DYS": |
| 75 |
participants.append(Worker.objects.get(old_sessad_dys_id=int(p_id)))
|
|
| 82 |
worker = worker_indexes.by_old_sessad_dys_id[p_id]
|
|
| 76 | 83 |
elif service.name == "SESSAD TED": |
| 77 |
participants.append(Worker.objects.get(old_sessad_ted_id=int(p_id))) |
|
| 78 |
except Worker.DoesNotExist: |
|
| 84 |
worker = worker_indexes.by_old_sessad_ted_id[p_id] |
|
| 85 |
participants.append(worker) |
|
| 86 |
except KeyError: |
|
| 79 | 87 |
logging.warning("%s %s_id %s thera_id %s" %\
|
| 80 | 88 |
(service.name, table_name, line['id'], p_id) +\ |
| 81 | 89 |
" therapeute non trouve") |
| ... | ... | |
| 91 | 99 |
return None |
| 92 | 100 |
return tables['ev'].get(ev_id[3:]) |
| 93 | 101 |
|
| 94 |
def create_event(line, service, tables_data): |
|
| 95 |
if not Event.objects.filter(old_rs_id=line['id'], services__in=[service]): |
|
| 102 |
ServicesThrough = Event.services.through |
|
| 103 |
ParticipantsThrough = Event.participants.through |
|
| 104 |
|
|
| 105 |
event_type_4 = EventType.objects.get(id=4) |
|
| 106 |
|
|
| 107 |
def create_event(line, service, tables_data, rs_by_id, events_by_id, events, |
|
| 108 |
participants_by_id, exceptions_set): |
|
| 109 |
if line['id'] not in rs_by_id: |
|
| 96 | 110 |
# Manage exception |
| 97 | 111 |
exception_date = None |
| 98 | 112 |
exception_to = None |
| ... | ... | |
| 101 | 115 |
assert line['rr_ev_id'].startswith('ev_')
|
| 102 | 116 |
ev_id = int(line['rr_ev_id'][3:]) |
| 103 | 117 |
exception_date = _to_date(line['date_rdv']) |
| 104 |
exception_to = Event.objects.filter(old_ev_id=ev_id, |
|
| 105 |
services__in=[service]) |
|
| 118 |
exception_to = events_by_id.get(str(ev_id)) |
|
| 106 | 119 |
if exception_to: |
| 107 |
exception_to = exception_to[0] |
|
| 108 |
if Event.objects.filter(exception_date=exception_date, |
|
| 109 |
exception_to=exception_to): |
|
| 120 |
p = (exception_date, exception_to) |
|
| 121 |
if p in exceptions_set: |
|
| 110 | 122 |
logging.info("rs_id %s is duplicated" % line["id"])
|
| 111 | 123 |
return |
| 124 |
exceptions_set.add(p) |
|
| 112 | 125 |
else: |
| 113 | 126 |
exception_error = True |
| 114 |
event_type = EventType.objects.get(id=4) |
|
| 115 | 127 |
start_datetime = datetime.strptime( |
| 116 | 128 |
line['date_rdv'][:-13] + ' ' + line['heure'][11:-4], |
| 117 | 129 |
"%Y-%m-%d %H:%M:%S") |
| ... | ... | |
| 120 | 132 |
minutes=int(line['duree'][14:-7]), |
| 121 | 133 |
) |
| 122 | 134 |
participants = _get_therapists(line, 'rs', service) |
| 135 |
if not participants: |
|
| 136 |
logging.error("%s rs_id %s exception aucun participant importable" % (service.name,
|
|
| 137 |
line["id"])) |
|
| 138 |
return |
|
| 123 | 139 |
if not exception_error: |
| 124 |
event = Event.objects.create(
|
|
| 140 |
event = Event( |
|
| 125 | 141 |
title=line['libelle'][:60], |
| 126 | 142 |
description=line['texte'], |
| 127 |
event_type=event_type, |
|
| 143 |
event_type=event_type_4,
|
|
| 128 | 144 |
start_datetime=start_datetime, |
| 129 | 145 |
end_datetime=end_datetime, |
| 130 | 146 |
old_rs_id=line['id'], |
| 131 |
exception_to = exception_to,
|
|
| 147 |
exception_to = Event(id=exception_to),
|
|
| 132 | 148 |
exception_date = exception_date |
| 133 | 149 |
) |
| 134 |
event.services = [service] |
|
| 135 |
event.participants = participants |
|
| 136 |
event.save() |
|
| 150 |
participants_by_id[line['id']] = participants |
|
| 151 |
events.append(event) |
|
| 137 | 152 |
else: |
| 138 | 153 |
logging.error("%s rs_id %s exception pas d'ev trouve %s" % (service.name,
|
| 139 | 154 |
line["id"], line)) |
| 140 | 155 |
|
| 141 |
def create_recurrent_event(line, service, tables_data): |
|
| 156 |
def create_recurrent_event(line, service, tables_data, events, participants_by_id):
|
|
| 142 | 157 |
if not Event.objects.filter(old_ev_id=line['id'], services__in=[service]): |
| 143 | 158 |
start_date = _to_date(line['date_debut']) |
| 144 | 159 |
end_date = _to_date(line['date_fin']) |
| ... | ... | |
| 160 | 175 |
(service.name, line['id'], line['rythme'])) |
| 161 | 176 |
|
| 162 | 177 |
participants = _get_therapists(line, 'ev', service) |
| 163 |
event_type = EventType.objects.get(id=4) |
|
| 178 |
if not participants: |
|
| 179 |
logging.error("%s ev_id %s exception aucun participant importable" % (service.name,
|
|
| 180 |
line["id"])) |
|
| 181 |
return |
|
| 164 | 182 |
try: |
| 165 |
event = Event.objects.create(
|
|
| 183 |
event = Event( |
|
| 166 | 184 |
start_datetime=start_datetime, |
| 167 | 185 |
end_datetime=end_datetime, |
| 168 |
event_type=event_type, |
|
| 186 |
event_type=event_type_4,
|
|
| 169 | 187 |
title=line['libelle'], |
| 170 | 188 |
old_ev_id=line['id'], |
| 171 | 189 |
description=line['note'], |
| 172 | 190 |
recurrence_periodicity=recurrence_periodicity, |
| 173 | 191 |
recurrence_end_date=end_date) |
| 174 |
event.services = [service] |
|
| 175 |
event.participants = participants |
|
| 176 |
event.save() |
|
| 192 |
participants_by_id[line['id']] = participants |
|
| 193 |
events.append(event) |
|
| 177 | 194 |
except django.core.exceptions.ValidationError, e: |
| 178 | 195 |
logging.error(service.name + ' ev recurrence non valide %s %s' % (line, e)) |
| 179 | 196 |
|
| 180 | 197 |
@transaction.commit_on_success |
| 181 | 198 |
def main(): |
| 182 |
logging.basicConfig(filename=log_file,level=logging.DEBUG) |
|
| 183 | 199 |
for db in dbs: |
| 184 | 200 |
if "F_ST_ETIENNE_CMPP" == db: |
| 185 | 201 |
service = Service.objects.get(name="CMPP") |
| ... | ... | |
| 201 | 217 |
tables_data[table][data['id']] = data |
| 202 | 218 |
csvfile.close() |
| 203 | 219 |
|
| 220 |
total = 0 |
|
| 221 |
events = [] |
|
| 222 |
participants = dict() |
|
| 204 | 223 |
for id, line in tables_data['ev'].iteritems(): |
| 205 | 224 |
if not _is_timetable(line): |
| 206 |
create_recurrent_event(line, service, tables_data) |
|
| 225 |
total += 1 |
|
| 226 |
create_recurrent_event(line, service, tables_data, events, |
|
| 227 |
participants) |
|
| 207 | 228 |
|
| 229 |
# create events |
|
| 230 |
logging.info('%s creating %s events on %s',
|
|
| 231 |
service.name, len(events), total) |
|
| 232 |
Event.objects.bulk_create(events) |
|
| 233 |
participants_through = [] |
|
| 234 |
# get their ids |
|
| 235 |
events_by_id = dict(Event.objects \ |
|
| 236 |
.exclude(event_type_id=1) \ |
|
| 237 |
.filter(services__isnull=True, old_ev_id__isnull=False) \ |
|
| 238 |
.values_list('old_ev_id', 'id'))
|
|
| 239 |
for line_id, participants in participants.iteritems(): |
|
| 240 |
for participant in participants: |
|
| 241 |
pt = ParticipantsThrough(event_id=events_by_id[line_id], |
|
| 242 |
people_id=participant.people_ptr_id) |
|
| 243 |
participants_through.append(pt) |
|
| 244 |
ParticipantsThrough.objects.bulk_create(participants_through) |
|
| 245 |
ServicesThrough.objects.bulk_create([ |
|
| 246 |
ServicesThrough(service_id=service.id, event_id=event_id) for event_id in events_by_id.values()]) |
|
| 247 |
participants = {}
|
|
| 248 |
|
|
| 249 |
total = 0 |
|
| 250 |
events = [] |
|
| 251 |
participants = dict() |
|
| 252 |
exceptions_set = set() |
|
| 253 |
# get their ids |
|
| 254 |
rs_by_id = dict(Event.objects \ |
|
| 255 |
.exclude(event_type_id=1) \ |
|
| 256 |
.filter(services=service, old_rs_id__isnull=False) \ |
|
| 257 |
.values_list('old_rs_id', 'id'))
|
|
| 208 | 258 |
for id, line in tables_data['rs'].iteritems(): |
| 209 | 259 |
if (not line['enfant_id'] or not int(line['enfant_id'])) \ |
| 210 | 260 |
and not _is_timetable(line) \ |
| 211 | 261 |
and (not _is_timetable(_get_ev(tables_data, line))): |
| 212 |
create_event(line, service, tables_data) |
|
| 262 |
total += 1 |
|
| 263 |
create_event(line, service, tables_data, rs_by_id, events_by_id, |
|
| 264 |
events, participants, exceptions_set) |
|
| 265 |
# create events |
|
| 266 |
logging.info('%s creating %s events on %s',
|
|
| 267 |
service.name, len(events), total) |
|
| 268 |
Event.objects.bulk_create(events) |
|
| 269 |
participants_through = [] |
|
| 270 |
# get their ids |
|
| 271 |
events_by_id = dict(Event.objects \ |
|
| 272 |
.exclude(event_type_id=1) \ |
|
| 273 |
.filter(services__isnull=True, old_rs_id__isnull=False) \ |
|
| 274 |
.values_list('old_rs_id', 'id'))
|
|
| 275 |
for line_id, participants in participants.iteritems(): |
|
| 276 |
for participant in participants: |
|
| 277 |
pt = ParticipantsThrough(event_id=events_by_id[line_id], |
|
| 278 |
people_id=participant.people_ptr_id) |
|
| 279 |
participants_through.append(pt) |
|
| 280 |
ParticipantsThrough.objects.bulk_create(participants_through) |
|
| 281 |
ServicesThrough.objects.bulk_create([ |
|
| 282 |
ServicesThrough(service_id=service.id, event_id=event_id) for event_id in events_by_id.values()]) |
|
| 283 |
participants = {}
|
|
| 284 |
|
|
| 213 | 285 |
|
| 214 | 286 |
if __name__ == "__main__": |
| 215 | 287 |
main() |
Also available in: Unified diff
scripts: bulk create events in import_ev_reunion
limit queries by using python dict as indexes.