0001-admin-add-anonymise-action.patch
| wcs/admin/forms.py | ||
|---|---|---|
| 27 | 27 |
except ImportError: |
| 28 | 28 |
M2Crypto = None |
| 29 | 29 | |
| 30 |
import datetime |
|
| 30 | 31 |
import tarfile |
| 31 | 32 |
import time |
| 32 | 33 |
from cStringIO import StringIO |
| ... | ... | |
| 206 | 207 | |
| 207 | 208 |
class FormDefPage(Directory): |
| 208 | 209 |
_q_exports = ['', 'fields', 'delete', 'duplicate', 'export', |
| 209 |
'archive', 'invite', 'enable', 'workflow', 'category',
|
|
| 210 |
'role', ('workflow-options', 'workflow_options'),
|
|
| 210 |
'anonymise', 'archive', 'invite', 'enable', 'workflow',
|
|
| 211 |
'category', 'role', ('workflow-options', 'workflow_options'),
|
|
| 211 | 212 |
('workflow-status-remapping', 'workflow_status_remapping'),
|
| 212 | 213 |
'roles', 'title', 'options', ('acl-read', 'acl_read')]
|
| 213 | 214 | |
| ... | ... | |
| 386 | 387 |
r += htmltext('<li><a href="duplicate">%s</a></li>') % _('Duplicate')
|
| 387 | 388 |
if ET: |
| 388 | 389 |
r += htmltext('<li><a href="export">%s</a></li>') % _('Export')
|
| 390 |
r += htmltext('<li><a href="anonymise">%s</a></li>') % _('Anonymise forms')
|
|
| 389 | 391 |
if not (get_publisher().has_site_option('postgresql') and get_cfg('postgresql', {})):
|
| 390 | 392 |
r += htmltext('<li><a href="archive">%s</a></li>') % _('Archive')
|
| 391 | 393 |
if self.formdef.roles: |
| ... | ... | |
| 847 | 849 |
'attachment; filename=%s-archive.wcs' % self.formdef.url_name) |
| 848 | 850 |
return job.file_content |
| 849 | 851 | |
| 852 |
def anonymise(self): |
|
| 853 |
if get_request().form.get('job'):
|
|
| 854 |
return self.anonymise_processing() |
|
| 855 | ||
| 856 |
endpoints = [] |
|
| 857 |
for status in self.formdef.workflow.get_endpoint_status(): |
|
| 858 |
endpoints.append((status.id, status.name)) |
|
| 859 | ||
| 860 |
form = Form(enctype='multipart/form-data') |
|
| 861 |
form.add(DateWidget, 'before_request_date', |
|
| 862 |
title=_('Requests ended before'),
|
|
| 863 |
value=datetime.date.today() - datetime.timedelta(30), |
|
| 864 |
required=True) |
|
| 865 |
form.add(CheckboxesWidget, 'endpoints', title=_('Status of the forms to anonymise'),
|
|
| 866 |
value=[endpoint[0] for endpoint in endpoints], |
|
| 867 |
elements=endpoints, |
|
| 868 |
inline=False, |
|
| 869 |
required=True) |
|
| 870 |
form.add_submit('submit', _('Submit'))
|
|
| 871 |
form.add_submit('cancel', _('Cancel'))
|
|
| 872 | ||
| 873 |
if form.get_widget('cancel').parse():
|
|
| 874 |
return redirect('.')
|
|
| 875 |
if not form.is_submitted() or form.has_errors(): |
|
| 876 |
get_response().breadcrumb.append(('anonymise', _('Anonymise')))
|
|
| 877 |
html_top('forms', title=_('Anonymise Forms'))
|
|
| 878 |
r = TemplateIO(html=True) |
|
| 879 |
r += htmltext('<h2>%s</h2>') % _('Anonymise Forms')
|
|
| 880 |
r += htmltext('<p>%s</p>' % _("You are about to irrevocably anonymise forms."))
|
|
| 881 |
r += form.render() |
|
| 882 |
return r.getvalue() |
|
| 883 |
else: |
|
| 884 |
return self.anonymise_submit(form) |
|
| 885 | ||
| 886 |
def anonymise_submit(self, form): |
|
| 887 | ||
| 888 |
class Anonymiser: |
|
| 889 | ||
| 890 |
def __init__(self, formdef, status_ids, before_date): |
|
| 891 |
self.formdef = formdef |
|
| 892 |
self.status_ids = ["wf-%s" % id for id in status_ids] |
|
| 893 |
self.before_date = before_date |
|
| 894 | ||
| 895 |
def anonymise(self, job=None): |
|
| 896 |
for formdata in self.formdef.data_class().select(): |
|
| 897 |
if formdata.anonymised: |
|
| 898 |
continue |
|
| 899 |
if formdata.status not in self.status_ids: |
|
| 900 |
continue |
|
| 901 |
if (formdata.evolution and formdata.evolution[-1].time >= self.before_date) \ |
|
| 902 |
or (formdata.receipt_time >= self.before_date): |
|
| 903 |
continue |
|
| 904 |
formdata.anonymise() |
|
| 905 | ||
| 906 |
before_date = form.get_widget('before_request_date').parse()
|
|
| 907 |
before_date = time.strptime(before_date, misc.date_format()) |
|
| 908 |
status_ids = form.get_widget('endpoints').parse()
|
|
| 909 |
count = self.formdef.data_class().count() |
|
| 910 |
anonymiser = Anonymiser(self.formdef, status_ids, before_date) |
|
| 911 |
if count > 100: # Arbitrary threshold |
|
| 912 |
job = get_response().add_after_job( |
|
| 913 |
str(N_('Anonymising forms')),
|
|
| 914 |
anonymiser.anonymise) |
|
| 915 |
return redirect('anonymise?job=%s' % job.id)
|
|
| 916 |
else: |
|
| 917 |
anonymiser.anonymise() |
|
| 918 | ||
| 919 |
return redirect('.')
|
|
| 920 | ||
| 921 |
def anonymise_processing(self): |
|
| 922 |
try: |
|
| 923 |
job = AfterJob.get(get_request().form.get('job'))
|
|
| 924 |
except KeyError: |
|
| 925 |
return redirect('.')
|
|
| 926 | ||
| 927 |
html_top('forms', title=_('Anonymising'))
|
|
| 928 |
r = TemplateIO(html=True) |
|
| 929 |
r += get_session().display_message() |
|
| 930 |
get_response().add_javascript(['jquery.js', 'afterjob.js']) |
|
| 931 |
r += htmltext('<dl class="job-status">')
|
|
| 932 |
r += htmltext('<dt>')
|
|
| 933 |
r += _(job.label) |
|
| 934 |
r += htmltext('</dt>')
|
|
| 935 |
r += htmltext('<dd>')
|
|
| 936 |
r += htmltext('<span class="afterjob" id="%s">') % job.id
|
|
| 937 |
r += _(job.status) |
|
| 938 |
r += htmltext('</span>')
|
|
| 939 |
r += htmltext('</dd>')
|
|
| 940 |
r += htmltext('</dl>')
|
|
| 941 | ||
| 942 |
r += htmltext('<div class="done">')
|
|
| 943 |
r += htmltext('<a href="./">%s</a>') % _('Back')
|
|
| 944 |
r += htmltext('</div>')
|
|
| 945 |
return r.getvalue() |
|
| 946 | ||
| 850 | 947 |
def invite(self): |
| 851 | 948 |
if not self.formdef.roles: |
| 852 | 949 |
return template.error_page( |
| wcs/backoffice/root.py | ||
|---|---|---|
| 423 | 423 |
fields.append(FakeField('user-label', 'user-label', _('User Label')))
|
| 424 | 424 |
fields.extend(self.formdef.fields) |
| 425 | 425 |
fields.append(FakeField('status', 'status', _('Status')))
|
| 426 |
fields.append(FakeField('anonymised', 'anonymised', _('Anonymised')))
|
|
| 426 | 427 | |
| 427 | 428 |
return fields |
| 428 | 429 | |
| wcs/fields.py | ||
|---|---|---|
| 118 | 118 |
prefill = None |
| 119 | 119 |
store_display_value = None |
| 120 | 120 | |
| 121 |
anonymise = True |
|
| 121 | 122 |
stats = None |
| 122 | 123 | |
| 123 | 124 |
def __init__(self, **kwargs): |
| ... | ... | |
| 554 | 555 | |
| 555 | 556 |
widget_class = CheckboxWidget |
| 556 | 557 |
required = False |
| 558 |
anonymise = False |
|
| 557 | 559 | |
| 558 | 560 |
def perform_more_widget_changes(self, form, kwargs, edit = True): |
| 559 | 561 |
if not edit: |
| ... | ... | |
| 765 | 767 | |
| 766 | 768 |
items = [] |
| 767 | 769 |
show_as_radio = False |
| 770 |
anonymise = False |
|
| 768 | 771 |
widget_class = SingleSelectHintWidget |
| 769 | 772 |
data_source = {}
|
| 770 | 773 | |
| ... | ... | |
| 1368 | 1371 |
items = [] |
| 1369 | 1372 |
randomize_items = False |
| 1370 | 1373 |
widget_class = RankedItemsWidget |
| 1374 |
anonymise = False |
|
| 1371 | 1375 | |
| 1372 | 1376 |
def perform_more_widget_changes(self, form, kwargs, edit = True): |
| 1373 | 1377 |
kwargs['elements'] = self.items or [] |
| wcs/formdata.py | ||
|---|---|---|
| 15 | 15 |
# along with this program; if not, see <http://www.gnu.org/licenses/>. |
| 16 | 16 | |
| 17 | 17 |
import copy |
| 18 |
import datetime |
|
| 18 | 19 |
import json |
| 19 | 20 |
import time |
| 20 | 21 | |
| ... | ... | |
| 130 | 131 |
user_hash = None |
| 131 | 132 |
receipt_time = None |
| 132 | 133 |
status = None |
| 134 |
anonymised = None |
|
| 133 | 135 |
page_no = None # page to use when restoring from draft |
| 134 | 136 |
evolution = None |
| 135 | 137 |
data = None |
| ... | ... | |
| 452 | 454 | |
| 453 | 455 |
return status_action_roles |
| 454 | 456 | |
| 457 |
def anonymise(self): |
|
| 458 |
for field in self.formdef.fields: |
|
| 459 |
if field.anonymise: |
|
| 460 |
self.data[field.id] = None |
|
| 461 |
if field.store_display_value: |
|
| 462 |
self.data['%s_display' % field.id] = None |
|
| 463 | ||
| 464 |
self.anonymised = datetime.datetime.now() |
|
| 465 |
self.user_id = None |
|
| 466 |
self.user_hash = None |
|
| 467 |
self.editable_by = None |
|
| 468 |
self._signature = None |
|
| 469 |
self.workflow_data = None |
|
| 470 |
self.workflow_roles = None |
|
| 471 | ||
| 472 |
for evo in self.evolution: |
|
| 473 |
evo.who = None |
|
| 474 |
evo.parts = None |
|
| 475 |
evo.comment = None |
|
| 476 |
evo.parts = None |
|
| 477 |
self.store() |
|
| 478 | ||
| 455 | 479 |
def export_to_json(self): |
| 456 | 480 |
data = {}
|
| 457 | 481 |
data['id'] = '%s/%s' % (self.formdef.url_name, self.id) |
| wcs/forms/backoffice.py | ||
|---|---|---|
| 202 | 202 |
style = 'even' |
| 203 | 203 |
else: |
| 204 | 204 |
style = 'odd' |
| 205 |
r += htmltext('<tr class="status-%s-%s %s">') % (filled.formdef.workflow.id, filled.status, style)
|
|
| 205 |
data = '' |
|
| 206 |
if filled.anonymised: |
|
| 207 |
data += ' data-anonymised="true"' |
|
| 208 |
r += htmltext('<tr class="status-%s-%s %s"%s>' % (filled.formdef.workflow.id,
|
|
| 209 |
filled.status, style, data)) |
|
| 206 | 210 |
link = str(filled.id) + '/' |
| 207 | 211 |
for i, f in enumerate(fields): |
| 208 | 212 |
if f.type == 'id': |
| ... | ... | |
| 218 | 222 |
r += htmltext('<td class="cell-user cell-no-user">-</td>')
|
| 219 | 223 |
elif f.type == 'status': |
| 220 | 224 |
r += htmltext('<td class="cell-status">%s</td>') % filled.get_status_label()
|
| 225 |
elif f.type == 'anonymised': |
|
| 226 |
if filled.anonymised: |
|
| 227 |
anonymised = _('Yes')
|
|
| 228 |
else: |
|
| 229 |
anonymised = _('No')
|
|
| 230 |
r += htmltext('<td class="cell-anonymised">%s</td>') % anonymised
|
|
| 221 | 231 |
else: |
| 222 | 232 |
r += htmltext('<td>')
|
| 223 | 233 |
value = filled.data.get('%s_display' % f.id, filled.data.get(f.id))
|
| wcs/sql.py | ||
|---|---|---|
| 113 | 113 |
user_id varchar, |
| 114 | 114 |
user_hash varchar, |
| 115 | 115 |
receipt_time timestamp, |
| 116 |
anonymised timestamptz, |
|
| 116 | 117 |
status varchar, |
| 117 | 118 |
page_no varchar, |
| 118 | 119 |
workflow_data bytea, |
| ... | ... | |
| 132 | 133 | |
| 133 | 134 |
needed_fields = set(['id', 'user_id', 'user_hash', 'receipt_time', |
| 134 | 135 |
'status', 'workflow_data', 'id_display', 'fts', 'page_no', |
| 135 |
'workflow_roles', 'workflow_roles_array']) |
|
| 136 |
'anonymised', 'workflow_roles', 'workflow_roles_array'])
|
|
| 136 | 137 | |
| 137 | 138 |
# migrations |
| 138 | 139 |
if not 'fts' in existing_fields: |
| ... | ... | |
| 148 | 149 |
if not 'page_no' in existing_fields: |
| 149 | 150 |
cur.execute('''ALTER TABLE %s ADD COLUMN page_no varchar''' % table_name)
|
| 150 | 151 | |
| 152 |
if not 'anonymised' in existing_fields: |
|
| 153 |
cur.execute('''ALTER TABLE %s ADD COLUMN anonymised timestamptz''' % table_name)
|
|
| 154 | ||
| 151 | 155 |
# add new fields |
| 152 | 156 |
for field in formdef.fields: |
| 153 | 157 |
assert field.id is not None |
| ... | ... | |
| 581 | 585 |
('receipt_time', 'timestamp'),
|
| 582 | 586 |
('status', 'varchar'),
|
| 583 | 587 |
('page_no', 'varchar'),
|
| 588 |
('anonymised', 'timestamptz'),
|
|
| 584 | 589 |
('workflow_data', 'bytea'),
|
| 585 | 590 |
('id_display', 'varchar'),
|
| 586 | 591 |
('workflow_roles', 'bytea'),
|
| ... | ... | |
| 638 | 643 |
'page_no': self.page_no, |
| 639 | 644 |
'workflow_data': bytearray(cPickle.dumps(self.workflow_data)), |
| 640 | 645 |
'id_display': self.id_display, |
| 646 |
'anonymised': self.anonymised, |
|
| 641 | 647 |
} |
| 642 | 648 |
if self.receipt_time: |
| 643 | 649 |
sql_dict['receipt_time'] = datetime.datetime.fromtimestamp(time.mktime(self.receipt_time)), |
| 644 |
- |
|