From 4e289b0758298dbae43c9bbc23d1b96dc1858ffa Mon Sep 17 00:00:00 2001
From: Thomas NOEL <tnoel@entrouvert.com>
Date: Wed, 9 Apr 2014 16:22:59 +0200
Subject: [PATCH] admin: add 'anonymise' action

---
 wcs/admin/forms.py      | 101 +++++++++++++++++++++++++++++++++++++++++++++++-
 wcs/backoffice/root.py  |   1 +
 wcs/fields.py           |   4 ++
 wcs/formdata.py         |  24 ++++++++++++
 wcs/forms/backoffice.py |  12 +++++-
 wcs/sql.py              |   8 +++-
 6 files changed, 146 insertions(+), 4 deletions(-)

diff --git a/wcs/admin/forms.py b/wcs/admin/forms.py
index d9199b8..ffaa2cf 100644
--- a/wcs/admin/forms.py
+++ b/wcs/admin/forms.py
@@ -27,6 +27,7 @@ try:
 except ImportError:
     M2Crypto = None
 
+import datetime
 import tarfile
 import time
 from cStringIO import StringIO
@@ -206,8 +207,8 @@ class WorkflowRoleDirectory(Directory):
 
 class FormDefPage(Directory):
     _q_exports = ['', 'fields', 'delete', 'duplicate', 'export',
-                  'archive', 'invite', 'enable', 'workflow', 'category',
-                  'role', ('workflow-options', 'workflow_options'),
+                  'anonymise', 'archive', 'invite', 'enable', 'workflow',
+                  'category', 'role', ('workflow-options', 'workflow_options'),
                   ('workflow-status-remapping', 'workflow_status_remapping'),
                   'roles', 'title', 'options', ('acl-read', 'acl_read')]
 
@@ -386,6 +387,7 @@ class FormDefPage(Directory):
         r += htmltext('<li><a href="duplicate">%s</a></li>') % _('Duplicate')
         if ET:
             r += htmltext('<li><a href="export">%s</a></li>') % _('Export')
+        r += htmltext('<li><a href="anonymise">%s</a></li>') % _('Anonymise forms')
         if not (get_publisher().has_site_option('postgresql') and get_cfg('postgresql', {})):
             r += htmltext('<li><a href="archive">%s</a></li>') % _('Archive')
         if self.formdef.roles:
@@ -847,6 +849,101 @@ class FormDefPage(Directory):
             'attachment; filename=%s-archive.wcs' % self.formdef.url_name)
         return job.file_content
 
+    def anonymise(self):
+        if get_request().form.get('job'):
+            return self.anonymise_processing()
+
+        endpoints = []
+        for status in self.formdef.workflow.get_endpoint_status():
+            endpoints.append((status.id, status.name))
+
+        form = Form(enctype='multipart/form-data')
+        form.add(DateWidget, 'before_request_date',
+                title=_('Requests ended before'),
+                value=datetime.date.today() - datetime.timedelta(30),
+                required=True)
+        form.add(CheckboxesWidget, 'endpoints', title=_('Status of the forms to anonymise'),
+                value=[endpoint[0] for endpoint in endpoints],
+                elements=endpoints,
+                inline=False,
+                required=True)
+        form.add_submit('submit', _('Submit'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+        if not form.is_submitted() or form.has_errors():
+            get_response().breadcrumb.append(('anonymise', _('Anonymise')))
+            html_top('forms', title=_('Anonymise Forms'))
+            r = TemplateIO(html=True)
+            r += htmltext('<h2>%s</h2>') % _('Anonymise Forms')
+            r += htmltext('<p>%s</p>' % _("You are about to irrevocably anonymise forms."))
+            r += form.render()
+            return r.getvalue()
+        else:
+            return self.anonymise_submit(form)
+
+    def anonymise_submit(self, form):
+
+        class Anonymiser:
+
+            def __init__(self, formdef, status_ids, before_date):
+                self.formdef = formdef
+                self.status_ids = ["wf-%s" % id for id in status_ids]
+                self.before_date = before_date
+
+            def anonymise(self, job=None):
+                for formdata in self.formdef.data_class().select():
+                    if formdata.anonymised:
+                        continue
+                    if formdata.status not in self.status_ids:
+                        continue
+                    if (formdata.evolution and formdata.evolution[-1].time >= self.before_date) \
+                            or (formdata.receipt_time >= self.before_date):
+                        continue
+                    formdata.anonymise()
+
+        before_date = form.get_widget('before_request_date').parse()
+        before_date = time.strptime(before_date, misc.date_format())
+        status_ids = form.get_widget('endpoints').parse()
+        count = self.formdef.data_class().count()
+        anonymiser = Anonymiser(self.formdef, status_ids, before_date)
+        if count > 100: # Arbitrary threshold
+            job = get_response().add_after_job(
+                str(N_('Anonymising forms')),
+                anonymiser.anonymise)
+            return redirect('anonymise?job=%s' % job.id)
+        else:
+            anonymiser.anonymise()
+
+        return redirect('.')
+
+    def anonymise_processing(self):
+        try:
+            job = AfterJob.get(get_request().form.get('job'))
+        except KeyError:
+            return redirect('.')
+
+        html_top('forms', title=_('Anonymising'))
+        r = TemplateIO(html=True)
+        r += get_session().display_message()
+        get_response().add_javascript(['jquery.js', 'afterjob.js'])
+        r += htmltext('<dl class="job-status">')
+        r += htmltext('<dt>')
+        r += _(job.label)
+        r += htmltext('</dt>')
+        r += htmltext('<dd>')
+        r += htmltext('<span class="afterjob" id="%s">') % job.id
+        r += _(job.status)
+        r += htmltext('</span>')
+        r += htmltext('</dd>')
+        r += htmltext('</dl>')
+
+        r += htmltext('<div class="done">')
+        r += htmltext('<a href="./">%s</a>') % _('Back')
+        r += htmltext('</div>')
+        return r.getvalue()
+
     def invite(self):
         if not self.formdef.roles:
             return template.error_page(
diff --git a/wcs/backoffice/root.py b/wcs/backoffice/root.py
index 9816815..04d53c6 100644
--- a/wcs/backoffice/root.py
+++ b/wcs/backoffice/root.py
@@ -423,6 +423,7 @@ class FormPage(Directory):
         fields.append(FakeField('user-label', 'user-label', _('User Label')))
         fields.extend(self.formdef.fields)
         fields.append(FakeField('status', 'status', _('Status')))
+        fields.append(FakeField('anonymised', 'anonymised', _('Anonymised')))
 
         return fields
 
diff --git a/wcs/fields.py b/wcs/fields.py
index 8cb5c9a..272eda2 100644
--- a/wcs/fields.py
+++ b/wcs/fields.py
@@ -118,6 +118,7 @@ class Field:
     prefill = None
     store_display_value = None
 
+    anonymise = True
     stats = None
 
     def __init__(self, **kwargs):
@@ -554,6 +555,7 @@ class BoolField(WidgetField):
 
     widget_class = CheckboxWidget
     required = False
+    anonymise = False
 
     def perform_more_widget_changes(self, form, kwargs, edit = True):
         if not edit:
@@ -765,6 +767,7 @@ class ItemField(WidgetField):
 
     items = []
     show_as_radio = False
+    anonymise = False
     widget_class = SingleSelectHintWidget
     data_source = {}
 
@@ -1368,6 +1371,7 @@ class RankedItemsField(WidgetField):
     items = []
     randomize_items = False
     widget_class = RankedItemsWidget
+    anonymise = False
 
     def perform_more_widget_changes(self, form, kwargs, edit = True):
         kwargs['elements'] = self.items or []
diff --git a/wcs/formdata.py b/wcs/formdata.py
index 65c11a2..1cefefe 100644
--- a/wcs/formdata.py
+++ b/wcs/formdata.py
@@ -15,6 +15,7 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import copy
+import datetime
 import json
 import time
 
@@ -130,6 +131,7 @@ class FormData(StorableObject):
     user_hash = None
     receipt_time = None
     status = None
+    anonymised = None
     page_no = None # page to use when restoring from draft
     evolution = None
     data = None
@@ -452,6 +454,28 @@ class FormData(StorableObject):
 
         return status_action_roles
 
+    def anonymise(self):
+        for field in self.formdef.fields:
+            if field.anonymise:
+                self.data[field.id] = None
+                if field.store_display_value:
+                    self.data['%s_display' % field.id] = None
+
+        self.anonymised = datetime.datetime.now()
+        self.user_id = None
+        self.user_hash = None
+        self.editable_by = None
+        self._signature = None
+        self.workflow_data = None
+        self.workflow_roles = None
+
+        for evo in self.evolution:
+            evo.who = None
+            evo.parts = None
+            evo.comment = None
+            evo.parts = None
+        self.store()
+
     def export_to_json(self):
         data = {}
         data['id'] = '%s/%s' % (self.formdef.url_name, self.id)
diff --git a/wcs/forms/backoffice.py b/wcs/forms/backoffice.py
index a18528e..3381d4f 100644
--- a/wcs/forms/backoffice.py
+++ b/wcs/forms/backoffice.py
@@ -202,7 +202,11 @@ class FormDefUI:
                 style = 'even'
             else:
                 style = 'odd'
-            r += htmltext('<tr class="status-%s-%s %s">') % (filled.formdef.workflow.id, filled.status, style)
+            data = ''
+            if filled.anonymised:
+                data += ' data-anonymised="true"'
+            r += htmltext('<tr class="status-%s-%s %s"%s>' % (filled.formdef.workflow.id,
+                    filled.status, style, data))
             link = str(filled.id) + '/'
             for i, f in enumerate(fields):
                 if f.type == 'id':
@@ -218,6 +222,12 @@ class FormDefUI:
                         r += htmltext('<td class="cell-user cell-no-user">-</td>')
                 elif f.type == 'status':
                     r += htmltext('<td class="cell-status">%s</td>') % filled.get_status_label()
+                elif f.type == 'anonymised':
+                    if filled.anonymised:
+                        anonymised = _('Yes')
+                    else:
+                        anonymised = _('No')
+                    r += htmltext('<td class="cell-anonymised">%s</td>') % anonymised
                 else:
                     r += htmltext('<td>')
                     value = filled.data.get('%s_display' % f.id, filled.data.get(f.id))
diff --git a/wcs/sql.py b/wcs/sql.py
index 2f97ce9..df55480 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -113,6 +113,7 @@ def do_formdef_tables(formdef):
                                     user_id varchar,
                                     user_hash varchar,
                                     receipt_time timestamp,
+                                    anonymised timestamptz,
                                     status varchar,
                                     page_no varchar,
                                     workflow_data bytea,
@@ -132,7 +133,7 @@ def do_formdef_tables(formdef):
 
     needed_fields = set(['id', 'user_id', 'user_hash', 'receipt_time',
         'status', 'workflow_data', 'id_display', 'fts', 'page_no',
-        'workflow_roles', 'workflow_roles_array'])
+        'anonymised', 'workflow_roles', 'workflow_roles_array'])
 
     # migrations
     if not 'fts' in existing_fields:
@@ -148,6 +149,9 @@ def do_formdef_tables(formdef):
     if not 'page_no' in existing_fields:
         cur.execute('''ALTER TABLE %s ADD COLUMN page_no varchar''' % table_name)
 
+    if not 'anonymised' in existing_fields:
+        cur.execute('''ALTER TABLE %s ADD COLUMN anonymised timestamptz''' % table_name)
+
     # add new fields
     for field in formdef.fields:
         assert field.id is not None
@@ -581,6 +585,7 @@ class SqlFormData(SqlMixin, wcs.formdata.FormData):
         ('receipt_time', 'timestamp'),
         ('status', 'varchar'),
         ('page_no', 'varchar'),
+        ('anonymised', 'timestamptz'),
         ('workflow_data', 'bytea'),
         ('id_display', 'varchar'),
         ('workflow_roles', 'bytea'),
@@ -638,6 +643,7 @@ class SqlFormData(SqlMixin, wcs.formdata.FormData):
                 'page_no': self.page_no,
                 'workflow_data': bytearray(cPickle.dumps(self.workflow_data)),
                 'id_display': self.id_display,
+                'anonymised': self.anonymised,
         }
         if self.receipt_time:
             sql_dict['receipt_time'] = datetime.datetime.fromtimestamp(time.mktime(self.receipt_time)),
-- 
1.9.1

