From a60899eb2d1390be9d7201605d85b4205ff8ed8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Fri, 14 Oct 2022 16:57:21 +0200
Subject: [PATCH] api: get formdata from history (#70271)

---
 tests/test_content_snapshots.py | 224 ++++++++++++++++++++++++++++++++
 wcs/backoffice/management.py    |  39 ++++--
 wcs/formdata.py                 |  36 ++++-
 wcs/forms/common.py             |  21 ++-
 4 files changed, 301 insertions(+), 19 deletions(-)

diff --git a/tests/test_content_snapshots.py b/tests/test_content_snapshots.py
index 3fcde915c..28b1af5d4 100644
--- a/tests/test_content_snapshots.py
+++ b/tests/test_content_snapshots.py
@@ -1,12 +1,16 @@
+import datetime
 import json
 import os
+import time
 
 import pytest
+from django.utils.timezone import make_aware
 from webtest import Upload
 
 from wcs import fields
 from wcs.api_access import ApiAccess
 from wcs.carddef import CardDef
+from wcs.formdata import Evolution
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.ident.password_accounts import PasswordAccount
@@ -739,3 +743,223 @@ def test_api_card_submit(pub, user, access, role):
     assert carddata.evolution[0].parts[0].formdef_id == str(carddef.id)
     assert carddata.evolution[0].parts[0].old_data == {}
     assert carddata.evolution[0].parts[0].new_data == {'1': 'xxx'}
+
+
+def test_api_formdata_at(pub, user, access, role):
+    app = get_app(pub)
+    app.set_authorization(('Basic', ('test', '12345')))
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.StringField(id='0', label='foobar', varname='foobar'),
+    ]
+    workflow = Workflow(name='foo')
+    workflow.possible_status = Workflow.get_default_workflow().possible_status[:]
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(id='bo1', label='bo field 1', type='string', varname='plop'),
+    ]
+    workflow.store()
+    formdef.workflow_id = workflow.id
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.data = {
+        '0': 'foo',
+        'bo1': 'bar',
+    }
+    formdata.just_created()
+    formdata.evolution[-1].parts[0].datetime = make_aware(datetime.datetime(2022, 1, 2, 3, 4))
+    formdata.store()
+
+    def get_evo_and_parts(formdata):
+        for evo in formdata.evolution:
+            for part in evo.parts or []:
+                if isinstance(part, ContentSnapshotPart):
+                    yield part.datetime.strftime('%Y-%m-%d %H:%M'), part.old_data, part.new_data
+
+    assert list(get_evo_and_parts(formdata)) == [('2022-01-02 03:04', {}, {'0': 'foo', 'bo1': 'bar'})]
+    resp = app.get('/api/forms/test/%s/' % formdata.id)
+    assert resp.json['fields'] == {'foobar': 'foo'}
+    assert resp.json['workflow']['fields'] == {'plop': 'bar'}
+    resp = app.get('/api/forms/test/list/?full=on')
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'foo'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'bar'}
+
+    # wrong format
+    resp = app.get('/api/forms/test/%s/?at=bad-format' % formdata.id, status=400)
+    assert resp.json['err_desc'] == 'Invalid value "bad-format" for "at"'
+    resp = app.get('/api/forms/test/list/?full=on&at=bad-format', status=400)
+    assert resp.json['err_desc'] == 'Invalid value "bad-format" for "at"'
+
+    # before formdata creation
+    resp = app.get(
+        '/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-02T00:00:00+01:00'}, status=400
+    )
+    assert resp.json['err_desc'] == 'No data found for this datetime'
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-02T00:00:00+01:00'})
+    assert len(resp.json) == 0
+    resp = app.get(
+        '/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-02T03:04:00+01:00'}, status=400
+    )
+    assert resp.json['err_desc'] == 'No data found for this datetime'
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-02T03:04:00+01:00'})
+    assert len(resp.json) == 0
+
+    # no ContentSnapshotPart (legacy formdata)
+    formdata.evolution[0].parts = []
+    formdata.store()
+    assert list(get_evo_and_parts(formdata)) == []
+
+    # add evolutions with ContentSnapshotPart
+    evo = formdata.evolution[0]
+    part = ContentSnapshotPart(formdata=formdata, old_data={})
+    part.new_data = {'0': 'bar', 'bo1': 'foo'}
+    part.datetime = make_aware(datetime.datetime(2022, 1, 2, 3, 4))
+    evo.add_part(part)
+
+    evo = Evolution()
+    evo.time = time.localtime()
+    evo.status = formdata.status
+    part = ContentSnapshotPart(formdata=formdata, old_data={'0': 'bar', 'bo1': 'foo'})
+    part.new_data = {'0': 'baz', 'bo1': 'foo'}
+    part.datetime = make_aware(datetime.datetime(2022, 1, 2, 3, 5))
+    evo.add_part(part)
+    formdata.evolution.append(evo)
+
+    evo = Evolution()
+    evo.time = time.localtime()
+    evo.status = formdata.status
+    part = ContentSnapshotPart(formdata=formdata, old_data={'0': 'baz', 'bo1': 'foo'})
+    part.new_data = {'0': 'foooo', '1': 'unknown', 'bo1': 'foo'}
+    part.datetime = make_aware(datetime.datetime(2022, 1, 4, 5, 6))
+    evo.add_part(part)
+    formdata.evolution.append(evo)
+
+    evo = Evolution()
+    evo.time = time.localtime()
+    evo.status = formdata.status
+    part = ContentSnapshotPart(formdata=formdata, old_data={'0': 'foooo', '1': 'unknown', 'bo1': 'foo'})
+    part.new_data = {'0': 'fooo', 'bo1': 'foo'}
+    part.datetime = make_aware(datetime.datetime(2022, 1, 5, 6, 7))
+    evo.add_part(part)
+    part = ContentSnapshotPart(formdata=formdata, old_data={'0': 'fooo', 'bo1': 'foo'})
+    part.new_data = {'0': 'foo', 'bo1': 'bar'}
+    part.datetime = make_aware(datetime.datetime(2022, 1, 5, 6, 7))
+    evo.add_part(part)
+    formdata.evolution.append(evo)
+
+    formdata._store_all_evolution = True
+    formdata.store()
+    assert list(get_evo_and_parts(formdata)) == [
+        ('2022-01-02 03:04', {}, {'0': 'bar', 'bo1': 'foo'}),
+        ('2022-01-02 03:05', {'0': 'bar', 'bo1': 'foo'}, {'0': 'baz', 'bo1': 'foo'}),
+        ('2022-01-04 05:06', {'0': 'baz', 'bo1': 'foo'}, {'0': 'foooo', '1': 'unknown', 'bo1': 'foo'}),
+        ('2022-01-05 06:07', {'0': 'foooo', '1': 'unknown', 'bo1': 'foo'}, {'0': 'fooo', 'bo1': 'foo'}),
+        ('2022-01-05 06:07', {'0': 'fooo', 'bo1': 'foo'}, {'0': 'foo', 'bo1': 'bar'}),
+    ]
+
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-03T00:00:00+01:00'})
+    assert resp.json['fields'] == {'foobar': 'baz'}
+    assert resp.json['workflow']['fields'] == {'plop': 'foo'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-03T00:00:00+01:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'baz'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'foo'}
+
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-02T03:05:00+01:00'})
+    assert resp.json['fields'] == {'foobar': 'bar'}
+    assert resp.json['workflow']['fields'] == {'plop': 'foo'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-02T03:05:00+01:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'bar'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'foo'}
+
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-02T03:06:00+01:00'})
+    assert resp.json['fields'] == {'foobar': 'baz'}
+    assert resp.json['workflow']['fields'] == {'plop': 'foo'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-02T03:06:00+01:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'baz'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'foo'}
+
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-04T00:00:00+01:00'})
+    assert resp.json['fields'] == {'foobar': 'baz'}
+    assert resp.json['workflow']['fields'] == {'plop': 'foo'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-04T00:00:00+01:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'baz'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'foo'}
+
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-05T00:00:00+01:00'})
+    assert resp.json['fields'] == {'foobar': 'foooo'}
+    assert resp.json['workflow']['fields'] == {'plop': 'foo'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-05T00:00:00+01:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'foooo'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'foo'}
+
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-05T06:07:00+01:00'})
+    assert resp.json['fields'] == {'foobar': 'foooo'}
+    assert resp.json['workflow']['fields'] == {'plop': 'foo'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-05T06:07:00+01:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'foooo'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'foo'}
+
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-05T06:08:00+01:00'})
+    assert resp.json['fields'] == {'foobar': 'foo'}
+    assert resp.json['workflow']['fields'] == {'plop': 'bar'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-05T06:08:00+01:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'foo'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'bar'}
+
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-06T00:00:00+01:00'})
+    assert resp.json['fields'] == {'foobar': 'foo'}
+    assert resp.json['workflow']['fields'] == {'plop': 'bar'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-06T00:00:00+01:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'foo'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'bar'}
+
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-07T00:00:00+01:00'})
+    assert resp.json['fields'] == {'foobar': 'foo'}
+    assert resp.json['workflow']['fields'] == {'plop': 'bar'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-07T00:00:00+01:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'foo'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'bar'}
+
+    # check with other TZ
+    resp = app.get(
+        '/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-02T04:04:00+02:00'}, status=400
+    )
+    assert resp.json['err_desc'] == 'No data found for this datetime'
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-02T04:04:00+02:00'})
+    assert len(resp.json) == 0
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-02T04:05:00+02:00'})
+    assert resp.json['fields'] == {'foobar': 'bar'}
+    assert resp.json['workflow']['fields'] == {'plop': 'foo'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-02T04:05:00+02:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'bar'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'foo'}
+
+    # check without TZ
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-02T03:04:00'}, status=400)
+    assert resp.json['err_desc'] == 'No data found for this datetime'
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-02T03:04:00'})
+    assert len(resp.json) == 0
+    resp = app.get('/api/forms/test/%s/' % formdata.id, params={'at': '2022-01-02T03:05:00'})
+    assert resp.json['fields'] == {'foobar': 'bar'}
+    assert resp.json['workflow']['fields'] == {'plop': 'foo'}
+    resp = app.get('/api/forms/test/list/', params={'full': 'on', 'at': '2022-01-02T03:05:00'})
+    assert len(resp.json) == 1
+    assert resp.json[0]['fields'] == {'foobar': 'bar'}
+    assert resp.json[0]['workflow']['fields'] == {'plop': 'foo'}
diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py
index dd9c302c3..cdfc491e2 100644
--- a/wcs/backoffice/management.py
+++ b/wcs/backoffice/management.py
@@ -26,6 +26,7 @@ import zipfile
 
 import vobject
 from django.utils.encoding import force_text
+from django.utils.timezone import is_naive, make_aware
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
 from quixote.errors import RequestError
@@ -38,7 +39,7 @@ from wcs.api_utils import get_user_from_api_query_string
 from wcs.carddef import CardDef
 from wcs.categories import Category
 from wcs.conditions import Condition
-from wcs.formdata import FormData
+from wcs.formdata import FormData, NoContentSnapshotAt
 from wcs.formdef import FormDef
 from wcs.forms.backoffice import FormDefUI
 from wcs.forms.common import FormdefDirectoryBase, FormStatusPage
@@ -2471,6 +2472,7 @@ class FormPage(FormdefDirectoryBase):
                 include_files=False,
                 include_roles=True,
                 include_unnamed_fields=False,
+                values_at=get_request().form.get('at'),
             )
         else:
             output = [
@@ -4297,7 +4299,16 @@ class JsonFileExportAfterJob(CsvExportAfterJob):
         include_files,
         include_roles,
         include_unnamed_fields,
+        values_at=None,
     ):
+        if values_at:
+            try:
+                values_at = datetime.datetime.fromisoformat(values_at)
+                if is_naive(values_at):
+                    values_at = make_aware(values_at)
+            except ValueError:
+                raise RequestError('Invalid value "%s" for "at"' % values_at)
+
         formdef = self.kwargs['formdef_class'].get(self.kwargs['formdef_id'])
         prefetched_users = None
         prefetched_roles = None
@@ -4327,17 +4338,21 @@ class JsonFileExportAfterJob(CsvExportAfterJob):
             }
         output = []
         for formdata in items:
-            data = formdata.get_json_export_dict(
-                anonymise=anonymise,
-                user=user,
-                digest_key=digest_key,
-                prefetched_users=prefetched_users,
-                prefetched_roles=prefetched_roles,
-                include_evolution=include_evolution,
-                include_files=include_files,
-                include_roles=include_roles,
-                include_unnamed_fields=include_unnamed_fields,
-            )
+            try:
+                data = formdata.get_json_export_dict(
+                    anonymise=anonymise,
+                    user=user,
+                    digest_key=digest_key,
+                    prefetched_users=prefetched_users,
+                    prefetched_roles=prefetched_roles,
+                    include_evolution=include_evolution,
+                    include_files=include_files,
+                    include_roles=include_roles,
+                    include_unnamed_fields=include_unnamed_fields,
+                    values_at=values_at,
+                )
+            except NoContentSnapshotAt:
+                continue
             data.pop('digests')
             if digest_key:
                 data['digest'] = (formdata.digests or {}).get(digest_key)
diff --git a/wcs/formdata.py b/wcs/formdata.py
index a9cc3c4b3..d7df4a915 100644
--- a/wcs/formdata.py
+++ b/wcs/formdata.py
@@ -24,6 +24,7 @@ import sys
 import time
 
 from quixote import get_publisher, get_request, get_session
+from quixote.errors import RequestError
 from quixote.html import htmltext
 from quixote.http_request import Upload
 
@@ -35,6 +36,10 @@ from .qommon.substitution import CompatibilityNamesDict, Substitutions, invalida
 from .qommon.template import Template
 
 
+class NoContentSnapshotAt(RequestError):
+    pass
+
+
 def get_dict_with_varnames(fields, data, formdata=None, varnames_only=False):
     new_data = {}
     for field in fields:
@@ -1334,9 +1339,9 @@ class FormData(StorableObject):
                     store_dict[store_key + '_structured'] = data.get('%s_structured' % field.id)
         return new_data
 
-    def get_json_dict(self, fields, include_files=True, anonymise=False, include_unnamed_fields=False):
+    def get_json_dict(self, data, fields, include_files=True, anonymise=False, include_unnamed_fields=False):
         return self.get_json_data_dict(
-            self.data,
+            data,
             fields,
             formdata=self,
             include_files=include_files,
@@ -1355,7 +1360,9 @@ class FormData(StorableObject):
         include_evolution=True,
         include_roles=True,
         include_unnamed_fields=False,
+        values_at=None,
     ):
+        # noqa pylint: disable=too-many-arguments
         data = {}
         data['id'] = str(self.id)
         data['digests'] = self.digests
@@ -1381,7 +1388,25 @@ class FormData(StorableObject):
 
             data['user'] = user.get_json_export_dict(full=isinstance(self.formdef, CardDef))
 
+        _data = self.data
+        if values_at:
+            from wcs.workflows import ContentSnapshotPart
+
+            matching_part = None
+            for evo in reversed(self.evolution or []):
+                for part in reversed(evo.parts or []):
+                    if isinstance(part, ContentSnapshotPart):
+                        if part.datetime < values_at:
+                            matching_part = part
+                        break
+                if matching_part:
+                    break
+            if not matching_part:
+                raise NoContentSnapshotAt('No data found for this datetime')
+            _data = matching_part.new_data
+
         data['fields'] = self.get_json_dict(
+            _data,
             self.formdef.fields,
             include_files=include_files,
             anonymise=anonymise,
@@ -1400,6 +1425,7 @@ class FormData(StorableObject):
             data['workflow']['data'] = self.workflow_data
         if self.formdef.workflow.get_backoffice_fields():
             data['workflow']['fields'] = self.get_json_dict(
+                _data,
                 self.formdef.workflow.get_backoffice_fields(),
                 include_files=include_files,
                 anonymise=anonymise,
@@ -1474,8 +1500,10 @@ class FormData(StorableObject):
 
         return data
 
-    def export_to_json(self, include_files=True, anonymise=False):
-        data = self.get_json_export_dict(include_files=include_files, anonymise=anonymise)
+    def export_to_json(self, include_files=True, anonymise=False, values_at=None):
+        data = self.get_json_export_dict(
+            include_files=include_files, anonymise=anonymise, values_at=values_at
+        )
         return json.dumps(data, cls=misc.JSONEncoder)
 
     def get_object_key(self):
diff --git a/wcs/forms/common.py b/wcs/forms/common.py
index a49572629..6325967e4 100644
--- a/wcs/forms/common.py
+++ b/wcs/forms/common.py
@@ -14,13 +14,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import datetime
 import json
 import os
 import time
 import urllib.parse
 
+from django.utils.timezone import is_naive, make_aware
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
+from quixote.errors import RequestError
 from quixote.html import TemplateIO, htmltext
 from quixote.util import randbytes
 
@@ -210,8 +213,18 @@ class FormStatusPage(Directory, FormTemplateMixin):
             if str(self.filled.id) not in [str(x) for x in item_ids]:
                 raise errors.TraversalError(_('ID not available in filtered view'))
 
+        values_at = get_request().form.get('at')
+        if values_at:
+            try:
+                values_at = datetime.datetime.fromisoformat(values_at)
+                if is_naive(values_at):
+                    values_at = make_aware(values_at)
+            except ValueError:
+                raise RequestError('Invalid value "%s" for "at"' % values_at)
         return self.export_to_json(
-            anonymise=anonymise, include_files=get_query_flag('include-files-content', default=True)
+            anonymise=anonymise,
+            include_files=get_query_flag('include-files-content', default=True),
+            values_at=values_at,
         )
 
     def tempfile(self):
@@ -421,9 +434,11 @@ class FormStatusPage(Directory, FormTemplateMixin):
             response.content_type = 'text/plain'
             return "Your browser should redirect you"
 
-    def export_to_json(self, include_files=True, anonymise=False):
+    def export_to_json(self, include_files=True, anonymise=False, values_at=None):
         get_response().set_content_type('application/json')
-        return self.filled.export_to_json(include_files=include_files, anonymise=anonymise)
+        return self.filled.export_to_json(
+            include_files=include_files, anonymise=anonymise, values_at=values_at
+        )
 
     def history(self):
         if not self.filled.evolution:
-- 
2.35.1

