From d98a0e0a257ea4ad6db203c01965d00cef915ae8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Fri, 7 Oct 2022 09:40:16 +0200
Subject: [PATCH 3/3] formdata: look at files in history when deleting unused
 files (#62800)

---
 tests/test_formdef.py | 44 +++++++++++++++++++++++++++++++++++--------
 wcs/formdata.py       |  8 +++++++-
 wcs/formdef.py        |  2 +-
 3 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/tests/test_formdef.py b/tests/test_formdef.py
index 51f0c738e..0f4c3b1be 100644
--- a/tests/test_formdef.py
+++ b/tests/test_formdef.py
@@ -20,6 +20,7 @@ from wcs.qommon.upload_storage import PicklableUpload
 from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormEvolutionPart, WorkflowFormFieldsFormDef
 from wcs.workflows import (
     AttachmentEvolutionPart,
+    ContentSnapshotPart,
     Workflow,
     WorkflowBackofficeFieldsFormDef,
     WorkflowVariablesFieldsFormDef,
@@ -340,12 +341,14 @@ def test_unused_file_removal_job(pub):
             pub.site_options.set('options', 'unused-files-behaviour', behaviour)
 
         formdata = formdef.data_class()()
-        formdata.just_created()
         formdata.data = {
             '5': PicklableUpload('test.txt', 'text/plain'),
         }
         formdata.data['5'].receive([b'hello world'])
+        formdata.just_created()
         formdata.store()
+        assert isinstance(formdata.evolution[0].parts[0], ContentSnapshotPart)
+        assert formdata.evolution[0].parts[0].new_data['5'].qfilename == formdata.data['5'].qfilename
 
         assert formdata.data['5'].qfilename in os.listdir(os.path.join(pub.app_dir, 'uploads'))
         clean_unused_files(pub)
@@ -356,12 +359,14 @@ def test_unused_file_removal_job(pub):
 
         for _ in range(5):
             formdata = formdef.data_class()()
-            formdata.just_created()
             formdata.data = {
                 '5': PicklableUpload('test.txt', 'text/plain'),
             }
             formdata.data['5'].receive([b'hello world'])
+            formdata.just_created()
             formdata.store()
+            assert isinstance(formdata.evolution[0].parts[0], ContentSnapshotPart)
+            assert formdata.evolution[0].parts[0].new_data['5'].qfilename == formdata.data['5'].qfilename
 
         # same file, deduplicated
         assert os.listdir(os.path.join(pub.app_dir, 'uploads')) == [formdata.data['5'].qfilename]
@@ -374,6 +379,29 @@ def test_unused_file_removal_job(pub):
         clean_unused_files(pub)
         assert os.listdir(os.path.join(pub.app_dir, 'uploads')) == []
 
+        # file referenced in formdata history, but not in formdata's data
+        formdata = formdef.data_class()()
+        formdata.data = {
+            '5': PicklableUpload('test.txt', 'text/plain'),
+        }
+        formdata.data['5'].receive([b'hello world'])
+        formdata.just_created()
+        formdata.store()
+        assert isinstance(formdata.evolution[0].parts[0], ContentSnapshotPart)
+        assert formdata.evolution[0].parts[0].new_data['5'].qfilename == formdata.data['5'].qfilename
+        qfilename = formdata.data['5'].qfilename
+        formdata.data['5'] = None
+        formdata.store()
+        assert formdata.evolution[0].parts[0].new_data['5'].qfilename == qfilename
+        assert os.listdir(os.path.join(pub.app_dir, 'uploads')) == [qfilename]
+        clean_unused_files(pub)
+        assert len(os.listdir(os.path.join(pub.app_dir, 'uploads'))) == 1
+        formdata.anonymise()
+        assert len(formdata.evolution) == 1
+        assert formdata.evolution[0].parts is None
+        clean_unused_files(pub)
+        assert os.listdir(os.path.join(pub.app_dir, 'uploads')) == []
+
         # file referenced in formdef option
         workflow = Workflow(name='variables')
 
@@ -387,11 +415,11 @@ def test_unused_file_removal_job(pub):
         formdef.store()
 
         formdata = formdef.data_class()()
-        formdata.just_created()
         formdata.data = {
             '5': PicklableUpload('test.txt', 'text/plain'),
         }
         formdata.data['5'].receive([b'hello world'])
+        formdata.just_created()
         formdata.store()
 
         assert len(os.listdir(os.path.join(pub.app_dir, 'uploads'))) == 1
@@ -407,7 +435,6 @@ def test_unused_file_removal_job(pub):
 
         # file in block field
         formdata = formdef.data_class()()
-        formdata.just_created()
         formdata.data = {
             '6': {
                 'data': [
@@ -420,6 +447,7 @@ def test_unused_file_removal_job(pub):
         formdata.data['6']['data'][0]['234'].receive([b'hello world'])
         formdata.data['6']['data'][1]['234'].receive([b'hello world block'])
         formdata.workflow_data = {'wscall': {'data': ['not', 'a', 'block'], 'err': 0}}
+        formdata.just_created()
         formdata.store()
         assert len(os.listdir(os.path.join(pub.app_dir, 'uploads'))) == 2
         clean_unused_files(pub)
@@ -430,13 +458,13 @@ def test_unused_file_removal_job(pub):
 
         # non local storage: nothing happens
         formdata = formdef.data_class()()
-        formdata.just_created()
         formdata.data = {
             '5': PicklableUpload('test.txt', 'text/plain'),
         }
         formdata.data['5'].receive([b'hello world'])
         formdata.data['5'].storage = 'remote'
         formdata.data['5'].storage_attrs = {'redirect_url': 'https://crypto.example.net/1234'}
+        formdata.just_created()
         formdata.store()
         assert len(os.listdir(os.path.join(pub.app_dir, 'uploads'))) == 0
         clean_unused_files(pub)
@@ -444,8 +472,8 @@ def test_unused_file_removal_job(pub):
 
         # workflow attachment
         formdata = formdef.data_class()()
-        formdata.just_created()
         formdata.data = {}
+        formdata.just_created()
         formdata.store()
 
         formdata.evolution[-1].parts = [
@@ -480,8 +508,8 @@ def test_unused_file_removal_job(pub):
 
         # file from workflow form
         formdata = formdef.data_class()()
-        formdata.just_created()
         formdata.data = {}
+        formdata.just_created()
         formdata.store()
 
         display_form = FormWorkflowStatusItem()
@@ -515,11 +543,11 @@ def test_unused_file_removal_job(pub):
     # unknown unused-files-behaviour: do nothing
     pub.site_options.set('options', 'unused-files-behaviour', 'foo')
     formdata = formdef.data_class()()
-    formdata.just_created()
     formdata.data = {
         '5': PicklableUpload('test-no-remove.txt', 'text/plain'),
     }
     formdata.data['5'].receive([b'hello world'])
+    formdata.just_created()
     formdata.store()
 
     assert formdata.data['5'].qfilename in os.listdir(os.path.join(pub.app_dir, 'uploads'))
diff --git a/wcs/formdata.py b/wcs/formdata.py
index 100acaa39..b93168cb8 100644
--- a/wcs/formdata.py
+++ b/wcs/formdata.py
@@ -414,8 +414,9 @@ class FormData(StorableObject):
             empty &= self.data.get(key) is None
         return empty
 
-    def get_all_file_data(self):
+    def get_all_file_data(self, with_history=False):
         from wcs.wf.form import WorkflowFormEvolutionPart
+        from wcs.workflows import ContentSnapshotPart
 
         for field_data in itertools.chain((self.data or {}).values(), (self.workflow_data or {}).values()):
             if misc.is_upload(field_data):
@@ -433,6 +434,11 @@ class FormData(StorableObject):
                 for field_data in (part.data or {}).values():
                     if misc.is_upload(field_data):
                         yield field_data
+            elif isinstance(part, ContentSnapshotPart):
+                # look in old and new values (belt and suspenders)
+                for field_data in list((part.old_data or {}).values()) + list((part.new_data or {}).values()):
+                    if misc.is_upload(field_data):
+                        yield field_data
 
     @classmethod
     def get_actionable_count(cls, user_roles):
diff --git a/wcs/formdef.py b/wcs/formdef.py
index 555bb222e..e3b0c44e6 100644
--- a/wcs/formdef.py
+++ b/wcs/formdef.py
@@ -2143,7 +2143,7 @@ def clean_unused_files(publisher, **kwargs):
                 if is_upload(option_data):
                     yield option_data.get_fs_filename()
             for formdata in formdef.data_class().select_iterator(ignore_errors=True, itersize=200):
-                for field_data in formdata.get_all_file_data():
+                for field_data in formdata.get_all_file_data(with_history=True):
                     if is_upload(field_data):
                         yield field_data.get_fs_filename()
                     elif is_attachment(field_data):
-- 
2.35.1

