From 6b69e78be88936d3c959e11068139459fbb9d9ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Thu, 24 Nov 2022 20:04:14 +0100
Subject: [PATCH] misc: remove conversion code for pickle files created with
 python 2 (#71692)

---
 tests/test_formdef.py  | 74 ------------------------------------------
 wcs/fields.py          | 10 ------
 wcs/formdef.py         |  4 +--
 wcs/qommon/__init__.py |  3 --
 wcs/qommon/storage.py  | 38 +---------------------
 wcs/sql.py             |  8 ++---
 wcs/workflows.py       | 10 +-----
 7 files changed, 7 insertions(+), 140 deletions(-)

diff --git a/tests/test_formdef.py b/tests/test_formdef.py
index cb24464e5..73bace6fc 100644
--- a/tests/test_formdef.py
+++ b/tests/test_formdef.py
@@ -4,10 +4,8 @@ import io
 import json
 import os
 import pickle
-import time
 
 import pytest
-from django.utils.encoding import force_bytes
 
 from wcs import fields
 from wcs.admin.settings import UserFieldsFormDef
@@ -653,78 +651,6 @@ def test_get_formdefs_of_all_kinds(pub):
     ]
 
 
-def test_pickle_2to3_conversion(pub):
-    FormDef.wipe()
-    Workflow.wipe()
-
-    workflow = Workflow(name='blah')
-    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
-    workflow.backoffice_fields_formdef.fields = [
-        fields.StringField(id='bo0', varname='foo_bovar', type='string', label='bo variable'),
-    ]
-    status = workflow.add_status('Status1')
-    display_form = status.add_action('form', id='_display_form')
-    display_form.by = []
-    display_form.varname = 'blah'
-    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
-    display_form.formdef.fields.append(
-        fields.StringField(id='1', label='Test', varname='str', type='string', required=True)
-    )
-    workflow.store()
-
-    formdef = FormDef()
-    formdef.name = 'basic formdef'
-    formdef.workflow_id = workflow.id
-    formdef.workflow_options = {'bo0': 'whatever'}
-    formdef.workflow_roles = {'_receiver': '1'}
-    formdef.fields = [
-        StringField(id='1', label='Test', type='string', varname='foo'),
-    ]
-    formdef.store()
-
-    formdef_id = formdef.id
-    formdef_filename = os.path.join(formdef.get_objects_dir(), str(formdef.id))
-    workflow_filename = os.path.join(workflow.get_objects_dir(), str(workflow.id))
-
-    # turn pickle to bytes
-
-    def deep_str2bytes(obj, seen=None):
-        # reverse deep_bytes2str
-        if seen is None:
-            seen = {}
-        if obj is None or isinstance(obj, (int, float, bytes, time.struct_time, type(Ellipsis))):
-            return obj
-        if id(obj) in seen:
-            return obj
-        if isinstance(obj, str):
-            return force_bytes(obj)
-        seen[id(obj)] = True
-        if isinstance(obj, dict):
-            new_d = {}
-            for k, v in obj.items():
-                new_d[force_bytes(k)] = deep_str2bytes(v, seen)
-            return new_d
-        if isinstance(obj, list):
-            return [deep_str2bytes(x, seen) for x in obj]
-        if hasattr(obj, '__class__') and obj.__class__.__module__.startswith(('wcs.', 'qommon.', 'modules.')):
-            obj.__dict__ = deep_str2bytes(obj.__dict__, seen)
-            return obj
-        return obj
-
-    formdef.__dict__ = deep_str2bytes(formdef.__dict__)
-    with open(formdef_filename, 'wb') as fd:
-        pickle.dump(formdef, fd, protocol=2)
-
-    workflow.__dict__ = deep_str2bytes(workflow.__dict__)
-    with open(workflow_filename, 'wb') as fd:
-        pickle.dump(workflow, fd, protocol=2)
-
-    formdef = FormDef.get(formdef_id)
-    assert formdef.fields[0].label == 'Test'
-    assert formdef.workflow.possible_status[0].items[0].varname == 'blah'
-    assert formdef.workflow.possible_status[0].items[0].formdef.fields[0].varname == 'str'
-
-
 def test_wipe_on_object(pub):
     FormDef.wipe()
 
diff --git a/wcs/fields.py b/wcs/fields.py
index 5c994c6e9..a130884ad 100644
--- a/wcs/fields.py
+++ b/wcs/fields.py
@@ -607,16 +607,6 @@ class Field:
             self.display_locations.append('listings')
             changed = True
             self.in_listing = None
-        # repair dictionary attributes that may have been kept as bytes in
-        # the initial python 2 -> 3 conversion.
-        from wcs.qommon.storage import deep_bytes2str
-
-        for key in ('prefill', 'data_source'):
-            value = getattr(self, key, None)
-            if not value:
-                continue
-            if b'type' in value:
-                setattr(self, key, deep_bytes2str(getattr(self, key)))
         return changed
 
     def evaluate_condition(self, dict_vars, formdef, condition):
diff --git a/wcs/formdef.py b/wcs/formdef.py
index 4aa330ef7..e2bdeb0e8 100644
--- a/wcs/formdef.py
+++ b/wcs/formdef.py
@@ -38,7 +38,7 @@ from quixote.http_request import Upload
 from . import data_sources, fields
 from .categories import Category
 from .formdata import FormData
-from .qommon import PICKLE_KWARGS, _, force_str, get_cfg, pgettext_lazy
+from .qommon import _, force_str, get_cfg, pgettext_lazy
 from .qommon.admin.emails import EmailsDirectory
 from .qommon.cron import CronJob
 from .qommon.form import Form, HtmlWidget, UploadedFile
@@ -1968,7 +1968,7 @@ class FormDef(StorableObject):
             return o
         if cls.lightweight:
             try:
-                o.fields = pickle.load(fd, **PICKLE_KWARGS)
+                o.fields = pickle.load(fd)
             except EOFError:
                 pass  # old format
         return o
diff --git a/wcs/qommon/__init__.py b/wcs/qommon/__init__.py
index 8480c0874..33b66beb2 100644
--- a/wcs/qommon/__init__.py
+++ b/wcs/qommon/__init__.py
@@ -34,9 +34,6 @@ except ImportError:
 # and a proper unicode string in Python 3
 force_str = force_text
 
-# unpickle python2 strings as bytes
-PICKLE_KWARGS = {'encoding': 'bytes', 'fix_imports': True}
-
 
 def gettext(message):
     pub = get_publisher()
diff --git a/wcs/qommon/storage.py b/wcs/qommon/storage.py
index 3cb33a5e8..cc37fd3a3 100644
--- a/wcs/qommon/storage.py
+++ b/wcs/qommon/storage.py
@@ -32,7 +32,6 @@ from django.utils.decorators import classonlymethod
 from django.utils.encoding import force_bytes
 from quixote import get_publisher
 
-from . import PICKLE_KWARGS, force_str
 from .vendor import locket
 
 # add compatibility names in case those were stored in pickles
@@ -107,39 +106,6 @@ def atomic_write(path, content, async_op=False):
         doit()
 
 
-def deep_bytes2str(obj, seen=None):
-    # Convert obj loaded by unpickle(encoding='bytes') to a proper object using
-    # strings; this is required as encoding='utf-8' is not possible when there
-    # are pickled datetime objects. <https://bugs.python.org/issue22005>
-    if seen is None:
-        seen = {}
-    if obj is None or isinstance(obj, (int, float, str, time.struct_time, type(Ellipsis))):
-        return obj
-    if isinstance(obj, bytes):
-        try:
-            return obj.decode('utf-8')
-        except UnicodeDecodeError:
-            return obj
-    if isinstance(obj, list):
-        return [deep_bytes2str(x, seen) for x in obj]
-    if isinstance(obj, dict):
-        new_d = {}
-        for k, v in obj.items():
-            new_d[force_str(k)] = deep_bytes2str(v, seen)
-        return new_d
-    if id(obj) in seen:
-        return obj
-    seen[id(obj)] = True
-    if hasattr(obj, '__class__') and obj.__class__.__module__.startswith(('wcs.', 'qommon.', 'modules.')):
-        obj.__dict__ = deep_bytes2str(obj.__dict__, seen)
-        return obj
-    return obj
-
-
-def pickle_2to3_conversion(obj):
-    obj.__dict__ = deep_bytes2str(obj.__dict__)  # inplace
-
-
 class Criteria:
     def __init__(self, attribute, value, **kwargs):
         self.attribute = attribute
@@ -678,7 +644,7 @@ class StorableObject:
             unpickler = get_publisher().unpickler_class
         else:
             unpickler = pickle.Unpickler
-        return unpickler(fd, **PICKLE_KWARGS).load()
+        return unpickler(fd).load()
 
     @classmethod
     def get_filename(cls, filename, ignore_errors=False, ignore_migration=False, **kwargs):
@@ -711,8 +677,6 @@ class StorableObject:
                 fd.close()
         if cls._reset_class:
             o.__class__ = cls
-        if any(isinstance(k, bytes) for k in o.__dict__):
-            pickle_2to3_conversion(o)
         if not ignore_migration:
             o.id = str(o.id)  # makes sure 'id' is a string
             if hasattr(cls, 'migrate'):
diff --git a/wcs/sql.py b/wcs/sql.py
index 8fdb4a528..a095c7def 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -52,13 +52,13 @@ import wcs.roles
 import wcs.snapshots
 import wcs.tracking_code
 import wcs.users
-from wcs.qommon import PICKLE_KWARGS, force_str
+from wcs.qommon import force_str
 
 from . import qommon
 from .publisher import UnpicklerClass
 from .qommon import _, get_cfg
 from .qommon.misc import JSONEncoder, strftime
-from .qommon.storage import NothingToUpdate, _take, classonlymethod, deep_bytes2str
+from .qommon.storage import NothingToUpdate, _take, classonlymethod
 from .qommon.storage import parse_clause as parse_storage_clause
 from .qommon.substitution import invalidate_substitution_cache
 from .qommon.upload_storage import PicklableUpload
@@ -166,9 +166,7 @@ class LazyEvolutionList(list):
 def pickle_loads(value):
     if hasattr(value, 'tobytes'):
         value = value.tobytes()
-    obj = UnpicklerClass(io.BytesIO(force_bytes(value)), **PICKLE_KWARGS).load()
-    obj = deep_bytes2str(obj)
-    return obj
+    return UnpicklerClass(io.BytesIO(force_bytes(value))).load()
 
 
 class Criteria(qommon.storage.Criteria):
diff --git a/wcs/workflows.py b/wcs/workflows.py
index 868434cbc..83b3749dd 100644
--- a/wcs/workflows.py
+++ b/wcs/workflows.py
@@ -64,14 +64,7 @@ from .qommon.misc import (
     get_as_datetime,
     xml_node_text,
 )
-from .qommon.storage import (
-    Contains,
-    Null,
-    StorableObject,
-    StrictNotEqual,
-    atomic_write,
-    pickle_2to3_conversion,
-)
+from .qommon.storage import Contains, Null, StorableObject, StrictNotEqual, atomic_write
 from .qommon.template import Template, TemplateError
 from .qommon.upload_storage import PicklableUpload, get_storage_object
 from .roles import get_user_roles, logged_users_role
@@ -994,7 +987,6 @@ class Workflow(StorableObject):
 
     def __setstate__(self, dict):
         self.__dict__.update(dict)
-        pickle_2to3_conversion(self)
         for s in self.possible_status + (self.global_actions or []):
             s.parent = self
             triggers = getattr(s, 'triggers', None) or []
-- 
2.38.1

