Project

General

Profile

0001-misc-remove-conversion-code-for-pickle-files-created.patch

Frédéric Péters, 25 November 2022 07:47 AM

Download (10.4 KB)

View differences:

Subject: [PATCH] misc: remove conversion code for pickle files created with
 python 2 (#71692)

 tests/test_formdef.py  | 74 ------------------------------------------
 wcs/fields.py          | 10 ------
 wcs/formdef.py         |  4 +--
 wcs/qommon/__init__.py |  3 --
 wcs/qommon/storage.py  | 38 +---------------------
 wcs/sql.py             |  8 ++---
 wcs/workflows.py       | 10 +-----
 7 files changed, 7 insertions(+), 140 deletions(-)
tests/test_formdef.py
4 4
import json
5 5
import os
6 6
import pickle
7
import time
8 7

  
9 8
import pytest
10
from django.utils.encoding import force_bytes
11 9

  
12 10
from wcs import fields
13 11
from wcs.admin.settings import UserFieldsFormDef
......
653 651
    ]
654 652

  
655 653

  
656
def test_pickle_2to3_conversion(pub):
657
    FormDef.wipe()
658
    Workflow.wipe()
659

  
660
    workflow = Workflow(name='blah')
661
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
662
    workflow.backoffice_fields_formdef.fields = [
663
        fields.StringField(id='bo0', varname='foo_bovar', type='string', label='bo variable'),
664
    ]
665
    status = workflow.add_status('Status1')
666
    display_form = status.add_action('form', id='_display_form')
667
    display_form.by = []
668
    display_form.varname = 'blah'
669
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
670
    display_form.formdef.fields.append(
671
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True)
672
    )
673
    workflow.store()
674

  
675
    formdef = FormDef()
676
    formdef.name = 'basic formdef'
677
    formdef.workflow_id = workflow.id
678
    formdef.workflow_options = {'bo0': 'whatever'}
679
    formdef.workflow_roles = {'_receiver': '1'}
680
    formdef.fields = [
681
        StringField(id='1', label='Test', type='string', varname='foo'),
682
    ]
683
    formdef.store()
684

  
685
    formdef_id = formdef.id
686
    formdef_filename = os.path.join(formdef.get_objects_dir(), str(formdef.id))
687
    workflow_filename = os.path.join(workflow.get_objects_dir(), str(workflow.id))
688

  
689
    # turn pickle to bytes
690

  
691
    def deep_str2bytes(obj, seen=None):
692
        # reverse deep_bytes2str
693
        if seen is None:
694
            seen = {}
695
        if obj is None or isinstance(obj, (int, float, bytes, time.struct_time, type(Ellipsis))):
696
            return obj
697
        if id(obj) in seen:
698
            return obj
699
        if isinstance(obj, str):
700
            return force_bytes(obj)
701
        seen[id(obj)] = True
702
        if isinstance(obj, dict):
703
            new_d = {}
704
            for k, v in obj.items():
705
                new_d[force_bytes(k)] = deep_str2bytes(v, seen)
706
            return new_d
707
        if isinstance(obj, list):
708
            return [deep_str2bytes(x, seen) for x in obj]
709
        if hasattr(obj, '__class__') and obj.__class__.__module__.startswith(('wcs.', 'qommon.', 'modules.')):
710
            obj.__dict__ = deep_str2bytes(obj.__dict__, seen)
711
            return obj
712
        return obj
713

  
714
    formdef.__dict__ = deep_str2bytes(formdef.__dict__)
715
    with open(formdef_filename, 'wb') as fd:
716
        pickle.dump(formdef, fd, protocol=2)
717

  
718
    workflow.__dict__ = deep_str2bytes(workflow.__dict__)
719
    with open(workflow_filename, 'wb') as fd:
720
        pickle.dump(workflow, fd, protocol=2)
721

  
722
    formdef = FormDef.get(formdef_id)
723
    assert formdef.fields[0].label == 'Test'
724
    assert formdef.workflow.possible_status[0].items[0].varname == 'blah'
725
    assert formdef.workflow.possible_status[0].items[0].formdef.fields[0].varname == 'str'
726

  
727

  
728 654
def test_wipe_on_object(pub):
729 655
    FormDef.wipe()
730 656

  
wcs/fields.py
607 607
            self.display_locations.append('listings')
608 608
            changed = True
609 609
            self.in_listing = None
610
        # repair dictionary attributes that may have been kept as bytes in
611
        # the initial python 2 -> 3 conversion.
612
        from wcs.qommon.storage import deep_bytes2str
613

  
614
        for key in ('prefill', 'data_source'):
615
            value = getattr(self, key, None)
616
            if not value:
617
                continue
618
            if b'type' in value:
619
                setattr(self, key, deep_bytes2str(getattr(self, key)))
620 610
        return changed
621 611

  
622 612
    def evaluate_condition(self, dict_vars, formdef, condition):
wcs/formdef.py
38 38
from . import data_sources, fields
39 39
from .categories import Category
40 40
from .formdata import FormData
41
from .qommon import PICKLE_KWARGS, _, force_str, get_cfg, pgettext_lazy
41
from .qommon import _, force_str, get_cfg, pgettext_lazy
42 42
from .qommon.admin.emails import EmailsDirectory
43 43
from .qommon.cron import CronJob
44 44
from .qommon.form import Form, HtmlWidget, UploadedFile
......
1968 1968
            return o
1969 1969
        if cls.lightweight:
1970 1970
            try:
1971
                o.fields = pickle.load(fd, **PICKLE_KWARGS)
1971
                o.fields = pickle.load(fd)
1972 1972
            except EOFError:
1973 1973
                pass  # old format
1974 1974
        return o
wcs/qommon/__init__.py
34 34
# and a proper unicode string in Python 3
35 35
force_str = force_text
36 36

  
37
# unpickle python2 strings as bytes
38
PICKLE_KWARGS = {'encoding': 'bytes', 'fix_imports': True}
39

  
40 37

  
41 38
def gettext(message):
42 39
    pub = get_publisher()
wcs/qommon/storage.py
32 32
from django.utils.encoding import force_bytes
33 33
from quixote import get_publisher
34 34

  
35
from . import PICKLE_KWARGS, force_str
36 35
from .vendor import locket
37 36

  
38 37
# add compatibility names in case those were stored in pickles
......
107 106
        doit()
108 107

  
109 108

  
110
def deep_bytes2str(obj, seen=None):
111
    # Convert obj loaded by unpickle(encoding='bytes') to a proper object using
112
    # strings; this is required as encoding='utf-8' is not possible when there
113
    # are pickled datetime objects. <https://bugs.python.org/issue22005>
114
    if seen is None:
115
        seen = {}
116
    if obj is None or isinstance(obj, (int, float, str, time.struct_time, type(Ellipsis))):
117
        return obj
118
    if isinstance(obj, bytes):
119
        try:
120
            return obj.decode('utf-8')
121
        except UnicodeDecodeError:
122
            return obj
123
    if isinstance(obj, list):
124
        return [deep_bytes2str(x, seen) for x in obj]
125
    if isinstance(obj, dict):
126
        new_d = {}
127
        for k, v in obj.items():
128
            new_d[force_str(k)] = deep_bytes2str(v, seen)
129
        return new_d
130
    if id(obj) in seen:
131
        return obj
132
    seen[id(obj)] = True
133
    if hasattr(obj, '__class__') and obj.__class__.__module__.startswith(('wcs.', 'qommon.', 'modules.')):
134
        obj.__dict__ = deep_bytes2str(obj.__dict__, seen)
135
        return obj
136
    return obj
137

  
138

  
139
def pickle_2to3_conversion(obj):
140
    obj.__dict__ = deep_bytes2str(obj.__dict__)  # inplace
141

  
142

  
143 109
class Criteria:
144 110
    def __init__(self, attribute, value, **kwargs):
145 111
        self.attribute = attribute
......
678 644
            unpickler = get_publisher().unpickler_class
679 645
        else:
680 646
            unpickler = pickle.Unpickler
681
        return unpickler(fd, **PICKLE_KWARGS).load()
647
        return unpickler(fd).load()
682 648

  
683 649
    @classmethod
684 650
    def get_filename(cls, filename, ignore_errors=False, ignore_migration=False, **kwargs):
......
711 677
                fd.close()
712 678
        if cls._reset_class:
713 679
            o.__class__ = cls
714
        if any(isinstance(k, bytes) for k in o.__dict__):
715
            pickle_2to3_conversion(o)
716 680
        if not ignore_migration:
717 681
            o.id = str(o.id)  # makes sure 'id' is a string
718 682
            if hasattr(cls, 'migrate'):
wcs/sql.py
52 52
import wcs.snapshots
53 53
import wcs.tracking_code
54 54
import wcs.users
55
from wcs.qommon import PICKLE_KWARGS, force_str
55
from wcs.qommon import force_str
56 56

  
57 57
from . import qommon
58 58
from .publisher import UnpicklerClass
59 59
from .qommon import _, get_cfg
60 60
from .qommon.misc import JSONEncoder, strftime
61
from .qommon.storage import NothingToUpdate, _take, classonlymethod, deep_bytes2str
61
from .qommon.storage import NothingToUpdate, _take, classonlymethod
62 62
from .qommon.storage import parse_clause as parse_storage_clause
63 63
from .qommon.substitution import invalidate_substitution_cache
64 64
from .qommon.upload_storage import PicklableUpload
......
166 166
def pickle_loads(value):
167 167
    if hasattr(value, 'tobytes'):
168 168
        value = value.tobytes()
169
    obj = UnpicklerClass(io.BytesIO(force_bytes(value)), **PICKLE_KWARGS).load()
170
    obj = deep_bytes2str(obj)
171
    return obj
169
    return UnpicklerClass(io.BytesIO(force_bytes(value))).load()
172 170

  
173 171

  
174 172
class Criteria(qommon.storage.Criteria):
wcs/workflows.py
64 64
    get_as_datetime,
65 65
    xml_node_text,
66 66
)
67
from .qommon.storage import (
68
    Contains,
69
    Null,
70
    StorableObject,
71
    StrictNotEqual,
72
    atomic_write,
73
    pickle_2to3_conversion,
74
)
67
from .qommon.storage import Contains, Null, StorableObject, StrictNotEqual, atomic_write
75 68
from .qommon.template import Template, TemplateError
76 69
from .qommon.upload_storage import PicklableUpload, get_storage_object
77 70
from .roles import get_user_roles, logged_users_role
......
994 987

  
995 988
    def __setstate__(self, dict):
996 989
        self.__dict__.update(dict)
997
        pickle_2to3_conversion(self)
998 990
        for s in self.possible_status + (self.global_actions or []):
999 991
            s.parent = self
1000 992
            triggers = getattr(s, 'triggers', None) or []
1001
-