Projet

Général

Profil

0001-general-replace-unicode-calls-by-force_text-37572.patch

Frédéric Péters, 11 novembre 2019 20:30

Télécharger (49,1 ko)

Voir les différences:

Subject: [PATCH] general: replace unicode() calls by force_text() (#37572)

 tests/test_form_pages.py          |  3 +-
 wcs/admin/bounces.py              |  4 ++-
 wcs/admin/users.py                |  4 ++-
 wcs/api.py                        | 11 ++++---
 wcs/backoffice/management.py      | 17 +++++-----
 wcs/ctl/check_hobos.py            |  8 +++--
 wcs/data_sources.py               |  3 +-
 wcs/fields.py                     | 26 +++++++--------
 wcs/formdef.py                    | 30 +++++++++--------
 wcs/portfolio.py                  |  7 ++--
 wcs/qommon/__init__.py            |  5 +--
 wcs/qommon/form.py                |  5 +--
 wcs/qommon/misc.py                | 11 ++++---
 wcs/qommon/ods.py                 |  4 ++-
 wcs/qommon/publisher.py           |  5 +--
 wcs/qommon/saml2.py               |  3 +-
 wcs/qommon/template.py            |  6 ++--
 wcs/qommon/templatetags/qommon.py | 11 ++++---
 wcs/qommon/xml_storage.py         |  3 +-
 wcs/roles.py                      | 11 ++++---
 wcs/sql.py                        |  8 +++--
 wcs/variables.py                  |  9 +++---
 wcs/wf/export_to_model.py         |  3 +-
 wcs/wf/wscall.py                  |  6 ++--
 wcs/workflows.py                  | 53 ++++++++++++++++---------------
 wcs/wscalls.py                    |  7 ++--
 26 files changed, 147 insertions(+), 116 deletions(-)
tests/test_form_pages.py
20 20
    Image = None
21 21

  
22 22
from quixote.http_request import Upload as QuixoteUpload
23
from django.utils.encoding import force_text
23 24
from wcs.qommon.emails import docutils
24 25
from wcs.qommon.form import UploadedFile
25 26
from wcs.qommon.ident.password_accounts import PasswordAccount
......
2017 2018
    assert data.data == {'0': {
2018 2019
        'sha1': hashlib.sha1(password).hexdigest(),
2019 2020
        'md5': hashlib.md5(password).hexdigest(),
2020
        'cleartext': unicode(password, 'utf-8'),
2021
        'cleartext': force_text(password, 'utf-8'),
2021 2022
        }}
2022 2023

  
2023 2024
def test_form_password_field_submit(pub):
wcs/admin/bounces.py
17 17
import time
18 18
import email.Parser
19 19

  
20
from django.utils.encoding import force_text
21

  
20 22
from quixote import get_response, redirect
21 23
from quixote.directory import Directory
22 24
from quixote.html import htmltext, TemplateIO
......
100 102
            subject, charset = email.Header.decode_header(msg['Subject'])[0]
101 103
            if charset:
102 104
                encoding = get_publisher().site_charset
103
                r += unicode(subject, charset).encode(encoding)
105
                r += force_text(subject, charset).encode(encoding)
104 106
            else:
105 107
                r += subject
106 108
            r += htmltext('</div>')
wcs/admin/users.py
14 14
# You should have received a copy of the GNU General Public License
15 15
# along with this program; if not, see <http://www.gnu.org/licenses/>.
16 16

  
17
from django.utils.encoding import force_text
18

  
17 19
from quixote import get_publisher, get_response, get_request, get_session, redirect
18 20
from quixote.directory import Directory
19 21
from quixote.html import TemplateIO, htmltext
......
433 435
        if get_request().form.get('q'):
434 436
            q = get_request().form.get('q')
435 437
            if type(q) is not unicode:
436
                q = unicode(q, get_publisher().site_charset)
438
                q = force_text(q, get_publisher().site_charset)
437 439
            r += htmltext('<input name="q" value="%s">') % q.encode(get_publisher().site_charset)
438 440
        else:
439 441
            r += htmltext('<input name="q">')
wcs/api.py
22 22
from quixote import get_request, get_publisher, get_response, get_session
23 23
from quixote.directory import Directory
24 24

  
25
from django.utils.encoding import force_text
25 26
from django.utils.six.moves.urllib import parse as urllib
26 27
from django.http import HttpResponse, HttpResponseBadRequest
27 28

  
......
477 478
                # anonymous API call, mark authentication as required
478 479
                authentication_required = True
479 480

  
480
            formdict = {'title': unicode(formdef.name, charset),
481
            formdict = {'title': force_text(formdef.name, charset),
481 482
                        'slug': formdef.url_name,
482 483
                        'url': formdef.get_url(),
483 484
                        'description': formdef.description or '',
......
509 510
                formdict['functions'][wf_role_id] = workflow_function
510 511

  
511 512
            if formdef.category:
512
                formdict['category'] = unicode(formdef.category.name, charset)
513
                formdict['category_slug'] = unicode(formdef.category.url_name, charset)
513
                formdict['category'] = force_text(formdef.category.name, charset)
514
                formdict['category_slug'] = force_text(formdef.category.url_name, charset)
514 515
                formdict['category_position'] = (formdef.category.position or 0)
515 516
            else:
516 517
                formdict['category_position'] = sys.maxint
......
581 582
        all_formdefs = FormDef.select(order_by='name', ignore_errors=True, lightweight=True)
582 583
        for category in categories:
583 584
            d = {}
584
            d['title'] = unicode(category.name, charset)
585
            d['title'] = force_text(category.name, charset)
585 586
            d['slug'] = category.url_name
586 587
            d['url'] = category.get_url()
587 588
            if category.description:
588
                d['description'] = unicode(str(category.get_description_html_text()), charset)
589
                d['description'] = force_text(str(category.get_description_html_text()), charset)
589 590
            formdefs = ApiFormdefsDirectory(category).get_list_forms(user,
590 591
                    formdefs=all_formdefs, list_all_forms=list_all_forms,
591 592
                    backoffice_submission=backoffice_submission)
wcs/backoffice/management.py
26 26
except ImportError:
27 27
    xlwt = None
28 28

  
29
from django.utils.encoding import force_text
29 30
from django.utils.six.moves.urllib import parse as urllib
30 31
from django.utils.six import StringIO
31 32

  
......
339 340
        if get_request().form.get('q'):
340 341
            q = get_request().form.get('q')
341 342
            if type(q) is not unicode:
342
                q = unicode(q, get_publisher().site_charset)
343
                q = force_text(q, get_publisher().site_charset)
343 344
            r += htmltext('<input name="q" value="%s">') % q.encode(get_publisher().site_charset)
344 345
        else:
345 346
            r += htmltext('<input name="q">')
......
1189 1190
            if get_request().form.get('q'):
1190 1191
                q = get_request().form.get('q')
1191 1192
                if type(q) is not unicode:
1192
                    q = unicode(q, get_publisher().site_charset)
1193
                    q = force_text(q, get_publisher().site_charset)
1193 1194
                r += htmltext('<input class="inline-input" name="q" value="%s">') % q.encode(get_publisher().site_charset)
1194 1195
            else:
1195 1196
                r += htmltext('<input class="inline-input" name="q">')
......
1850 1851
                            get_request().get_server().lower(),
1851 1852
                            formdef.url_name,
1852 1853
                            formdata.id)
1853
                    vevent.add('summary').value = unicode(formdata.get_display_name(), charset)
1854
                    vevent.add('summary').value = force_text(formdata.get_display_name(), charset)
1854 1855
                    vevent.add('dtstart').value = make_datetime(formdata.data[start_date_field_id])
1855 1856
                    if end_date_field_id and formdata.data.get(end_date_field_id):
1856 1857
                        vevent.add('dtend').value = make_datetime(formdata.data[end_date_field_id])
1857 1858
                    vevent.dtstart.value_param = 'DATE'
1858 1859
                    backoffice_url = formdata.get_url(backoffice=True)
1859 1860
                    vevent.add('url').value = backoffice_url
1860
                    form_name = unicode(formdef.name, charset)
1861
                    status_name = unicode(formdata.get_status_label(), charset)
1861
                    form_name = force_text(formdef.name, charset)
1862
                    status_name = force_text(formdata.get_status_label(), charset)
1862 1863
                    description = '%s | %s | %s\n'  % (form_name, formdata.get_display_id(), status_name)
1863 1864
                    description += backoffice_url
1864 1865
                    # TODO: improve performance by loading all users in one
1865 1866
                    # single query before the loop
1866 1867
                    if formdata.user:
1867
                        description += '\n%s' % unicode(formdata.user.get_display_name(), charset)
1868
                        description += '\n%s' % force_text(formdata.user.get_display_name(), charset)
1868 1869
                    vevent.add('description').value = description
1869 1870
                    cal.add(vevent)
1870 1871

  
......
2533 2534
        def safe(v):
2534 2535
            if isinstance(v, str):
2535 2536
                try:
2536
                    unicode(v, charset)
2537
                    force_text(v, charset)
2537 2538
                except UnicodeDecodeError:
2538 2539
                    v = repr(v)
2539 2540
            else:
2540 2541
                try:
2541
                    v = unicode(v).encode(charset)
2542
                    v = force_text(v).encode(charset)
2542 2543
                except:
2543 2544
                    v = repr(v)
2544 2545
            return v
wcs/ctl/check_hobos.py
24 24
import urlparse
25 25
import hashlib
26 26

  
27
from django.utils.encoding import force_text
28

  
27 29
from quixote import cleanup
28 30
from ..qommon import misc
29 31
from ..qommon.ctl import Command, make_option
......
337 339
            if not admin_attribute:
338 340
                admin_attribute = 'is_superuser=true'
339 341
            else:
340
                admin_attribute = unicode(admin_attribute).encode('utf-8')
342
                admin_attribute = force_text(admin_attribute).encode('utf-8')
341 343
            admin_attribute_dict = dict([admin_attribute.split('=')])
342 344
            pub.cfg['idp'][key_provider_id]['admin-attributes'] = admin_attribute_dict
343 345
            pub.cfg['idp'][key_provider_id]['nameidformat'] = 'unspecified'
......
428 430
            if not 'variables' in config.sections():
429 431
                config.add_section('variables')
430 432
            for key, value in variables.items():
431
                key = unicode(key).encode('utf-8')
432
                value = unicode(value).encode('utf-8')
433
                key = force_text(key).encode('utf-8')
434
                value = force_text(value).encode('utf-8')
433 435
                config.set('variables', key, value)
434 436

  
435 437
        if not 'api-secrets' in config.sections():
wcs/data_sources.py
18 18
import hashlib
19 19
import xml.etree.ElementTree as ET
20 20

  
21
from django.utils.encoding import force_text
21 22
from django.utils.six.moves.urllib import parse as urllib
22 23
from django.utils.six.moves.urllib import parse as urlparse
23 24

  
......
317 318
    def export_data_source_to_xml(self, element, attribute_name, charset):
318 319
        data_source = getattr(self, attribute_name)
319 320
        ET.SubElement(element, 'type').text = data_source.get('type')
320
        ET.SubElement(element, 'value').text = unicode(data_source.get('value') or '', charset)
321
        ET.SubElement(element, 'value').text = force_text(data_source.get('value') or '', charset)
321 322

  
322 323
    def import_data_source_from_xml(self, element, charset):
323 324
        return {
wcs/fields.py
28 28
from quixote import get_request, get_publisher
29 29
from quixote.html import htmltext, TemplateIO
30 30

  
31
from django.utils.encoding import smart_text
31
from django.utils.encoding import force_text, smart_text
32 32
from django.utils.formats import date_format as django_date_format
33 33
from django.utils.html import urlize
34 34
from django.utils.six.moves.html_parser import HTMLParser
......
193 193
    @property
194 194
    def unhtmled_label(self):
195 195
        charset = get_publisher().site_charset
196
        return HTMLParser().unescape(unicode(
196
        return HTMLParser().unescape(force_text(
197 197
            re.sub('<.*?>', ' ', self.label), charset)).strip().encode(charset)
198 198

  
199 199
    def get_admin_attributes(self):
......
241 241
                if type(val) is dict:
242 242
                    for k, v in val.items():
243 243
                        if isinstance(v, str):
244
                            text_value = unicode(v, charset, 'replace')
244
                            text_value = force_text(v, charset, errors='replace')
245 245
                        else:
246 246
                            # field having non str value in dictionnary field must overload
247 247
                            # import_to_xml to handle import
248
                            text_value = unicode(v)
248
                            text_value = force_text(v)
249 249
                        ET.SubElement(el, k).text = text_value
250 250
                elif type(val) is list:
251 251
                    if attribute[-1] == 's':
......
253 253
                    else:
254 254
                        atname = 'item'
255 255
                    for v in val:
256
                        ET.SubElement(el, atname).text = unicode(v, charset, 'replace')
256
                        ET.SubElement(el, atname).text = force_text(v, charset, errors='replace')
257 257
                elif type(val) in (str, unicode):
258 258
                    if type(val) is unicode:
259 259
                        el.text = val
260 260
                    else:
261
                        el.text = unicode(val, charset, 'replace')
261
                        el.text = force_text(val, charset, errors='replace')
262 262
                else:
263 263
                    el.text = str(val)
264 264
        return field
......
754 754
        value = value or ''
755 755
        if value.startswith('http://') or value.startswith('https://'):
756 756
            charset = get_publisher().site_charset
757
            value = unicode(value, charset)
757
            value = force_text(value, charset)
758 758
            return htmltext(urlize(value, nofollow=True, autoescape=True).encode(charset))
759 759
        return str(value)
760 760

  
......
1786 1786
        for post_condition in self.post_conditions:
1787 1787
            post_condition_node = ET.SubElement(conditions_node, 'post_condition')
1788 1788
            condition_node = ET.SubElement(post_condition_node, 'condition')
1789
            ET.SubElement(condition_node, 'type').text = unicode(
1790
                    post_condition['condition'].get('type') or '', charset, 'replace')
1791
            ET.SubElement(condition_node, 'value').text = unicode(
1792
                    post_condition['condition'].get('value') or '', charset, 'replace')
1793
            ET.SubElement(post_condition_node, 'error_message').text = unicode(
1794
                    post_condition['error_message'] or '', charset, 'replace')
1789
            ET.SubElement(condition_node, 'type').text = force_text(
1790
                    post_condition['condition'].get('type') or '', charset, errors='replace')
1791
            ET.SubElement(condition_node, 'value').text = force_text(
1792
                    post_condition['condition'].get('value') or '', charset, errors='replace')
1793
            ET.SubElement(post_condition_node, 'error_message').text = force_text(
1794
                    post_condition['error_message'] or '', charset, errors='replace')
1795 1795

  
1796 1796
    def fill_admin_form(self, form):
1797 1797
        form.add(StringWidget, 'label', title = _('Label'), value = self.label,
wcs/formdef.py
24 24
import xml.etree.ElementTree as ET
25 25
import datetime
26 26

  
27
from django.utils.encoding import force_text
28

  
27 29
from quixote import get_request, get_publisher
28 30
from quixote.http_request import Upload
29 31

  
......
721 723
    def export_to_json(self, include_id=False, indent=None, anonymise=True):
722 724
        charset = get_publisher().site_charset
723 725
        root = {}
724
        root['name'] = unicode(self.name, charset)
726
        root['name'] = force_text(self.name, charset)
725 727
        if include_id and self.id:
726 728
            root['id'] = str(self.id)
727 729
        if self.category:
728
            root['category'] = unicode(self.category.name, charset)
730
            root['category'] = force_text(self.category.name, charset)
729 731
            root['category_id'] = str(self.category.id)
730 732
        if self.workflow:
731 733
            root['workflow'] = self.workflow.get_json_export_dict(include_id=include_id)
......
868 870
        for text_attribute in list(self.TEXT_ATTRIBUTES):
869 871
            if not hasattr(self, text_attribute) or not getattr(self, text_attribute):
870 872
                continue
871
            ET.SubElement(root, text_attribute).text = unicode(
873
            ET.SubElement(root, text_attribute).text = force_text(
872 874
                    getattr(self, text_attribute), charset)
873 875
        for boolean_attribute in self.BOOLEAN_ATTRIBUTES:
874 876
            if not hasattr(self, boolean_attribute):
......
882 884

  
883 885
        if self.category:
884 886
            elem = ET.SubElement(root, 'category')
885
            elem.text = unicode(self.category.name, charset)
887
            elem.text = force_text(self.category.name, charset)
886 888
            if include_id:
887 889
                elem.attrib['category_id'] = str(self.category.id)
888 890

  
889 891
        if self.workflow:
890 892
            elem = ET.SubElement(root, 'workflow')
891
            elem.text = unicode(self.workflow.name, charset)
893
            elem.text = force_text(self.workflow.name, charset)
892 894
            if include_id:
893 895
                elem.attrib['workflow_id'] = str(self.workflow.id)
894 896

  
......
921 923
                    continue
922 924
                role_id = str(role_id)
923 925
                if role_id.startswith('_') or role_id == 'logged-users':
924
                    role = unicode(role_id, charset)
926
                    role = force_text(role_id, charset)
925 927
                else:
926 928
                    try:
927
                        role = unicode(Role.get(role_id).name, charset)
929
                        role = force_text(Role.get(role_id).name, charset)
928 930
                    except KeyError:
929
                        role = unicode(role_id, charset)
931
                        role = force_text(role_id, charset)
930 932
                sub = ET.SubElement(roles, 'role')
931 933
                if include_id:
932 934
                    sub.attrib['role_id'] = role_id
......
939 941
                    continue
940 942
                role_id = str(role_id)
941 943
                if role_id.startswith('_') or role_id == 'logged-users':
942
                    role = unicode(role_id, charset)
944
                    role = force_text(role_id, charset)
943 945
                else:
944 946
                    try:
945
                        role = unicode(Role.get(role_id).name, charset)
947
                        role = force_text(Role.get(role_id).name, charset)
946 948
                    except KeyError:
947
                        role = unicode(role_id, charset)
949
                        role = force_text(role_id, charset)
948 950
                sub = ET.SubElement(roles, 'role')
949 951
                sub.attrib['role_key'] = role_key
950 952
                if include_id:
......
957 959
            element.attrib['varname'] = option
958 960
            option_value = self.workflow_options.get(option)
959 961
            if isinstance(option_value, basestring):
960
                element.text = unicode(self.workflow_options.get(option, ''), charset)
962
                element.text = force_text(self.workflow_options.get(option, ''), charset)
961 963
            elif hasattr(option_value, 'base_filename'):
962 964
                ET.SubElement(element, 'filename').text = option_value.base_filename
963 965
                ET.SubElement(element, 'content_type').text = (
......
973 975
        for geoloc_key, geoloc_label in (self.geolocations or {}).items():
974 976
            element = ET.SubElement(geolocations, 'geolocation')
975 977
            element.attrib['key'] = geoloc_key
976
            element.text = unicode(geoloc_label, charset)
978
            element.text = force_text(geoloc_label, charset)
977 979

  
978 980
        if self.required_authentication_contexts:
979 981
            element = ET.SubElement(root, 'required_authentication_contexts')
980 982
            for auth_context in self.required_authentication_contexts:
981
                ET.SubElement(element, 'method').text = unicode(auth_context)
983
                ET.SubElement(element, 'method').text = force_text(auth_context)
982 984

  
983 985
        return root
984 986

  
wcs/portfolio.py
19 19
import urllib
20 20
import base64
21 21

  
22
from django.utils.encoding import force_text
22 23
from django.utils.six.moves.urllib import parse as urlparse
23 24

  
24 25
from .qommon import get_logger
......
85 86
    charset = get_publisher().site_charset
86 87
    payload = {}
87 88
    if user.name_identifiers:
88
        payload['user_nameid'] = unicode(user.name_identifiers[0], 'ascii')
89
        payload['user_nameid'] = force_text(user.name_identifiers[0], 'ascii')
89 90
    elif user.email:
90
        payload['user_email'] = unicode(user.email, 'ascii')
91
        payload['user_email'] = force_text(user.email, 'ascii')
91 92
    payload['origin'] = urlparse.urlparse(get_publisher().get_frontoffice_url()).netloc
92
    payload['file_name'] = unicode(filename, charset)
93
    payload['file_name'] = force_text(filename, charset)
93 94
    stream.seek(0)
94 95
    payload['file_b64_content'] = base64.b64encode(stream.read())
95 96
    async_post = fargo_post_json_async('/api/documents/push/', payload)
wcs/qommon/__init__.py
19 19

  
20 20
import django.apps
21 21
from django.conf import settings
22
from django.utils.encoding import force_text
22 23
from django.utils.six.moves import configparser as ConfigParser
23 24

  
24 25
from quixote import get_publisher
......
32 33
    pub = get_publisher()
33 34
    if pub is None:
34 35
        return message
35
    return unicode(pub.gettext(str(message)), 'utf-8').encode(pub.site_charset)
36
    return force_text(pub.gettext(str(message)), 'utf-8').encode(pub.site_charset)
36 37

  
37 38
def ngettext(*args):
38 39
    pub = get_publisher()
39 40
    if pub is None:
40 41
        return message
41
    return unicode(pub.ngettext(*args), 'utf-8').encode(pub.site_charset)
42
    return force_text(pub.ngettext(*args), 'utf-8').encode(pub.site_charset)
42 43

  
43 44
from .publisher import get_cfg, get_logger, get_publisher_class
44 45
from . import publisher
wcs/qommon/form.py
62 62
from quixote.html import htmltext, htmltag, htmlescape, TemplateIO
63 63
from quixote.util import randbytes
64 64

  
65
from django.utils.encoding import force_text
65 66
from django.utils.six.moves.html_parser import HTMLParser
66 67
from django.utils.six import StringIO
67 68

  
......
870 871
                    self.error = _('invalid address domain')
871 872
                    return
872 873
                if not type(domain) is unicode:
873
                    domain = unicode(domain, 'utf-8', 'ignore')
874
                    domain = force_text(domain, 'utf-8', 'ignore')
874 875
                try:
875 876
                    domain = domain.encode('idna')
876 877
                except UnicodeError:
......
1488 1489
            parser = HTMLParser()
1489 1490
            charset = get_publisher().site_charset
1490 1491
            def unquote_django(matchobj):
1491
                return parser.unescape(unicode(matchobj.group(0), charset)).encode(charset)
1492
                return parser.unescape(force_text(matchobj.group(0), charset)).encode(charset)
1492 1493
            self.value = re.sub('{[{%](.*?)[%}]}', unquote_django, self.value)
1493 1494

  
1494 1495
    def add_media(self):
wcs/qommon/misc.py
37 37

  
38 38
from django.conf import settings
39 39
from django.utils import datetime_safe
40
from django.utils.encoding import force_text
40 41
from django.utils.html import strip_tags
41 42
from django.template import engines, TemplateSyntaxError, VariableDoesNotExist
42 43
from django.utils.six.moves.html_parser import HTMLParser
......
161 162
        return ''
162 163
    if not isinstance(s, unicode):
163 164
        if get_publisher() and get_publisher().site_charset:
164
            s = unicode('%s' % s, get_publisher().site_charset, 'ignore')
165
            s = force_text('%s' % s, get_publisher().site_charset, 'ignore')
165 166
        else:
166
            s = unicode('%s' % s, 'iso-8859-1', 'ignore')
167
            s = force_text('%s' % s, 'iso-8859-1', 'ignore')
167 168
    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
168 169
    s = re.sub(r'[^\w\s\'_\-%s]' % space, '', s).strip().lower()
169 170
    s = re.sub(r'[\s\'_\-%s]+' % space, space, s)
......
241 242
    if isinstance(s, str):
242 243
        return s
243 244
    if not isinstance(s, unicode):
244
        s = unicode(s)
245
        s = force_text(s)
245 246
    return s.encode(get_publisher().site_charset)
246 247

  
247 248
def ellipsize(s, length = 30):
248 249
    if type(s) is not unicode:
249
        s = unicode(s, get_publisher().site_charset, 'replace')
250
        s = force_text(s, get_publisher().site_charset, 'replace')
250 251
    if not s or len(s) < length:
251 252
        return s.encode(get_publisher().site_charset)
252 253
    return s[:length-5].encode(get_publisher().site_charset) + ' (...)'
......
615 616

  
616 617
def html2text(text):
617 618
    if isinstance(text, (htmltext, str)):
618
        text = unicode(str(text), get_publisher().site_charset)
619
        text = force_text(str(text), get_publisher().site_charset)
619 620
    return site_encode(HTMLParser().unescape(strip_tags(text)))
620 621

  
621 622

  
wcs/qommon/ods.py
20 20
import xml.etree.ElementTree as ET
21 21
import zipfile
22 22

  
23
from django.utils.encoding import force_text
24

  
23 25
from .misc import date_format, datetime_format, strftime
24 26
from .evalutils import make_date, make_datetime
25 27

  
......
168 170
        if value is None:
169 171
            value = ''
170 172
        if type(value) is not unicode:
171
            value = unicode(value, 'utf-8')
173
            value = force_text(value, 'utf-8')
172 174
        self.worksheet = worksheet
173 175
        for i in range(0x20):  # remove control characters
174 176
            char = chr(i)
wcs/qommon/publisher.py
44 44
from django.conf import settings
45 45
from django.http import Http404
46 46
from django.utils import translation
47
from django.utils.encoding import force_text
47 48
from django.utils.translation import gettext, ngettext
48 49

  
49 50
from quixote.publish import Publisher, get_request, get_response, get_publisher, redirect
......
180 181
    def _generate_plaintext_error(self, request, original_response,
181 182
                                  exc_type, exc_value, tb, limit = None):
182 183
        if exc_value:
183
            exc_value = unicode(str(exc_value), errors='ignore').encode('ascii')
184
            exc_value = force_text(str(exc_value), errors='ignore').encode('ascii')
184 185
        if not self.USE_LONG_TRACES:
185 186
            if not request:
186 187
                # this happens when an exception is raised by an afterjob
......
269 270

  
270 271
        error_summary = traceback.format_exception_only(exc_type, exc_value)
271 272
        error_summary = error_summary[0][0:-1] # de-listify and strip newline
272
        error_summary = unicode(str(error_summary), errors='ignore').encode('ascii')
273
        error_summary = force_text(str(error_summary), errors='ignore').encode('ascii')
273 274

  
274 275
        plain_error_msg = self._generate_plaintext_error(request,
275 276
                                                         original_response,
wcs/qommon/saml2.py
24 24
except ImportError:
25 25
    lasso = None
26 26

  
27
from django.utils.encoding import force_text
27 28
from django.utils.six.moves.urllib import parse as urlparse
28 29

  
29 30
from quixote import get_request, get_response, redirect, get_field, get_publisher
......
938 939

  
939 940
    def metadata(self):
940 941
        try:
941
            metadata = unicode(open(misc.get_abs_path(
942
            metadata = force_text(open(misc.get_abs_path(
942 943
                            get_cfg('sp')['saml2_metadata'])).read(), 'utf-8')
943 944
        except KeyError:
944 945
            raise errors.TraversalError()
wcs/qommon/template.py
24 24
                             TemplateSyntaxError as DjangoTemplateSyntaxError,
25 25
                             VariableDoesNotExist as DjangoVariableDoesNotExist)
26 26
from django.template.loader import render_to_string
27
from django.utils.encoding import smart_text
27
from django.utils.encoding import force_text, smart_text
28 28
from django.utils.safestring import SafeString, SafeUnicode
29 29

  
30 30
from quixote import get_session, get_request, get_response, get_publisher
......
99 99
    publisher = get_publisher()
100 100
    def encode_string(x):
101 101
        if publisher:
102
            return unicode(x).encode(publisher.site_charset)
102
            return force_text(x).encode(publisher.site_charset)
103 103
        return x
104 104
    name = encode_string(tree.attrib['name'])
105 105
    version = tree.attrib.get('version')
......
533 533
    if isinstance(value, SafeString):
534 534
        return SafeUnicode(value, 'utf-8')
535 535
    if isinstance(value, str):
536
        return unicode(value, 'utf-8')
536
        return force_text(value, 'utf-8')
537 537
    return value
538 538

  
539 539
if not getattr(django.template.base.Variable, 'monkey_patched', False):
wcs/qommon/templatetags/qommon.py
29 29
from django import template
30 30
from django.template import defaultfilters
31 31
from django.utils import dateparse
32
from django.utils.encoding import force_text
32 33
from django.utils.safestring import mark_safe
33 34
from wcs.qommon import evalutils
34 35
from wcs.qommon import tokens
......
45 46

  
46 47
@register.filter
47 48
def startswith(string, substring):
48
    return string and unicode(string).startswith(unicode(substring))
49
    return string and force_text(string).startswith(force_text(substring))
49 50

  
50 51
@register.filter
51 52
def split(string, separator=' '):
52 53
    if not string:
53 54
        return []
54
    return unicode(string).split(unicode(separator))
55
    return force_text(string).split(force_text(separator))
55 56

  
56 57
@register.filter
57 58
def strip(string, chars=None):
58 59
    if not string:
59 60
        return ''
60 61
    if chars:
61
        return unicode(string).strip(unicode(chars))
62
        return force_text(string).strip(force_text(chars))
62 63
    else:
63
        return unicode(string).strip()
64
        return force_text(string).strip()
64 65

  
65 66
@register.filter
66 67
def parse_date(date_string):
......
316 317

  
317 318
@register.filter
318 319
def token_check(token1, token2):
319
    return unicode(token1).strip().upper() == unicode(token2).strip().upper()
320
    return force_text(token1).strip().upper() == force_text(token2).strip().upper()
320 321

  
321 322

  
322 323
def get_latlon(obj):
wcs/qommon/xml_storage.py
17 17
import datetime
18 18
import xml.etree.ElementTree as ET
19 19

  
20
from django.utils.encoding import force_text
20 21
from quixote import get_publisher
21 22

  
22 23
from .misc import indent_xml
......
53 54
        return root
54 55

  
55 56
    def export_str_to_xml(self, element, attribute_name, charset):
56
        element.text = unicode(getattr(self, attribute_name), charset)
57
        element.text = force_text(getattr(self, attribute_name), charset)
57 58

  
58 59
    def export_int_to_xml(self, element, attribute_name, charset):
59 60
        element.text = str(getattr(self, attribute_name))
wcs/roles.py
14 14
# You should have received a copy of the GNU General Public License
15 15
# along with this program; if not, see <http://www.gnu.org/licenses/>.
16 16

  
17
from django.utils.encoding import force_text
17 18
from quixote import get_publisher
18 19

  
19 20
from .qommon import _, misc
......
86 87
    def get_json_export_dict(self):
87 88
        charset = get_publisher().site_charset
88 89
        return {
89
            'name': unicode(self.name, charset),
90
            'text': unicode(self.name, charset), # generic key
90
            'name': force_text(self.name, charset),
91
            'text': force_text(self.name, charset), # generic key
91 92
            'allows_backoffice_access': self.allows_backoffice_access,
92
            'emails': [unicode(email, charset) for email in self.emails or []],
93
            'details': unicode(self.details or '', charset),
93
            'emails': [force_text(email, charset) for email in self.emails or []],
94
            'details': force_text(self.details or '', charset),
94 95
            'emails_to_members': self.emails_to_members,
95
            'slug': unicode(self.slug, charset),
96
            'slug': force_text(self.slug, charset),
96 97
            'id': self.id}
97 98

  
98 99
    @classmethod
wcs/sql.py
22 22
import re
23 23
import cPickle
24 24

  
25
from django.utils.encoding import force_text
26

  
25 27
from quixote import get_publisher
26 28
from . import qommon
27 29
from .qommon.storage import _take, parse_clause as parse_storage_clause
......
260 262

  
261 263
def site_unicode(value):
262 264
    if not isinstance(value, basestring):
263
        value = unicode(value)
265
        value = force_text(value)
264 266
    if isinstance(value, unicode):
265 267
        return value
266
    return unicode(value, get_publisher().site_charset)
268
    return force_text(value, get_publisher().site_charset)
267 269

  
268 270
def get_connection(new=False):
269 271
    if new:
......
1175 1177
                elif field.key == 'password':
1176 1178
                    d = {}
1177 1179
                    for fmt, val in value:
1178
                        d[fmt] = unicode(val, 'utf-8')
1180
                        d[fmt] = force_text(val, 'utf-8')
1179 1181
                    value = d
1180 1182
                if sql_type == 'date':
1181 1183
                    value = value.timetuple()
wcs/variables.py
16 16

  
17 17
import warnings
18 18

  
19
from django.utils.encoding import force_text
19 20
from django.utils.functional import SimpleLazyObject
20 21

  
21 22
from quixote import get_publisher, get_request
......
513 514
        if self._field.convert_value_to_str:
514 515
            return self._field.convert_value_to_str(value)
515 516
        if isinstance(value, str):
516
            return unicode(value, get_publisher().site_charset)
517
            return force_text(value, get_publisher().site_charset)
517 518
        return value
518 519

  
519 520
    def __str__(self):
......
533 534
        return value in self.get_value()
534 535

  
535 536
    def __unicode__(self):
536
        return unicode(str(self), get_publisher().site_charset)
537
        return force_text(str(self), get_publisher().site_charset)
537 538

  
538 539
    def __eq__(self, other):
539
        return unicode(self) == unicode(other)
540
        return force_text(self) == force_text(other)
540 541

  
541 542
    def __ne__(self, other):
542
        return unicode(self) != unicode(other)
543
        return force_text(self) != force_text(other)
543 544

  
544 545
    def __getitem__(self, key):
545 546
        if isinstance(key, (int, slice)):
wcs/wf/export_to_model.py
23 23
import tempfile
24 24
import shutil
25 25

  
26
from django.utils.encoding import force_text
26 27
from django.utils.six import StringIO
27 28

  
28 29
from quixote import get_response, get_request, get_publisher
......
420 421
                if isinstance(t, unicode):
421 422
                    t = t.encode(get_publisher().site_charset)
422 423
                t = template_on_context(context, t, autoescape=False)
423
                return unicode(t, get_publisher().site_charset)
424
                return force_text(t, get_publisher().site_charset)
424 425
            for node in root.iter():
425 426
                got_blank_lines = False
426 427
                # apply template to user-field-decl and update user-field-get
wcs/wf/wscall.py
23 23
import mimetypes
24 24
from StringIO import StringIO
25 25

  
26
from django.utils.encoding import force_text
27

  
26 28
from quixote.html import TemplateIO, htmltext
27 29

  
28 30
from ..qommon import _
......
477 479
            if type(key) is unicode:
478 480
                ET.SubElement(item, 'name').text = key
479 481
            elif type(key) is str:
480
                ET.SubElement(item, 'name').text = unicode(key, charset, 'replace')
482
                ET.SubElement(item, 'name').text = force_text(key, charset, errors='replace')
481 483
            else:
482 484
                raise AssertionError('unknown type for key (%r)' % key)
483 485
            if type(value) is unicode:
484 486
                ET.SubElement(item, 'value').text = value
485 487
            elif type(value) is str:
486
                ET.SubElement(item, 'value').text = unicode(value, charset, 'replace')
488
                ET.SubElement(item, 'value').text = force_text(value, charset, errors='replace')
487 489
            else:
488 490
                raise AssertionError('unknown type for value (%r)' % key)
489 491

  
wcs/workflows.py
25 25
import time
26 26
import uuid
27 27

  
28
from django.utils.encoding import force_text
28 29
from django.utils.six import StringIO
29 30

  
30 31
from quixote import get_request, get_response, redirect
......
520 521
        root = ET.Element('workflow')
521 522
        if include_id and self.id and not str(self.id).startswith('_'):
522 523
            root.attrib['id'] = str(self.id)
523
        ET.SubElement(root, 'name').text = unicode(self.name, charset)
524
        ET.SubElement(root, 'name').text = force_text(self.name, charset)
524 525

  
525 526
        roles_node = ET.SubElement(root, 'roles')
526 527
        if self.roles:
527 528
            for role_id, role_label in self.roles.items():
528 529
                role_node = ET.SubElement(roles_node, 'role')
529 530
                role_node.attrib['id'] = role_id
530
                role_node.text = unicode(role_label, charset)
531
                role_node.text = force_text(role_label, charset)
531 532

  
532 533
        if self.last_modification_time:
533 534
            elem = ET.SubElement(root, 'last_modification')
......
667 668
    def get_json_export_dict(self, include_id=False):
668 669
        charset = get_publisher().site_charset
669 670
        root = {}
670
        root['name'] = unicode(self.name, charset)
671
        root['name'] = force_text(self.name, charset)
671 672
        if include_id and self.id:
672 673
            root['id'] = str(self.id)
673 674
        if self.last_modification_time:
......
675 676
                                                           self.last_modification_time)
676 677
        roles = root['functions'] = {}
677 678
        for role, label in self.roles.iteritems():
678
            roles[role] = unicode(label, charset)
679
            roles[role] = force_text(label, charset)
679 680
        statuses = root['statuses'] = []
680 681
        endpoint_status_ids = [s.id for s in self.get_endpoint_status()]
681 682
        waitpoint_status_ids = [s.id for s in self.get_waitpoint_status()]
682 683
        for status in self.possible_status:
683 684
            statuses.append({
684 685
                'id': status.id,
685
                'name': unicode(status.name, charset),
686
                'name': force_text(status.name, charset),
686 687
                'forced_endpoint': status.forced_endpoint,
687 688
                'endpoint': status.id in endpoint_status_ids,
688 689
                'waitpoint': status.id in waitpoint_status_ids,
......
845 846
                val = getattr(self, attribute)
846 847
                if type(val) is dict:
847 848
                    for k, v in val.items():
848
                        ET.SubElement(el, k).text = unicode(v, charset, 'replace')
849
                        ET.SubElement(el, k).text = force_text(v, charset, errors='replace')
849 850
                elif type(val) is list:
850 851
                    if attribute[-1] == 's':
851 852
                        atname = attribute[:-1]
852 853
                    else:
853 854
                        atname = 'item'
854 855
                    for v in val:
855
                        ET.SubElement(el, atname).text = unicode(str(v), charset, 'replace')
856
                        ET.SubElement(el, atname).text = force_text(str(v), charset, errors='replace')
856 857
                elif type(val) in (str, unicode):
857 858
                    if type(val) is unicode:
858 859
                        el.text = val
859 860
                    else:
860
                        el.text = unicode(val, charset, 'replace')
861
                        el.text = force_text(val, charset, errors='replace')
861 862
                else:
862 863
                    el.text = str(val)
863 864
        return node
......
903 904
                continue
904 905
            role_id = str(role_id)
905 906
            if role_id.startswith('_') or role_id == 'logged-users':
906
                role = unicode(role_id, charset)
907
                role = force_text(role_id, charset)
907 908
            else:
908 909
                try:
909
                    role = unicode(Role.get(role_id).name, charset)
910
                    role = force_text(Role.get(role_id).name, charset)
910 911
                except KeyError:
911
                    role = unicode(role_id, charset)
912
                    role = force_text(role_id, charset)
912 913
            sub = ET.SubElement(el, 'item')
913 914
            sub.attrib['role_id'] = role_id
914 915
            sub.text = role
......
928 929
            return
929 930
        role_id = str(getattr(self, attribute))
930 931
        if role_id.startswith('_') or role_id == 'logged-users':
931
            role = unicode(role_id, charset)
932
            role = force_text(role_id, charset)
932 933
        else:
933 934
            try:
934
                role = unicode(Role.get(role_id).name, charset)
935
                role = force_text(Role.get(role_id).name, charset)
935 936
            except KeyError:
936
                role_id = role = unicode(role_id, charset)
937
                role_id = role = force_text(role_id, charset)
937 938
        sub = ET.SubElement(item, attribute)
938 939
        if include_id:
939 940
            sub.attrib['role_id'] = role_id
......
1343 1344

  
1344 1345
    def export_to_xml(self, charset, include_id=False):
1345 1346
        status = ET.Element('action')
1346
        ET.SubElement(status, 'id').text = unicode(self.id, charset)
1347
        ET.SubElement(status, 'name').text = unicode(self.name, charset)
1347
        ET.SubElement(status, 'id').text = force_text(self.id, charset)
1348
        ET.SubElement(status, 'name').text = force_text(self.name, charset)
1348 1349

  
1349 1350
        if self.backoffice_info_text:
1350
            ET.SubElement(status, 'backoffice_info_text').text = unicode(
1351
            ET.SubElement(status, 'backoffice_info_text').text = force_text(
1351 1352
                    self.backoffice_info_text, charset)
1352 1353

  
1353 1354
        items = ET.SubElement(status, 'items')
......
1397 1398

  
1398 1399
    def export_to_xml(self, charset, include_id=False):
1399 1400
        level = ET.Element('criticality-level')
1400
        ET.SubElement(level, 'id').text = unicode(self.id, charset) if self.id else ''
1401
        ET.SubElement(level, 'name').text = unicode(self.name, charset)
1401
        ET.SubElement(level, 'id').text = force_text(self.id, charset) if self.id else ''
1402
        ET.SubElement(level, 'name').text = force_text(self.name, charset)
1402 1403
        if self.colour:
1403
            ET.SubElement(level, 'colour').text = unicode(self.colour, charset)
1404
            ET.SubElement(level, 'colour').text = force_text(self.colour, charset)
1404 1405
        return level
1405 1406

  
1406 1407
    def init_with_xml(self, elem, charset, include_id=False):
......
1629 1630

  
1630 1631
    def export_to_xml(self, charset, include_id=False):
1631 1632
        status = ET.Element('status')
1632
        ET.SubElement(status, 'id').text = unicode(self.id, charset)
1633
        ET.SubElement(status, 'name').text = unicode(self.name, charset)
1634
        ET.SubElement(status, 'colour').text = unicode(self.colour, charset)
1633
        ET.SubElement(status, 'id').text = force_text(self.id, charset)
1634
        ET.SubElement(status, 'name').text = force_text(self.name, charset)
1635
        ET.SubElement(status, 'colour').text = force_text(self.colour, charset)
1635 1636
        if self.extra_css_class:
1636
            ET.SubElement(status, 'extra_css_class').text = unicode(self.extra_css_class, charset)
1637
            ET.SubElement(status, 'extra_css_class').text = force_text(self.extra_css_class, charset)
1637 1638

  
1638 1639
        if self.forced_endpoint:
1639 1640
            ET.SubElement(status, 'forced_endpoint').text = 'true'
1640 1641

  
1641 1642
        if self.backoffice_info_text:
1642
            ET.SubElement(status, 'backoffice_info_text').text = unicode(
1643
            ET.SubElement(status, 'backoffice_info_text').text = force_text(
1643 1644
                    self.backoffice_info_text, charset)
1644 1645

  
1645 1646
        visibility_node = ET.SubElement(status, 'visibility')
......
2293 2294
            if type(self.button_label) is unicode:
2294 2295
                el.text = self.button_label
2295 2296
            elif type(self.button_label) is str:
2296
                el.text = unicode(self.button_label, charset, 'replace')
2297
                el.text = force_text(self.button_label, charset, errors='replace')
2297 2298
            else:
2298 2299
                raise AssertionError('unknown type for button_label (%r)' % self.button_label)
2299 2300

  
wcs/wscalls.py
19 19
import sys
20 20
import xml.etree.ElementTree as ET
21 21

  
22
from django.utils.encoding import force_text
22 23
from django.utils.six.moves.urllib import parse as urllib
23 24
from django.utils.six.moves.urllib import parse as urlparse
24 25

  
......
144 145
    def export_request_to_xml(self, element, attribute_name, charset):
145 146
        request = getattr(self, attribute_name)
146 147
        for attr in ('url', 'request_signature_key', 'method'):
147
            ET.SubElement(element, attr).text = unicode(request.get(attr) or '', charset)
148
            ET.SubElement(element, attr).text = force_text(request.get(attr) or '', charset)
148 149
        for attr in ('qs_data', 'post_data'):
149 150
            data_element = ET.SubElement(element, attr)
150 151
            for k, v in (request.get(attr) or {}).items():
151 152
                sub = ET.SubElement(data_element, 'param')
152
                sub.attrib['key'] = unicode(k, charset)
153
                sub.text = unicode(v, charset)
153
                sub.attrib['key'] = force_text(k, charset)
154
                sub.text = force_text(v, charset)
154 155
        if request.get('post_formdata'):
155 156
            ET.SubElement(element, 'post_formdata')
156 157

  
157
-