From cdcc9c110597e9095d9763b90e8eb25e3054e2ca Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Mon, 9 Dec 2019 12:27:55 +0100
Subject: [PATCH] utils: factorize use of unicodedata module (#38237)

---
 passerelle/apps/base_adresse/models.py    |  5 ++---
 passerelle/apps/csvdatasource/models.py   | 10 +++-------
 passerelle/apps/gdc/views.py              |  5 ++---
 passerelle/apps/solis/models.py           | 18 +-----------------
 passerelle/apps/sp_fr/models.py           |  4 ++--
 passerelle/contrib/agoraplus/normalize.py |  5 +++--
 passerelle/contrib/agoraplus/views.py     |  6 +-----
 passerelle/utils/conversion.py            |  8 +++++++-
 8 files changed, 21 insertions(+), 40 deletions(-)

diff --git a/passerelle/apps/base_adresse/models.py b/passerelle/apps/base_adresse/models.py
index 1f3c6be2..8ea08989 100644
--- a/passerelle/apps/base_adresse/models.py
+++ b/passerelle/apps/base_adresse/models.py
@@ -1,7 +1,6 @@
 import bz2
 import json
 import urlparse
-import unicodedata
 
 from requests import RequestException
 
@@ -15,7 +14,7 @@ from django.utils.six.moves.urllib.parse import urljoin
 
 from passerelle.base.models import BaseResource
 from passerelle.utils.api import endpoint
-from passerelle.utils.conversion import simplify
+from passerelle.utils.conversion import normalize_lower, simplify
 from passerelle.utils.jsonresponse import APIError
 
 
@@ -383,7 +382,7 @@ class BaseAdresse(BaseResource):
 class UnaccentNameMixin(object):
 
     def save(self, *args, **kwargs):
-        self.unaccent_name = unicodedata.normalize('NFKD', self.name).encode('ascii', 'ignore').lower()
+        self.unaccent_name = normalize_lower(self.name)
         super(UnaccentNameMixin, self).save(*args, **kwargs)
 
 
diff --git a/passerelle/apps/csvdatasource/models.py b/passerelle/apps/csvdatasource/models.py
index 7a06e6ba..52e09a27 100644
--- a/passerelle/apps/csvdatasource/models.py
+++ b/passerelle/apps/csvdatasource/models.py
@@ -17,7 +17,6 @@
 import os
 import re
 import csv
-import unicodedata
 from collections import OrderedDict
 
 import six
@@ -39,6 +38,7 @@ from passerelle.base.models import BaseResource
 from passerelle.utils import batch
 from passerelle.utils.jsonresponse import APIError
 from passerelle.utils.api import endpoint
+from passerelle.utils.conversion import normalize_lower
 
 identifier_re = re.compile(r"^[^\d\W]\w*\Z", re.UNICODE)
 
@@ -56,10 +56,6 @@ def get_code(expr):
     return code_cache[expr]
 
 
-def normalize(value):
-    return unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
-
-
 class Query(models.Model):
     resource = models.ForeignKey('CsvDataSource', on_delete=models.CASCADE)
     slug = models.SlugField(_('Name (slug)'))
@@ -325,7 +321,7 @@ class CsvDataSource(BaseResource):
                 row_vars['query'] = kwargs
                 for i, (code, expr) in enumerate(codes):
                     try:
-                        result = eval(code, {'normalize': normalize}, row_vars)
+                        result = eval(code, {'normalize_lower': normalize_lower}, row_vars)
                     except Exception as e:
                         data = {
                             'expr': expr,
@@ -400,7 +396,7 @@ class CsvDataSource(BaseResource):
         # filtering is done there after projection because we need a projection named text for
         # retro-compatibility with previous use of the csvdatasource with select2
         if 'q' in request.GET:
-            filters = ["%s in normalize(text.lower())" % repr(normalize(request.GET['q'].lower()))]
+            filters = ["%s in normalize_lower(text)" % repr(normalize_lower(request.GET['q']))]
             data = [row for new_row, row in stream_expressions(filters, data, kind='filters')
                     if new_row[0]]
 
diff --git a/passerelle/apps/gdc/views.py b/passerelle/apps/gdc/views.py
index f6b1d401..5601c4b6 100644
--- a/passerelle/apps/gdc/views.py
+++ b/passerelle/apps/gdc/views.py
@@ -1,11 +1,11 @@
 import json
-import unicodedata
 
 from django.http import Http404
 from django.views.generic.base import View
 from django.views.generic.detail import SingleObjectMixin, DetailView
 
 import passerelle.utils as utils
+from passerelle.utils.conversion import normalize
 
 from .models import Gdc, phpserialize, phpserialize_loads, SOAPpy
 
@@ -138,8 +138,7 @@ class PostDemandeView(View, SingleObjectMixin):
             except GdcCrash:
                 result = {'result': 'gdc soap crash'}
                 return utils.response_for_json(request, result)
-            normalized_voie = unicodedata.normalize('NFKD', voie_str).encode('ascii', 'ignore').decode('ascii')
-            normalized_voie = normalized_voie.upper()
+            normalized_voie = normalize(voie_str).decode('ascii').upper()
             for k, v in voies:
                 if v == normalized_voie or k == normalized_voie:
                     voie_id = k
diff --git a/passerelle/apps/solis/models.py b/passerelle/apps/solis/models.py
index 76474040..1f444fad 100644
--- a/passerelle/apps/solis/models.py
+++ b/passerelle/apps/solis/models.py
@@ -18,7 +18,6 @@ import base64
 import copy
 import json
 import re
-import unicodedata
 
 from django.db import models
 from django.template.loader import get_template
@@ -29,22 +28,7 @@ from django.utils.http import urlencode
 from passerelle.base.models import BaseResource
 from passerelle.utils.api import endpoint
 from passerelle.utils.jsonresponse import APIError
-from passerelle.utils.conversion import to_pdf
-
-
-def simplify(s):
-    '''
-    Simplify a string, trying to transform it to lower ascii chars (a-z, 0-9)
-    and minimize spaces. Used to compare strings on ?q=something requests.
-    '''
-    if not s:
-        return ''
-    if not isinstance(s, unicode):
-        s = unicode(s, 'utf-8', 'ignore')
-    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
-    s = re.sub(r'[^\w\s\'-]', '', s)
-    s = re.sub(r'[\s\'_-]+', ' ', s)
-    return s.strip().lower()
+from passerelle.utils.conversion import simplify, to_pdf
 
 
 def unflat(flatten_dict, separator='_'):
diff --git a/passerelle/apps/sp_fr/models.py b/passerelle/apps/sp_fr/models.py
index 114ff483..640a0535 100644
--- a/passerelle/apps/sp_fr/models.py
+++ b/passerelle/apps/sp_fr/models.py
@@ -22,7 +22,6 @@ import zipfile
 import collections
 import base64
 import datetime
-import unicodedata
 
 from lxml import etree as ET
 
@@ -40,6 +39,7 @@ from passerelle.utils.api import endpoint
 from passerelle.utils.sftp import SFTPField
 from passerelle.utils.wcs import FormDefField, get_wcs_choices
 from passerelle.utils.xml import text_content
+from passerelle.utils.conversion import normalize
 
 from .xsd import Schema
 
@@ -82,7 +82,7 @@ def simplify(s):
         return ''
     if not isinstance(s, six.text_type):
         s = six.text_type(s, 'utf-8', 'ignore')
-    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
+    s = normalize(s)
     s = re.sub(r'[^\w\s\'-_]', '', s)
     s = s.replace('-', '_')
     s = re.sub(r'[\s\']+', '', s)
diff --git a/passerelle/contrib/agoraplus/normalize.py b/passerelle/contrib/agoraplus/normalize.py
index 2af86628..636ab748 100644
--- a/passerelle/contrib/agoraplus/normalize.py
+++ b/passerelle/contrib/agoraplus/normalize.py
@@ -14,7 +14,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import unicodedata
 import re
 from datetime import datetime, timedelta
 from decimal import Decimal
@@ -23,6 +22,8 @@ import logging
 # handle years before 1900
 from django.utils import datetime_safe
 
+from passerelle.utils.conversion import normalize
+
 # NNNN@mailprov.no emails are not real ones: we will ignore them.
 # (They are used by Agora+ to create "fake" login for each user id=NNNN)
 PROVISIONING_DOMAIN = 'mailprov.no'
@@ -41,7 +42,7 @@ def slugify(s, space='-'):
         return ''
     if not isinstance(s, unicode):
         s = s.decode('utf-8', errors='ignore')
-    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
+    s = normalize(s)
     s = re.sub(r'[^\w\s\'%s]' % space, '', s).strip().lower()
     s = re.sub(r'[\s\'%s]+' % space, space, s)
     return s
diff --git a/passerelle/contrib/agoraplus/views.py b/passerelle/contrib/agoraplus/views.py
index cb7e20f4..4ff3ce4f 100644
--- a/passerelle/contrib/agoraplus/views.py
+++ b/passerelle/contrib/agoraplus/views.py
@@ -15,7 +15,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
-import unicodedata
 import logging
 
 from django.core.exceptions import ObjectDoesNotExist
@@ -27,6 +26,7 @@ from django.utils.translation import ugettext_lazy as _
 from django.utils.http import urlencode
 
 import passerelle.utils as utils
+from passerelle.utils.conversion import normalize_lower
 
 from .models import AgoraPlus, AgoraPlusLink, AgoraAPIError
 from .wcs import Formdata
@@ -36,10 +36,6 @@ from .provisioning import provision_attributes, provision_roles
 logger = logging.getLogger('passerelle.contrib.agoraplus')
 
 
-def normalize_lower(s):
-    return unicodedata.normalize('NFKD', s).encode('ascii', 'ignore').lower()
-
-
 class AgoraPlusDetailView(GenericDetailView):
     model = AgoraPlus
     template_name = 'passerelle/contrib/agoraplus/detail.html'
diff --git a/passerelle/utils/conversion.py b/passerelle/utils/conversion.py
index 7c5e3070..14a00914 100644
--- a/passerelle/utils/conversion.py
+++ b/passerelle/utils/conversion.py
@@ -94,6 +94,12 @@ def exception_to_text(e):
     return u'%s(%s)' % (e.__class__.__name__, content)
 
 
+def normalize(s):
+    return unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
+
+def normalize_lower(s):
+    return normalize(s).lower()
+
 def simplify(s):
     '''
     Simplify a string, trying to transform it to lower ascii chars (a-z, 0-9)
@@ -103,7 +109,7 @@ def simplify(s):
         return ''
     if not isinstance(s, unicode):
         s = unicode(s, 'utf-8', 'ignore')
-    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
+    s = normalize(s)
     s = re.sub(r'[^\w\s\'-]', '', s)
     s = re.sub(r'[\s\'_-]+', ' ', s)
     return s.strip().lower()
-- 
2.24.0

