From 58adf622197f976a7c398c9081af6c01ddafb754 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Mon, 9 Dec 2019 12:27:55 +0100
Subject: [PATCH] utils: factorize use of unicodedata module (#38237)

---
 passerelle/apps/csvdatasource/models.py |  6 +-----
 passerelle/apps/gdc/views.py            |  6 ++----
 passerelle/apps/solis/models.py         | 17 +----------------
 passerelle/apps/sp_fr/models.py         |  4 ++--
 passerelle/utils/conversion.py          |  6 +++++-
 5 files changed, 11 insertions(+), 28 deletions(-)

diff --git a/passerelle/apps/csvdatasource/models.py b/passerelle/apps/csvdatasource/models.py
index a8d85467..76ce4636 100644
--- a/passerelle/apps/csvdatasource/models.py
+++ b/passerelle/apps/csvdatasource/models.py
@@ -12,17 +12,16 @@
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import re
 import csv
-import unicodedata
 from collections import OrderedDict
 
 import six
 import pytz
 
 from pyexcel_ods import get_data as get_data_ods
 from pyexcel_xls import get_data as get_data_xls
 
@@ -34,16 +33,17 @@ from django.conf import settings
 from django.db import models, transaction
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext_lazy as _
 
 from passerelle.base.models import BaseResource
 from passerelle.utils import batch
 from passerelle.utils.jsonresponse import APIError
 from passerelle.utils.api import endpoint
+from passerelle.utils.conversion import normalize
 
 identifier_re = re.compile(r"^[^\d\W]\w*\Z", re.UNICODE)
 
 
 code_cache = OrderedDict()
 
 
 def get_code(expr):
@@ -51,20 +51,16 @@ def get_code(expr):
     if len(code_cache) > 1024:
         for key in list(code_cache.keys())[:len(code_cache) - 1024]:
             code_cache.pop(key)
     if expr not in code_cache:
         code_cache[expr] = compile(expr, '<inline>', 'eval')
     return code_cache[expr]
 
 
-def normalize(value):
-    return unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
-
-
 class Query(models.Model):
     resource = models.ForeignKey('CsvDataSource', on_delete=models.CASCADE)
     slug = models.SlugField(_('Name (slug)'))
     label = models.CharField(_('Label'), max_length=100)
     description = models.TextField(_('Description'), blank=True)
     filters = models.TextField(
         _('Filters'),
         blank=True,
diff --git a/passerelle/apps/gdc/views.py b/passerelle/apps/gdc/views.py
index ae9af14d..48410f4f 100644
--- a/passerelle/apps/gdc/views.py
+++ b/passerelle/apps/gdc/views.py
@@ -1,16 +1,15 @@
-import unicodedata
-
 from django.http import Http404
 from django.views.generic.base import View
 from django.views.generic.detail import SingleObjectMixin, DetailView
 
 from passerelle.compat import json_loads
 import passerelle.utils as utils
+from passerelle.utils.conversion import normalize
 
 from .models import Gdc, phpserialize, phpserialize_loads, SOAPpy
 
 
 class GdcCrash(Exception):
     pass
 
 
@@ -133,18 +132,17 @@ class PostDemandeView(View, SingleObjectMixin):
         if voie_str and not voie_id:
             # look for a voie with that name, so we can provide an identifier
             # to gdc
             try:
                 voies = get_voies(self.get_object().service_url, insee)
             except GdcCrash:
                 result = {'result': 'gdc soap crash'}
                 return utils.response_for_json(request, result)
-            normalized_voie = unicodedata.normalize('NFKD', voie_str).encode('ascii', 'ignore').decode('ascii')
-            normalized_voie = normalized_voie.upper()
+            normalized_voie = normalize(voie_str).decode('ascii').upper()
             for k, v in voies:
                 if v == normalized_voie or k == normalized_voie:
                     voie_id = k
                     break
 
         objet = self.request.GET.get('objet')
         if objet is None:
             objet = data['fields'].get('objet_raw')
diff --git a/passerelle/apps/solis/models.py b/passerelle/apps/solis/models.py
index a34a73b3..63524da8 100644
--- a/passerelle/apps/solis/models.py
+++ b/passerelle/apps/solis/models.py
@@ -12,43 +12,28 @@
 # GNU Affero General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import base64
 import copy
 import re
-import unicodedata
 
 from django.db import models
 from django.template.loader import get_template
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 from django.utils.http import urlencode
 
 from passerelle.base.models import BaseResource
 from passerelle.compat import json_loads
 from passerelle.utils.api import endpoint
 from passerelle.utils.jsonresponse import APIError
-from passerelle.utils.conversion import to_pdf
-
-
-def simplify(s):
-    '''
-    Simplify a string, trying to transform it to lower ascii chars (a-z, 0-9)
-    and minimize spaces. Used to compare strings on ?q=something requests.
-    '''
-    if not s:
-        return ''
-    s = force_text(s, 'utf-8', 'ignore')
-    s = force_text(unicodedata.normalize('NFKD', s).encode('ascii', 'ignore'))
-    s = re.sub(r'[^\w\s\'-]', '', s)
-    s = re.sub(r'[\s\'_-]+', ' ', s)
-    return s.strip().lower()
+from passerelle.utils.conversion import simplify, to_pdf
 
 
 def unflat(flatten_dict, separator='_'):
     '''
     Expand a "flatten" dict:
     >>> unflat({'foo': 'bar', 'two_foo': 'one', 'two_bar': 'two'})
     {'foo': 'bar', 'two': {'foo': 'one', 'bar': 'two'}}
     '''
diff --git a/passerelle/apps/sp_fr/models.py b/passerelle/apps/sp_fr/models.py
index 114ff483..640a0535 100644
--- a/passerelle/apps/sp_fr/models.py
+++ b/passerelle/apps/sp_fr/models.py
@@ -17,17 +17,16 @@
 
 import re
 import os
 import stat
 import zipfile
 import collections
 import base64
 import datetime
-import unicodedata
 
 from lxml import etree as ET
 
 from django.core.urlresolvers import reverse
 from django.core.files import File
 from django.db import models, transaction
 from django.template import engines
 from django.utils import six
@@ -35,16 +34,17 @@ from django.utils.translation import ugettext_lazy as _, ugettext
 
 from jsonfield import JSONField
 
 from passerelle.base.models import BaseResource
 from passerelle.utils.api import endpoint
 from passerelle.utils.sftp import SFTPField
 from passerelle.utils.wcs import FormDefField, get_wcs_choices
 from passerelle.utils.xml import text_content
+from passerelle.utils.conversion import normalize
 
 from .xsd import Schema
 
 MAX_REQUESTS_PER_ITERATION = 200
 
 PROCEDURE_DOC = 'DOC'
 PROCEDURE_RCO = 'recensementCitoyen'
 PROCEDURE_DDPACS = 'depotDossierPACS'
@@ -77,17 +77,17 @@ ET.register_namespace('dgme-metier', 'http://finances.gouv.fr/dgme/gf/composants
 
 
 def simplify(s):
     '''Simplify XML node tag names because XSD from DGME are garbage'''
     if not s:
         return ''
     if not isinstance(s, six.text_type):
         s = six.text_type(s, 'utf-8', 'ignore')
-    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
+    s = normalize(s)
     s = re.sub(r'[^\w\s\'-_]', '', s)
     s = s.replace('-', '_')
     s = re.sub(r'[\s\']+', '', s)
     return s.strip().lower()
 
 
 class Resource(BaseResource):
     category = _('Business Process Connectors')
diff --git a/passerelle/utils/conversion.py b/passerelle/utils/conversion.py
index 475e16d3..ab9a34a4 100644
--- a/passerelle/utils/conversion.py
+++ b/passerelle/utils/conversion.py
@@ -91,25 +91,29 @@ def exception_to_text(e):
             content = six.text_type(repr(args)) if args != [] else ''
         except Exception:
             content = '<exception-while-rendering-args>'
     except AttributeError:
         content = ''
     return u'%s(%s)' % (e.__class__.__name__, content)
 
 
+def normalize(s):
+    return unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
+
+
 def simplify(s):
     '''
     Simplify a string, trying to transform it to lower ascii chars (a-z, 0-9)
     and minimize spaces. Used to compare strings on ?q=something requests.
     '''
     if not s:
         return ''
     s = force_text(s, 'utf-8', 'ignore')
-    s = force_str(unicodedata.normalize('NFKD', s).encode('ascii', 'ignore'))
+    s = force_text(normalize(s))
     s = re.sub(r'[^\w\s\'-]', '', s)
     s = re.sub(r'[\s\'_-]+', ' ', s)
     return s.strip().lower()
 
 
 def num2deg(xtile, ytile, zoom):
     # http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Python
     n = 2.0 ** zoom
-- 
2.25.1

