Projet

Général

Profil

0001-sql-switch-fts-normalization-to-unidecode-57009.patch

Frédéric Péters, 20 octobre 2021 09:00

Télécharger (2,47 ko)

Voir les différences: 

Subject: [PATCH] sql: switch fts normalization to unidecode (#57009)


 debian/control | 1 +
 setup.py       | 1 +
 wcs/sql.py     | 7 ++++---
 3 files changed, 6 insertions(+), 3 deletions(-)
debian/control
28 28 
         python3-pyproj,
29 29 
         python3-quixote,
30 30 
         python3-requests,
31 
         pytohn3-unidecode,
31 32 
         python3-uwsgidecorators,
32 33 
         python3-vobject,
33 34 
         python3-xstatic-leaflet,
setup.py
174 174 
        'XStatic-Leaflet',
175 175 
        'XStatic-Leaflet-GestureHandling',
176 176 
        'pyproj',
177 
        'unidecode',
177 178 
    ],
178 179 
    package_dir={'wcs': 'wcs'},
179 180 
    packages=find_packages(),
wcs/sql.py
20 20 
import json
21 21 
import re
22 22 
import time
23 
import unicodedata
24 23 
import uuid
25 24 

  		




  26
  25
  
    
import psycopg2

  		




  27
  26
  
    
import psycopg2.extensions

  		




  28
  27
  
    
import psycopg2.extras

  		




  
  28
  
    
import unidecode

  		




  29
  29
  
    

  		




  30
  30
  
    
try:

  		




  31
  31
  
    
    import cPickle as pickle

  		




  ......




  284
  284
  
    

  		




  285
  285
  
    
    @classmethod

  		




  286
  286
  
    
    def get_fts_value(cls, value):

  		




  287
  
  
    
        return unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')

  		




  
  287
  
    
        return unidecode.unidecode(value)

  		




  288
  288
  
    

  		




  289
  289
  
    
    def as_sql(self):

  		




  290
  290
  
    
        return 'fts @@ plainto_tsquery(%%(c%s)s)' % id(self.value)

  		




  ......




  3450
  3450
  
    
# latest migration, number + description (description is not used

  		




  3451
  3451
  
    
# programmaticaly but will make sure git conflicts if two migrations are

  		




  3452
  3452
  
    
# separately added with the same number)

  		




  3453
  
  
    
SQL_LEVEL = (54, 'add patch column on snapshot table')

  		




  
  3453
  
    
SQL_LEVEL = (55, 'update full text normalisation (switch to unidecode)')

  		




  3454
  3454
  
    

  		




  3455
  3455
  
    

  		




  3456
  3456
  
    
def migrate_global_views(conn, cur):

  		




  ......




  3587
  3587
  
    
        # 38: extract submission_agent_id to its own column

  		




  3588
  3588
  
    
        # 41: update full text normalization

  		




  3589
  3589
  
    
        # 51: add index on formdata blockdef fields

  		




  
  3590
  
    
        # 55: update full text normalisation (switch to unidecode)

  		




  3590
  3591
  
    
        set_reindex('formdata', 'needed', conn=conn, cur=cur)

  		




  3591
  3592
  
    
    if sql_level < 46:

  		




  3592
  3593
  
    
        from wcs.carddef import CardDef

  		




  3593
  
  
    
-