Changeset 183 for trunk/aeres/scripts/build_firstname_id.py

Timestamp:

04/10/12 14:20:30 (12 years ago)

Author:

pinsard

Message:

progress on Excel reading

File:

• trunk/aeres/scripts/build_firstname_id.py (modified) (4 diffs)

trunk/aeres/scripts/build_firstname_id.py

@@ -18 +18 @@
 
 :ref:`firstname_id.xsl`
+
+:mod:`unidecode`
 
 EXAMPLES
@@ -46 +48 @@
 import string
 import sys
+import re
+
+from unidecode import unidecode
 
 def build_firstname_id(firstname):
@@ -53 +58 @@
     >>> firstname = []
     []
-    >>> firstname.append(' Ginette')
-    [' Ginette']
-    >>> firstname.append('Gin ette ')
-    [' Ginette', 'Gin  ette']
-    >>> firstname.append("G\'in ette ")
-    [' Ginette', 'Gin ette', "G\'in ette"]
+    >>> firstname.append(u' Ginette')
+    >>> firstname
+    [u' Ginette']
+    >>> firstname.append(u'Gin ette ')
+    >>> firstname
+    [u' Ginette', u'Gin  ette ']
+    >>> firstname.append(u"G\'in ette ")
+    >>> firstname
+    [u' Ginette', u'Gin ette ', u"G\'in ette "]
+    >>> firstname.append(u"Gïnette")
+    >>> firstname
+    [u' Ginette', u'Gin ette ', u"G'in ette ", u'G\xefnette']
     >>> firstname_id = build_firstname_id(firstname)
-    ['ginette','ginette','ginette']
+    >>> #firstname_id
+    ['ginette', 'ginette', 'ginette', 'ginette']
     """
 
@@ -67 +79 @@
         sys.exit(-1)
 
-    # convert to str
-    #++firstname_str = [str(item) for item in firstname]
-    #++firstname_str = [item.encode('iso-8859-1','replace') for item in firstname]
-    firstname_str = [item.encode('iso-8859-1','xmlcharrefreplace') for item in firstname]
+    firstname_nondiacritics = []
     for item in firstname:
-        print ('iii : item type %s : %s ' % (item, type(item)))
+        #print ('iii : item row  %s ' % (item))
+        item_nondiacritics = unidecode(item)
+        #print ('iii : item non diacritics  %s ' % (item_nondiacritics))
+        firstname_nondiacritics.append(item_nondiacritics)
 
-    # remove white space before and after
-    firstname_id = map(str.strip,firstname_str)
-    #print ('1 sans blan debut fin %s' % firstname_id)
-    #
+    #for item in firstname_nondiacritics:
+    #    print ('iii : item type %s : %s ' % (item, type(item)))
+
+    sl = firstname_nondiacritics
+
+    # remove white space
+    firstname_noblanks =[]
+    for item in sl:
+        #print ('iii : item avant sup blancs %s ' % (item))
+        item_noblanks = re.sub(u' ',u'',item)
+        #print ('iii : item apres sup blancs %s ' % (item_noblanks))
+        firstname_noblanks.append(item_noblanks)
+
+    sl = firstname_noblanks
+
+
     # lower
-    firstname_id = map(str.lower,firstname_id)
-    #print ('2 upper %s' % firstname_id)
-    #
-    # remove white space inside
-    firstname_id_no_spaces = [x.replace(' ', '') for x in firstname_id]
-    #print ('3 sans blanc milieu %s' % firstname_id_no_spaces)
-    firstname_id = firstname_id_no_spaces
+    firstname_lower = []
+    for item in sl:
+        #print ('iii : item avant lower %s ' % (item))
+        item_lower = item.lower()
+        #print ('iii : item apres lower %s ' % (item_lower))
+        firstname_lower.append(item_lower)
+
+    sl = firstname_lower
+    firstname_id = sl
     #
     # remove punctuation