Changeset 182 for trunk/aeres/scripts

Timestamp:

04/09/12 19:21:48 (12 years ago)

Author:

pinsard

Message:

try to progress in author list

Location:

trunk/aeres/scripts

Files:

• aeresrh.py (modified) (12 diffs)
• build_firstname_id.py (copied) (copied from trunk/aeres/scripts/surname_id.py) (1 diff)
• build_surname_id.py (moved) (moved from trunk/aeres/scripts/surname_id.py) (2 diffs)

trunk/aeres/scripts/aeresrh.py

@@ -51 +51 @@
 ::
 
- aeresrh.py -ifile ${PROJECT}/data/aeresrh.xls
+ aeresrh.py --ifile ${PROJECT}/data/aeresrh.xls
 
 TODO
@@ -60 +60 @@
 ne marche pas avec des chemins relatifs
 
-apprendre a faire un slab pour supprimer header/title/footer
-
 append sur ifiles car liste permanent et liste thesard
 
@@ -70 +68 @@
 missing values
 
-remove hard coded number of columns for surname, firstname, ...
-
 coding rules (pylint)
-
 
 EVOLUTIONS
@@ -81 +76 @@
 
 $URL$
+
+- fplod 20120409
+
+  * no more hard coded column number
 
 - fplod 20120406
@@ -107 +106 @@
     print("Failed to import xlrd from any known place")
     sys.exit(1)
+
+from build_surname_id import build_surname_id
+from build_firstname_id import build_firstname_id
 
 def get_option_parser ():
@@ -130 +132 @@
     return parser
 
+def find_in_dict(d, criterion, lookup):
+    """
+    >>> people = {'pk1':{'firstname':'Brian', 'age':42}, 'pk2':{'firstname':'Alex', 'age':50}}
+    >>> find_in_dict(people, 'age', 42)
+
+    http://stackoverflow.com/questions/8987631/searching-2-dimensional-dictionary
+    """
+
+    result = []
+    for key, val in d.iteritems():
+        if criterion in val and val[criterion] == lookup:
+            result.append((key,val))
+    return result
+
 def readrh(ifile, is_verbose):
 
@@ -160 +176 @@
             print('values of in col %d : %s' % (colnum, sh.col_values(colnum)))
 
-    offsetrow = 12
-
-    colsurname = 1
-    colfirstname = 2
-    colarrival = 14
-    colpubli = 17
+    # detect title row
+    for rownum in range(sh.nrows):
+        if sh.row_values(rownum)[0] == 't':
+            titlerow = rownum 
+    if is_verbose == True:
+            print('titlerow : %d ' % (titlerow))
+
+    # deduce first usefull row
+    firstrow = titlerow + 1
+    #
+    # detect last usefull row
+    uselessrow = []
+    for rownum in range(titlerow,sh.nrows):
+        if sh.row_values(rownum)[0] == '': 
+            uselessrow.append(rownum)
+    lastrow = uselessrow[0]
+    if is_verbose == True:
+            print('lastrow : %d ' % (lastrow))
+
+    # detect surname column
+    for colnum in range(sh.ncols):
+        if sh.col_values(colnum)[titlerow] == 'nom':
+            colsurname = colnum
+    if is_verbose == True:
+            print('colsurname: %d ' % (colsurname))
+
+    # detect firstname column
+    for colnum in range(sh.ncols):
+        if sh.col_values(colnum)[titlerow] == 'prenom':
+            colfirstname = colnum
+    if is_verbose == True:
+            print('colfirstname: %d ' % (colfirstname))
+
+    # detect arrival column
+    for colnum in range(sh.ncols):
+        if sh.col_values(colnum)[titlerow] == u'arrivée':
+            colarrival = colnum
+    if is_verbose == True:
+            print('colarrival: %d ' % (colarrival))
+
+    # detect publi column
+    for colnum in range(sh.ncols):
+        if sh.col_values(colnum)[titlerow] == u'publi':
+            colpubli = colnum
+    if is_verbose == True:
+            print('colpubli: %d ' % (colpubli))
 
     datemode = 0 
 
-    surname = sh.col_values(colsurname)
-    # remove elements because it is the title/header/footer
-    surname[0:offsetrow] = 'remove'
-    surname[-1] = 'remove'
-    surname = filter(lambda surname: surname != 'remove', surname)
+    surname = sh.col_values(colsurname)[firstrow:lastrow]
+    print ('surname 0000 %s ' % surname)
 
     if is_verbose == True:
@@ -179 +232 @@
        print(' surname %s' % surname)
 
-    firstname = sh.col_values(colfirstname)
-    # remove elements because it is the title/header/footer
-    firstname[0:offsetrow] = 'remove'
-    firstname[-1] = 'remove'
-    firstname = filter(lambda firstname: firstname != 'remove', firstname)
+    firstname = sh.col_values(colfirstname)[firstrow:lastrow]
 
     if is_verbose == True:
@@ -189 +238 @@
        print(' firstname %s' % firstname)
 
-    temp = sh.col_values(colarrival)
-    # remove elements because it is the title/header/footer
-    temp[0:offsetrow] = 'remove'
-    temp[-1] = 'remove'
-    temp = filter(lambda temp: temp != 'remove', temp)
-    # transform to datetime
+    temp = sh.col_values(colarrival)[firstrow:lastrow]
     arrival = []
     for rownum in range(len(temp)):
@@ -206 +250 @@
             #arrival.append(datetime.datetime(*xlrd.xldate_as_tuple(temp[rownum],datemode)))
 
-
     if is_verbose == True:
        print(' nb arrival %d' % len(arrival))
        print(' arrival %s' % arrival)
 
-    publi = sh.col_values(colpubli)
-    # remove elements because it is the title/header/footer
-    publi[0:offsetrow] = 'remove'
-    publi[-1] = 'remove'
-    publi = filter(lambda publi: publi != 'remove', publi)
+    publi = sh.col_values(colpubli)[firstrow:lastrow]
 
     if is_verbose == True:
@@ -225 +264 @@
 def aeresrh():
     """main
-        """
+    """
     try:
         parser = get_option_parser ()
@@ -238 +277 @@
 
     surname, firstname, arrival, publi = readrh(ifile, is_verbose)
-
-    for i, surname, firstname in enumerate(surname, firstname):
-        print (' %s_%s' % (surname[i], firstname[i]))
+    #
+    # build surname id
+    surname_id = build_surname_id(surname)
+    firstname_id = build_firstname_id(firstname)
+    #
+    # populate dictionary
+    #++people = {}
+    #++for index, onesurname in enumarate(surname):
+    #++   print (' surname %s' % (surname))
+    #++   people[n] = surname[index]
+
+    find_in_dict(people, 'surname', 'Pinsard')
 
 # Run main, if called from the command line

trunk/aeres/scripts/build_firstname_id.py

@@ +1 @@
+#!/usr/bin/env python
+# -*- coding: iso-8859-1 -*-
+
 """
-    generation de la partie "prenom" de l'id
+
+=====================
+build_firstname_id.py
+=====================
+
+DESCRIPTION
+===========
+
+generation de la partie "prénom" de l'id
+
 
 SEE ALSO
 ========
 
-:ref:`surname_id.xsl`
+:ref:`firstname_id.xsl`
+
+EXAMPLES
+========
+
+cf. doctest
+
+::
+
+    python -v build_firstname_id
 
 TODO
 ====
 
-still draft
+EVOLUTIONS
+==========
 
-flat ascii
+$Id$
 
-enlever les punctuations
-enlever les blancs debut et fin
-enlever le blancs milieu
+$URL$
+
+- fplod 20120409
+
+  * creation
 
 """
-def surname_id(surname):
+
+import string
+import sys
+
+def build_firstname_id(firstname):
 
 
-    surname_id = surname.upper()
+    """
+    >>> firstname = []
+    []
+    >>> firstname.append(' Ginette')
+    [' Ginette']
+    >>> firstname.append('Gin ette ')
+    [' Ginette', 'Gin  ette']
+    >>> firstname.append("G\'in ette ")
+    [' Ginette', 'Gin ette', "G\'in ette"]
+    >>> firstname_id = build_firstname_id(firstname)
+    ['ginette','ginette','ginette']
+    """
 
-    return surname_id
+    if len(firstname) == 0:
+        print('eee : firstname empty')
+        sys.exit(-1)
 
+    # convert to str
+    #++firstname_str = [str(item) for item in firstname]
+    #++firstname_str = [item.encode('iso-8859-1','replace') for item in firstname]
+    firstname_str = [item.encode('iso-8859-1','xmlcharrefreplace') for item in firstname]
+    for item in firstname:
+        print ('iii : item type %s : %s ' % (item, type(item)))
+
+    # remove white space before and after
+    firstname_id = map(str.strip,firstname_str)
+    #print ('1 sans blan debut fin %s' % firstname_id)
+    #
+    # lower
+    firstname_id = map(str.lower,firstname_id)
+    #print ('2 upper %s' % firstname_id)
+    #
+    # remove white space inside
+    firstname_id_no_spaces = [x.replace(' ', '') for x in firstname_id]
+    #print ('3 sans blanc milieu %s' % firstname_id_no_spaces)
+    firstname_id = firstname_id_no_spaces
+    #
+    # remove punctuation
+    for c in string.punctuation:
+        firstname_id_no_punctuation = [x.replace(c, '') for x in firstname_id]
+        #print ('4 sans %s avec replace string.punctuation %s' % (c, firstname_id_no_punctuation))
+        firstname_id = firstname_id_no_punctuation
+
+    return firstname_id
+
+if __name__ == "__main__":
+    import doctest
+    doctest.testmod()

trunk/aeres/scripts/build_surname_id.py

@@ +1 @@
+#!/usr/bin/env python
+# -*- coding: iso-8859-1 -*-
+
 """
-    generation de la partie "prenom" de l'id
+
+===================
+build_surname_id.py
+===================
+
+DESCRIPTION
+===========
+
+generation de la partie "nom" de l'id
+
 
 SEE ALSO
@@ -7 +19 @@
 :ref:`surname_id.xsl`
 
+EXAMPLES
+========
+
+cf. doctest
+
+::
+
+    python -v build_surname_id
+
 TODO
 ====
 
-still draft
+EVOLUTIONS
+==========
 
-flat ascii
+$Id$
 
-enlever les punctuations
-enlever les blancs debut et fin
-enlever le blancs milieu
+$URL$
+
+- fplod 20120409
+
+  * work with fake data
+
+- fplod 20120407
+
+  * creation
 
 """
-def surname_id(surname):
+
+import string
+import sys
+
+def build_surname_id(surname):
 
 
-    surname_id = surname.upper()
+    """
+    >>> surname = []
+    []
+    >>> surname.append(' Tartempillion ')
+    [' Tartempillion ']
+    >>> surname.append('Tar tempillion ')
+    [' Tartempillion ', 'Tar tempillion ']
+    >>> surname.append("T\'ar tempillion ")
+    [' Tartempillion ', 'Tar tempillion ', "T\'ar tempillion "]
+    >>> surname_id = build_surname_id(surname)
+    ['TARTEMPILLION','TARTEMPILLION','TARTEMPILLION']
+    """
+
+    if len(surname) == 0:
+        print('eee : surname empty')
+        sys.exit(-1)
+
+    # convert to str
+    surname_str = [str(item) for item in surname]
+    for item in surname:
+        print ('iii : item type %s : %s ' % (item, type(item)))
+
+    # remove white space before and after
+    surname_id = map(str.strip,surname_str)
+    #print ('1 sans blan debut fin %s' % surname_id)
+    #
+    # upper
+    surname_id = map(str.upper,surname_id)
+    #print ('2 upper %s' % surname_id)
+    #
+    # remove white space inside
+    surname_id_no_spaces = [x.replace(' ', '') for x in surname_id]
+    #print ('3 sans blanc milieu %s' % surname_id_no_spaces)
+    surname_id = surname_id_no_spaces
+    #
+    # remove punctuation
+    for c in string.punctuation:
+        surname_id_no_punctuation = [x.replace(c, '') for x in surname_id]
+        #print ('4 sans %s avec replace string.punctuation %s' % (c, surname_id_no_punctuation))
+        surname_id = surname_id_no_punctuation
 
     return surname_id
 
+if __name__ == "__main__":
+    import doctest
+    doctest.testmod()