source: trunk/src/traite_amsuab.sh @ 537

Last change on this file since 537 was 537, checked in by pinsard, 12 years ago

improvement for batch and idl profiler
  • Property svn:eol-style set to native
  • Property svn:executable set to *
  • Property svn:keywords set to Id URL
File size: 17.0 KB
Line 
1#! /bin/sh
2#+
3#
4# .. program:: traite_amsuab.sh
5#
6# .. _traite_amsuab.sh:
7#
8# ================
9# traite_amsuab.sh
10# ================
11#
12# SYNOPSIS
13# ========
14#
15# ::
16#
17#  $ traite_amsuab.sh -b yyyymmdd -e yyyymmdd -lonmin lonmin -lonmax lonmax -latmin latmin -latmax latmax -numch numch
18#
19# DESCRIPTION
20# ===========
21#
22# +todo+
23#
24# .. option:: -b beginning date <yyyymmdd>
25# .. option:: -e end date <yyyymmdd>
26# .. option:: -lonmin <lonmin>
27# .. option:: -lonmax <lonmax>
28# .. option:: -latmin <latmin>
29# .. option:: -latmax <latmax>
30# .. option:: -numch <channel>
31#
32# ``traite_amsuab.sh`` read :file:`${PROJECT_ID}/AMSU/AMSU*/*/*.L1C``
33# and :file:`${PROJECT}/SRC/dataref_amsu/COR*.dat`
34#
35# Some IDL programs are launched here using ${IDL_CMD} command
36# (see :ref:`varamma_profile.sh` to know how to set this environnement variable).
37#
38# +todo+
39#
40# Log file is written on
41# :file:`${PROJECT_LOG}/traite_amsuab.log.{YYYYMMDDTHHMMSSZ}`
42#
43# .. only:: man
44#
45#    Figure is visible on PDF and HTML documents only.
46#
47# .. only:: html or latex
48#
49#     .. graphviz::
50#
51#        digraph traite_amsuab {
52#
53#           amsu_t1 [shape=ellipse,fontname=Courier,label="${PROJECT_ID}/AMSU/AMSU*/L1C/yyyy/yyyy_mm_dd/*.L1C"];
54#           amsu_a_cor_land [shape=diamond,fontname=Courier,label="${PROJECT}/src/dataref_amsu/CORR_LAND_AMSUA5_DataFromJune2010_40deg.DAT"];
55#           amsu_a_cor_sea [shape=diamond,fontname=Courier,label="${PROJECT}/src/dataref_amsu/CORR_SEA_AMSUA5_DataFromJune2010_40deg.DA"];
56#           amsu_t2 [shape=ellipse,fontname=Courier,label="${PROJECT_ID}/yyyy/mm/amsua_amsub_yyyymmdd_geomin_geomax.dat"];
57#           amsu_t3 [shape=ellipse,fontname=Courier,label="${PROJECT_ID}/yyyy/mm/amsua_amsub_yyyymmdd_geomin_geomax_nadir.dat"];
58#           amsu_t4 [shape=ellipse,fontname=Courier,label="${PROJECT_ID}/yyyy/mm/amsua_amsub_yyyymmdd_geomin_geomax_nadir.nc"];
59#           mask [shape=ellipse,fontname=Courier,label="${PROJECT_ID}/MASK/ETOPO1_Ice_g_gmt4.nc"];
60#
61#           log [shape=ellipse,fontname=Courier,label="${PROJECT_LOG}/traite_amsuab.log{YYYYMMDDTHHMMSSZ}"];
62#
63#           traite_amsuab [shape=box,
64#           fontname=Courier,
65#           color=blue,
66#           URL="http://forge.ipsl.jussieu.fr/varamma/browser/trunk/src/traite_amsuab.sh",
67#           label="${PROJECT}/src/traite_amsuab.sh"];
68#
69#           {amsu_t1 amsu_a_cor_land amsu_a_cor_sea mask} -> {traite_amsuab} -> {log amsu_t2 amsu_t3 amsu_t4}
70#
71#         }
72#
73# EXAMPLES
74# ========
75#
76# You already used :ref:`get_amsu.sh` to get AMSU data including channel a5
77# during [20060801, 20060802[ and you want to produce NetCDF files::
78#
79#   traite_amsuab.sh -b 20060801 -e 20060802 -lonmin -25 -lonmax 25 -latmin -5 -latmax 20 -numch a5
80#
81# And look at log file with ::
82#
83#   tlogd.sh traite_amsuab
84#
85# A more realistic example:
86#
87#   traite_amsuab.sh -b 20060101 -e 20060630 -lonmin -32 -lonmax 32 -latmin -2 -latmax 42
88#
89# and of course on files in ${PROJECT_ID}.
90#
91# If you want to work on channel b5 of AMSU::
92#
93#   traite_amsuab.sh -b 20060813 -e 20060814 -lonmin -25 -lonmax 25 -latmin -5 -latmax 20 -numch b5
94#
95# SEE ALSO
96# ========
97#
98# IDL_
99#
100# .. _IDL: http://www.ittvis.com/ProductServices/IDL.aspx
101#
102# :ref:`varamma_profile.sh`
103# :ref:`project_init.pro`
104#
105# :ref:`run_traite_amsuab.sh` to submit job
106#
107# Previous step : :ref:`get_amsu.sh`
108#
109# Use :
110# :ref:`extract_amsua.pro`
111# :ref:`amsu2ncdf.pro`
112# :func:`search_amsufiles`
113#
114# :ref:`geolocation_to_string_sh.sh`
115#
116# .. todo::
117#
118# TODO
119# ====
120#
121# usage of profile on parameter true only
122#
123# get all procedure in profiler report (pb of sequence between
124# compile statements and call to profiler ?)
125#
126# se bloque sur
127# /homedata/pinsard/varamma_d/AMSU/AMSUBN17/L1C/2006/2006_08_01/NSS.AMBX.NM.D06213.S1029.E1224.B2132223.WI.L1C
128# avec use_amsua 0
129#
130# why need .compile ?!
131#
132# build/complete catalog
133#
134# adjust yyyymmddb_min and yyyymmdde_max
135#
136# idl status = 0 even if not ok
137#
138# handle -f option
139#
140# test if demo mode idl (or force cancel if demo)
141#
142# what about
143# lire_amsuab_new.pro:if (ambfile eq '/bdd/AMSU-1C/AMSUBN15/L1C/2006/2006_03_14/NSS.AMBX.NK.D06073.S1701.E1846.B4072728.GC.L1C') then goto,labett ;exception fichier erronne
144# lire_amsuab_new.pro:if (ambfile eq '/bdd/AMSU-1C/AMSUBN15/L1C/2004/2004_06_25/NSS.AMBX.NK.D04177.S1824.E1950.B3180102.WI.L1C') then goto,labett ;exception fichier erronne
145#
146# EVOLUTIONS
147# ==========
148#
149# $Id$
150#
151# $URL$
152#
153# - fplod 20120420
154#
155#   * add some compile to complete profiler report
156#   * add a realistic example from laurence
157#
158# - fplod 20120419
159#
160#   * add test on files_list
161#   * new technique to validate yyyymmdd
162#     because ::
163#        yyyymmddb=200601010
164#        ${date_cmd} -d "${yyyymmddb}"
165#        Sun Oct 10 00:00:00 CET 20060
166#
167#     alors qu'on avait pu supposer que cela renvoie "date invalid" (année sur 5 digits !)
168#
169# - fplod 20120417
170#
171#   * add some Profiler usage thanks to
172#     http://www.idlcoyote.com/code_tips/whyslow.html
173#
174# - pinsard 20120416
175#
176#   * update min max yyyymmdd
177#
178# - pinsard 20120323
179#
180#   * replace creation of :file:`${PROJECT_ID}/list_file` by a call to
181#     :func:`search_amsufiles`
182#   * transform the IDL snippet to a function to be abble to use ``CASE``
183#
184# - pinsard 20120106
185#
186#   * update header (no more use_amsu, correction files under src/dataref_amsu)
187#   * remplace VARAMMA by PROJECT
188#   * test if at least one file to be read
189#   * ajout pour connaitre la durée du traitement de chaque jour
190#   * add ref to :ref:`run_traite_amsuab.sh`
191#
192# - pinsard 2011-08-22T14:32:21Z loholt1.ipsl.polytechnique.fr (Linux)
193#
194#   * usage of ${IDL_CMD} instead of idl to allow run on idl 6.4 (/opt/idl-6.4/idl/bin/idl on climserv)
195#
196# - fplod 20110818T154338Z cratos (Linux)
197#
198#   * test on cratos after copying files from climserv
199#
200#     as get_amsu.sh do not yet work on cratos copies have been done using
201#     this kind of sequence
202#     ::
203#
204#      cratos$ mkdir -p ${PROJECT_ID}/AMSU/AMSUBN17/L1C/2006/
205#      loholt1$ scp -r /bdd/AMSU-1C/AMSUBN17/L1C/2006/2006_08_01 fplod@cerbere.locean-ipsl.upmc.fr:/usr/work/incas/fplod/varamma_d/AMSU/AMSUBN17/L1C/2006/
206#
207# - pinsard 2011-08-18T10:23:42Z loholt1.ipsl.polytechnique.fr (Linux)
208#
209#   * reach end with amsu-b only on loholt1
210#
211# - pinsard 2011-08-18T07:37:43Z loholt1.ipsl.polytechnique.fr (Linux)
212#
213#   * use geolocation_to_string_sh to build geobox
214#
215# - pinsard 2011-08-10T15:36:50Z loholt1.ipsl.polytechnique.fr (Linux)
216#
217#   * update amsu2ncdf call
218#
219# - fplod 20110810T081339Z aedon.locean-ipsl.upmc.fr (Darwin)
220#
221#   * correct_nadir_LE06 has been replaced by correct_nadir_amsu.
222#   * correction of graph
223#
224# - lelod/fplod 20110809
225#
226#   * new idl tools : need a list of file and new parameters
227#
228# - fplod 20110531T085708Z cratos.locean-ipsl.upmc.fr (Linux)
229#
230#   * remove "_" in parameter variables
231#
232# - pinsard 2011-05-30T14:32:02Z loholt1.ipsl.polytechnique.fr (Linux)
233#
234#   * produce daily files
235#   * complete graphic
236#
237# - pinsard 2011-05-26T10:57:48Z loholt1.ipsl.polytechnique.fr
238#
239#   * add lonmin,lonmax,latmin,latmax parameters
240#   * create a temporary file for idl command
241#   * amsu2ncdf should work now on one day
242#
243# - pinsard 2011-05-26T10:37:42Z loholt1.ipsl.polytechnique.fr
244#
245#   * correct_nadir_LE06 work now on one day
246#   * add mask in doc
247#
248# - pinsard 2011-05-25T13:16:37Z loholt1.ipsl.polytechnique.fr (Linux)
249#
250#   * get rid of yr and jm
251#
252# - fplod 20110511T102251Z cratos.locean-ipsl.upmc.fr (Linux)
253#
254#   * add -a parameter to handle use_amsua and use_amsub parameters of IDL
255#     program
256#
257# - pinsard 2011-05-05T07:11:18Z loholt1.ipsl.polytechnique.fr (Linux)
258#
259#   * consolidation
260#     nb : tricky string to integer pb. see http://www.mail-archive.com/bug-bash@gnu.org/msg06358.html
261#
262# - fplod 20110426T090939Z aedon.locean-ipsl.upmc.fr (Darwin)
263#
264#   * usage of PROJECT*
265#
266# - fplod 20100608T143812Z aedon.locean-ipsl.upmc.fr (Darwin)
267#
268#   * minimal header
269#-
270system=$(uname)
271case "${system}" in
272   AIX|IRIX64)
273     echo "${command} : www : no specific posix checking"
274     date_cmd=date
275   ;;
276   Darwin)
277     set -o posix
278     date_cmd=gdate
279   ;;
280   Linux)
281     set -o posix
282     date_cmd=date
283   ;;
284   *)
285    set -o posix
286   ;;
287esac
288unset system
289#
290set -u
291#
292LANG=POSIX
293#
294command=$(basename ${0})
295log_date=$(date -u +"%Y%m%dT%H%M%SZ")
296#
297usage=" Usage : ${command} [-f] -b yyyymmdd -e yyyymmdd -lonmin lonmin -lonmax lonmax -latmin latmin -latmax latmax -numch numch"
298#
299hostname=$(hostname)
300#
301yyyymmddb_min=20000101
302yyyymmdde_max=20121231
303#
304# default
305debug=0
306force=0
307yyyymmddb=20060420
308yyyymmdde=20060420
309numch='a5'
310jdeb=1
311jfin=10
312netcdf_build=0
313#
314minargcount=4
315if [ ${#} -lt ${minargcount} ]
316then
317   echo "${command} : eee : not enought arguments"
318   echo "${usage}"
319   exit 1
320fi
321unset minargcount
322#
323while [ ${#} -gt 0 ]
324do
325   case ${1} in
326     -b)
327        # first date to get
328        yyyymmddb=${2}
329        shift
330     ;;
331     -e)
332        # last date to get
333        yyyymmdde=${2}
334        shift
335     ;;
336     -lonmin)
337        lonmin=${2}
338        shift
339     ;;
340     -lonmax)
341        lonmax=${2}
342        shift
343     ;;
344     -latmin)
345        latmin=${2}
346        shift
347     ;;
348     -latmax)
349        latmax=${2}
350        shift
351     ;;
352     -numch)
353        numch=${2}
354        shift
355     ;;
356     -f)
357        force=1
358     ;;
359     -h)
360        echo "${usage}"
361        exit 0
362     ;;
363     *)
364        # other choice
365        echo "${command} : eee : unknown option ${1}"
366        echo "${usage}"
367        exit 1
368     ;;
369   esac
370   # next flag
371   shift
372done
373unset usage
374#
375# check parameters
376# and define associated variables
377${date_cmd} -d "${yyyymmddb}" > /dev/null
378status_date=${?}
379if [ ${status_date} -ne 0 ]
380then
381    echo "${command} : eee : yyyymmddb ${yyyymmddb} argument invalid"
382    exit 1
383fi
384unset status_date
385#
386d=$((${yyyymmddb} - ${yyyymmddb_min}))
387if [ ${d} -lt 0 ] 
388then
389    echo "${command} : eee : yyyymmddb ${yyyymmddb} must be equal or greater than yyyymmddb_min ${yyyymmddb_min}"
390    exit 1
391fi
392#
393d=$((${yyyymmddb} - ${yyyymmdde_max}))
394if [ ${d} -gt 0 ] 
395then
396    echo "${command} : eee : yyyymmddb ${yyyymmddb} must be equal or lower than yyyymmdde_max ${yyyymmdde_max}"
397    exit 1
398fi
399#
400${date_cmd} -d "${yyyymmdde}" > /dev/null
401status_date=${?}
402if [ ${status_date} -ne 0 ]
403then
404    echo "${command} : eee : yyyymmdde ${yyyymmdde} argument invalid"
405    exit 1
406fi
407unset status_date
408#
409d=$((${yyyymmdde} - ${yyyymmdde_max}))
410if [ ${d} -gt 0 ] 
411then
412    echo "${command} : eee : yyyymmdde ${yyyymmdde} must be lower or equal to yyyymmdde_max ${yyyymmdde_max}"
413    exit 1
414fi
415#
416if [[ "${yyyymmdde}" < "${yyyymmddb}" ]]
417then
418    echo "${command} : eee : yyyymmdde ${yyyymmdde} must be greater than yyyymmddb ${yyyymmddb}"
419    exit 1
420fi
421#
422if [[ "${yyyymmdde}" = "${yyyymmddb}" ]]
423then
424     echo "${command} : eee : yyyymmdde ${yyyymmdde} must be greater than yyyymmddb ${yyyymmddb}"
425     exit 1
426fi
427#
428tool=${IDL_CMD}
429type ${tool} 1> /dev/null 2>&1
430status=${?}
431if [ ${status} -ne 0 ]
432then
433   echo "${command} : eee : tool ${IDL_CMD} not found"
434   echo "${command} : eee : check varamma_profile.sh sequence"
435   exit 1
436fi
437unset status
438unset tool
439#
440#
441tool=ncrcat
442type ${tool} 1> /dev/null 2>&1
443status=${?}
444if [ ${status} -ne 0 ]
445then
446   echo "${command} : eee : tool ${tool} not found"
447   exit 1
448fi
449unset status
450unset tool
451#
452# check for write permission on PROJECT_LOG
453if [ ! -w ${PROJECT_LOG} ]
454then
455   echo "${command} : eee : ${PROJECT_LOG} not writable"
456   exit 1
457fi
458#
459log=${PROJECT_LOG}/$(basename ${0} .sh).log.${log_date}
460echo "[Context]" 1>> ${log}
461echo "command=$(basename ${0})" 1>>${log}
462echo "hostname=${hostname}" 1>> ${log}
463echo "runtime=${log_date}" 1>> ${log}
464echo "log=${log}" 1>> ${log}
465echo "IDL_CMD=${IDL_CMD}" 1>> ${log}
466unset log_date
467echo "" 1>> ${log}
468echo "[Parameters]" 1>> ${log}
469echo "yyyymmddb=${yyyymmddb}" 1>> ${log}
470echo "yyyymmdde=${yyyymmdde}" 1>> ${log}
471echo "lonmin=${lonmin}" 1>> ${log}
472echo "lonmax=${lonmax}" 1>> ${log}
473echo "latmin=${latmin}" 1>> ${log}
474echo "latmax=${latmax}" 1>> ${log}
475echo "numch=${numch}"  1>> ${log}
476echo "" 1>> ${log}
477#
478# build geomax from parameters
479geomin=$(geolocation_to_string_sh.sh -lon ${lonmin} -lat ${latmin})
480status=${?}
481if [ ${status} -ne 0 ]
482then
483   echo "${command} : eee : pb geomin with geolocation_to_string_sh.sh" 1>> ${log}
484   exit 1
485fi
486geomax=$(geolocation_to_string_sh.sh -lon ${lonmax} -lat ${latmax})
487status=${?}
488if [ ${status} -ne 0 ]
489then
490   echo "${command} : eee : pb geomax with geolocation_to_string_sh.sh" 1>> ${log}
491   exit 1
492fi
493#
494# loop over all dates
495# set current and end date
496current=$(${date_cmd} -u -d "${yyyymmddb}" +%s | awk '{printf "%10.10d",$1}')
497end=$(${date_cmd} -u -d "${yyyymmdde}" +%s | awk '{printf "%10.10d",$1}')
498delta=86400
499while [[ "${current}" < "${end}" ]]
500do
501    # get current time at the beginning of the process for one day
502    timebd="$(date +%s)"
503    echo "${command} : iii : looking for files of $(${date_cmd} -u -d "1970-01-01 ${current} sec" +%Y%m%dT%H%M)" 1>> ${log}
504    yyyymmdd=$(${date_cmd} -u -d "1970-01-01 ${current} sec" +%Y%m%d)
505    yyyy=$(${date_cmd} -u -d "1970-01-01 ${current} sec" +%Y)
506    mm=$(${date_cmd} -u -d "1970-01-01 ${current} sec" +%m)
507    dd=$(${date_cmd} -u -d "1970-01-01 ${current} sec" +%d)
508    hh=$(${date_cmd} -u -d "1970-01-01 ${current} sec" +%H)
509    mn=$(${date_cmd} -u -d "1970-01-01 ${current} sec" +%M)
510    #
511    prefix=$(printf  "%1c" ${numch} | tr [:lower:] [:upper:])
512
513    cat >> ${PROJECT}/src/traite_amsu_${$}.pro << EOF
514function traite_amsu_${$}
515;
516compile_opt idl2, strictarrsubs
517;
518@cm_project
519@common
520;
521result = -1
522;
523result = search_amsufiles('${numch}', ${yyyy},${mm}, ${dd})
524result_type=size(result,/type)
525IF (result_type EQ 3) THEN BEGIN
526   IF (result EQ -1) THEN BEGIN
527       print, 'eee : pb search_amsufiles'
528       return, result
529   ENDIF
530ENDIF ELSE BEGIN
531   files_list = result
532ENDELSE
533;
534result = file_bathy_to_mem(${lonmin}, ${lonmax}, ${latmin}, ${latmax})
535IF (result EQ -1) THEN BEGIN
536   print, 'eee : pb file_bathy_to_mem'
537   return, result
538ENDIF
539;
540; uncomment the following lines to test with the last file of the list
541; files_list_new_array = strarr(1)
542; files_list_new_array[0] = files_list[N_ELEMENTS(files_list) - 1]
543; files_list = files_list_new_array
544;
545; uncomment the following lines to test with the two last files of the list
546; files_list_new_array = strarr(2)
547; files_list_new_array[0] = files_list[N_ELEMENTS(files_list) - 2]
548; files_list_new_array[1] = files_list[N_ELEMENTS(files_list) - 1]
549; files_list = files_list_new_array
550;
551; uncomment the following lines to test with the first file of the list
552; files_list_new_array = strarr(1)
553; files_list_new_array[0] = files_list[1]
554; files_list = files_list_new_array
555;
556CASE size(files_list,/DIMENSION) OF
557   0L : BEGIN
558            print, 'www : no file found'
559        END
560   ELSE: BEGIN
561            print, 'iii : nb de fichiers', size(files_list,/DIMENSION)
562            resol=1
563            extract_amsua, '${numch}', files_list, ${yyyy}, ${mm}, ${dd}, resol, ${lonmin}, ${lonmax}, ${latmin}, ${latmax}
564        END
565ENDCASE
566EOF
567    if [ ${netcdf_build} -eq 1 ]
568    then
569        cat >> ${PROJECT}/src/traite_amsu_${$}.pro << EOF
570amsu2ncdf, ${use_amsua}, ${yyyy},${mm},${dd}, ${lonmin}, ${lonmax}, ${latmin}, ${latmax}
571EOF
572     fi
573     cat >> ${PROJECT}/src/traite_amsu_${$}.pro << EOF
574return, result
575end
576EOF
577    ${IDL_CMD} << EOF >> ${log} 2>&1
578key_performance = 0
579; totally shut down all profiling
580profiler,/CLEAR,/SYSTEM & profiler,/CLEAR,/RESET
581; add system procedure
582Profiler, /SYSTEM
583; compile users procedures to be scan by Profiler
584.compile interpol_correc
585.compile extract_amsua
586.compile read_amsua1c
587.compile define_amsua_header_struct
588.compile define_amsua_struct
589.compile pixelsize
590.compile mem_to_file_amsu_t2
591.compile geolocation_to_string_idl
592.compile file_bathy_to_mem
593; add users procedures
594Profiler
595result = traite_amsu_${$}()
596print, 'ppp : profiler report begin'
597Profiler, /REPORT
598print, 'ppp : profiler report end'
599EOF
600    status=${?}
601    if [ ${status} -ne 0 ]
602    then
603       echo "${command} : eee: ${IDL_CMD} failed : ${status}" >> ${log} 2>&1
604       cat ${PROJECT}/src/traite_amsu_${$}.pro >> ${log} 2>&1
605       exit 1
606    fi
607    rm ${PROJECT}/src/traite_amsu_${$}.pro
608    unset status
609    if [ ${netcdf_build} -eq 1 ]
610    then
611         # concatenate yyyymmdd-hhmnss NetCDF files
612         list=${PROJECT_ID}/AMSU/${yyyy}/${mm}/${prefix}${yyyy}${mm}${dd}_??????_${geomin}_${geomax}_nadir.nc
613         fileout=${PROJECT_ID}/AMSU/${yyyy}/${mm}/${prefix}${yyyy}${mm}${dd}_${geomin}_${geomax}_nadir.nc
614         ncrcat -O ${list} ${fileout} >> ${log} 2>&1
615         status=${?}
616         if [ ${status} -ne 0 ]
617         then
618             echo "${command} : eee : pb with ncrcat" 1>> ${log}
619             for file in ${list}
620             do
621                 ncdump -h ${file} >> ${log} 2>&1
622             done
623             exit 1
624          else
625             echo "${command} : iii : adding ${fileout}" >> ${log} 2>&1
626          fi
627    fi
628    unset dd
629    # get current time at the end of the process for one day
630    timeed="$(date +%s)"
631    elapsed_seconds="$(( ${timeed} - ${timebd} ))"
632    echo "${command} : iii : Elapsed time for this day (s) : ${elapsed_seconds}" >> ${log} 2>&1
633    unset timebd
634    unset timeed
635    # next time step
636    current=$((${current} + ${delta}))
637done
638#
639unset yyyy
640unset mm
641#
642unset command
643unset log
644unset hostname
645unset usage
646#
647unset date_cmd
648#
649# end
650#++set
651exit
Note: See TracBrowser for help on using the repository browser.