#!/bin/bash
#------------------------------------------------------
switchtotar=0
maxtries=3
#---------------------------------------------
while [ $# -ne 0 ]
do
        case $1 in
       --switchtotar)
		switchtotar=1
		shift 1 ;;
       --maxtries)
		maxtries=$2
		shift 2 ;;
       -h|--help|-help)
                echo "----------------------------------------------------------------------------"
		echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
		echo
		echo "Description:"
                echo "   Pass a ncrcat, tar or cp command on files from prepared sets of files."
                echo "   For ncrcat command 2 shots will be done at each try".
                echo  
                echo "   A status file and a log file will be created in output_ncrcat directory."
                echo "   Copy a set to output_tar directory for later processing after N tries."
		echo
		echo "Arguments:"
                echo "   dirin is the input directory where files to concatenate will be read"
                echo "   dirout is the output directory where concatenated files will be written"
		echo
                echo "   The following directory must exist where the command enlarge_my_files is run:"
		echo "       enlarge_my_files"
		echo "                       |-- output_ncrcat"
		echo "                       |   |-- set_000001.list"
		echo "                       |   |-- set_000002.list"
		echo "                       |   |-- set_000003.list"
		echo "                       |   |-- set_000004.list"
		echo "                       |   \`-- set_000005.list"
		echo "                       \`-- output_tar"
		echo
		echo "Options:"
                echo "   -h, --help, -help"
                echo "       Print this manual."
                echo "   --switchtotar"
                echo "       Abort ncrcat operation and switch to tar command"
                echo "   --maxtries N"
                echo "       Maximum number of tries before switching to tar command" 
                echo "       Default is 3"
		echo
		echo "----------------------------------------------------------------------------"
		exit ;;
        -*)
                `dirname $0`/enlarge_my_files.sh -h
                exit ;;
        *)
                break ;;
        esac
done

if [ $# -lt 2 ] ; then
        `dirname $0`/enlarge_my_files.sh -h
        exit
fi

if ( \
     [ ! -d $PATH_SIMU/output_ncrcat ] || \
     [ ! -d $PATH_SIMU/output_tar ]    || \
     [ ! -d $PATH_SIMU/restart_tar ]   || \
     [ ! -d $PATH_SIMU/store_cp ]      || \
     [ ! -d $PATH_SIMU/work_cp ]       || \
     [ ! -d $PATH_SIMU/debug_tar ]     || \
     [ ! -d $PATH_SIMU/other_tar ] \
   ) \
   && \
   [ ! -e $PATH_SIMU/tar_full_simul.list ]
then
        `dirname $0`/enlarge_my_files.sh -h
        exit
fi

#------------------------------------------------------
dirin=$1
dirout=$2
dirout_work=$3

#------------------------------------------------------
function find_varstoexclude {
nbfile=0
for file in `cat $1` ; do
 #   ncdump -h ${file} | gawk '{if (match($0, /(byte|char|short|int|float|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
    ncdump -h ${file} | grep -Ew "byte|char|short|int|float|double"  | awk -F\( '{split($1,a," ") ; split($2,b,")") ; if (a[2] != b[1]) print a[2]}'  >> tmp_$$.txt
    let nbfile=nbfile+1
done

varstoexclude=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d','`
varstoexcludefiltered=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' | paste -s -d','`
nbvars=`echo $varstoexcludefiltered | gawk -F',' '{print NF}'`

rm -f tmp_$$.txt

return 0 
}

# gpdebug : pour test showPackProgress.sh *************************
export RANDOM=$$

function gives_0_or_1
{
   bit=-1
   let "bit = RANDOM % 2"
   echo $bit
}

resultCmd=
# gpdebug : ****************************** fin ********************

source ${EXE_DIR}/DEM_utilities.sh # gpdebug : pour trouver les fonctions de mesure de temps

if ( \
     [ -d $PATH_SIMU/output_ncrcat ] || \
     [ -d $PATH_SIMU/output_tar ]    || \
     [ -d $PATH_SIMU/restart_tar ]   || \
     [ -d $PATH_SIMU/store_cp ]      || \
     [ -d $PATH_SIMU/work_cp ]       || \
     [ -d $PATH_SIMU/debug_tar ]     || \
     [ -d $PATH_SIMU/other_tar ] \
   )
then

    #------------------------------------------------------
    for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do

	    #-----------------------------------
	    echo "#-------------------"
	    echo "Set: $set"
            datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	    datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	    statusfile=${set%%.list}.status
	    logfile=${set%%.list}_${datestr}.log

	    #-----------------------------------
	    # Create status file if not exists
	    if [ ! -e $statusfile ] ; then
		    echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	    fi

	    #-----------------------------------
	    # Switch to tar command
	    if [ $switchtotar -eq 1 ] ; then
		    echo "====> DELEGATE to tar command with option --switchtotar"
		    echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
		    mv $set $PATH_SIMU/output_tar
		    continue
	    fi

	    #-----------------------------------
	    # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	    if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
		    printf "====> "
		    tail -1 $statusfile
		    continue
	    fi

	    #-----------------------------------
	    # nbtries = number of lines from status file (first comment line gives 1 for the first try)
	    nbtries=`wc -l $statusfile | cut -d' ' -f1`		
	    echo "Nb of tries: $nbtries"

	    #-----------------------------------
	    # Set output file name from list name

 	    fileout1=$( basename $set )
 	    fileout=${fileout1%.list}

	    filefirst=`head -n 1 $set`
	    diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`

	    outputfile=$diroutputfile/${fileout}
	    mkdir -p $diroutputfile
	    echo "Output file to write: $outputfile"

	    #-----------------------------------
	    # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
	    extratar=0
	    find_varstoexclude $set
	    if [ $nbvars -eq 0 ] ; then
		    varstoexclude="t_inst_.*"
	    else
		    # There are others variables than 't_inst_.*' so an extra tar file will be created
		    varstoexclude="t_inst_.*,$varstoexcludefiltered"
		    extratar=1
		    # Copy to output_tar directory 
		    echo "====> COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*"
		    echo "COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" >> $statusfile
		    cp $set $PATH_SIMU/output_tar
	    fi

	    #-----------------------------------
	    # Try 2 times before fire "FAILED"
	    ncrcatcmd=ncrcat
	    ncrcatoptions=""
	    if [ "x${ON_IDRIS}" == "xno" ]
	    then
	        ncrcatoptions="--md5_digest"	    
	    fi
    #	ncrcatoptions=""

	    startTime=$( getDateMilliSeconds ) # gpdebug : time
	    resCmd=0
	    cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
	    resCmd=$?
	    meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time

	    # resultCmd=$( gives_0_or_1 )
	    # resultCmd=1
	    # if [ $resultCmd -eq 0 ] ; then
	    if [ $resCmd -eq 0 ] ; then
	            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
		    datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		    echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
		    echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
		    echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
	    else
		    # Add --md5_digest
	            rm -f $outputfile*ncrcat.tmp
		    startTime=$( getDateMilliSeconds ) # gpdebug : time
		    resCmd=0
		    cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
		    resCmd=$?
		    meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
	            # resultCmd=$( gives_0_or_1 )
		    # resultCmd=1
	            # if [ $resultCmd -eq 0 ] ; then
		    if [ $resCmd -eq 0 ] ; then
		            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
			    datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
			    echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
			    echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
			    echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
		    else
		            rm -f $outputfile*ncrcat.tmp
			    datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
			    echo "====> FAILED at try #$nbtries" 
			    echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
			    if [ $nbtries -ge $maxtries ] ; then
				    echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
				    echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
			            cp $set $PATH_SIMU/output_tar # gpdebug : a retablir
			    fi
		    fi
	    fi

    done	

    #------------------------------------------------------

    for set in $( ls $PATH_SIMU/output_tar/*list ) ; do

	    #-----------------------------------
	    echo "#-------------------"
	    echo "Set: $set"
            datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	    datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	    statusfile=${set%%.list}.status
	    logfile=${set%%.list}_${datestr}.log

	    #-----------------------------------
	    # Create status file if not exists
	    if [ ! -e $statusfile ] ; then
		    echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	    fi

	    #-----------------------------------
	    # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	    if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
		    printf "====> "
		    tail -1 $statusfile
		    continue
	    fi

	    #-----------------------------------
	    # nbtries = number of lines from status file (first comment line gives 1 for the first try)
	    nbtries=`wc -l $statusfile | cut -d' ' -f1`		
	    # echo "Nb of tries: $nbtries"

	    #-----------------------------------
	    # Set output file name from list name

 	    fileout1=$( basename $set )
 	    fileout=${fileout1%.list}.tar

	    filefirst=`head -n 1 $set`
	    dirinputfile=`dirname $filefirst`
	    diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`	

	    outputfile=$diroutputfile/${fileout}
	    mkdir -p $diroutputfile
	    echo "Output file to write: $outputfile"

	    #-----------------------------------
	    # Try 1 time before fire "FAILED"
	    # Waiting for CCRT command
	    tarcmd=tar
	    taroptions="--format=posix -W -cf"
            #taroptions="--format=posix -cf"
	    cd $dirinputfile

	    set_local=$set".local"
	    for file in $( cat $set) ; do
		basename $file >> $set".local" 
	    done

    # 	#-----------------------------------
            startTime=$( getDateMilliSeconds ) # gpdebug : time
	    resCmd=0
	    $tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1 # gpdebug : a retablir
	    resCmd=$?
	    meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time

	    # resultCmd=$( gives_0_or_1 )
	    # if [ $resultCmd -eq 0 ] ; then
	    if [ $resCmd -eq 0 ] ; then
		sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
		echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
		echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	    else
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "====> FAILED at try #$nbtries" 
		echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile

	    fi
	    rm -f $set".local" 
	    #-----------------------------------

    done	


    #------------------------------------------------------

    for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do
	    #-----------------------------------
	    echo "#-------------------"
	    echo "Set: $set"
            datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	    datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	    statusfile=${set%%.list}.status
	    logfile=${set%%.list}_${datestr}.log

	    #-----------------------------------
	    # Create status file if not exists
	    if [ ! -e $statusfile ] ; then
		    echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	    fi

	    #-----------------------------------
	    # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	    if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
		    printf "====> "
		    tail -1 $statusfile
		    continue
	    fi

	    #-----------------------------------
	    # nbtries = number of lines from status file (first comment line gives 1 for the first try)
	    nbtries=`wc -l $statusfile | cut -d' ' -f1`		
	    # echo "Nb of tries: $nbtries"

	    #-----------------------------------
	    # Set output file name from list name
	    dirin1=$PATH_SIMU/RESTART
	    dirin2=${dirin1##${IGCM_DEM}}
	    dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
	    diroutputfile=${OUTPUT_STORE}${dirin3}
 	    fileout1=$( basename $set )
 	    fileout=${fileout1%.list}.tar
	    outputfile=$diroutputfile/${fileout}
	    mkdir -p $diroutputfile
	    echo "Output file to write: $outputfile"

	    #-----------------------------------
	    # Try 1 time before fire "FAILED"
	    # Waiting for CCRT command
	    tarcmd=tar
	    taroptions="--format=posix -W -cf"
    #	taroptions="--format=posix -cf"
	    cd $dirin1
    #	ls

	    startTime=$( getDateMilliSeconds ) # gpdebug : time
	    resCmd=0
	    $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
	    resCmd=$?
	    meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time

	    # resultCmd=$( gives_0_or_1 )
	    # if [ $resultCmd -eq 0 ] ; then
	    if [ $resCmd -eq 0 ] ; then
		sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
		echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
		echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	    else
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "====> FAILED at try #$nbtries" 
		echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile

	    fi

	    #-----------------------------------

    done	

    #------------------------------------------------------
    for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do

	    #-----------------------------------
	    echo "#-------------------"
	    echo "Set: $set"
            datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	    datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	    statusfile=${set%%.list}.status
	    logfile=${set%%.list}_${datestr}.log

	    #-----------------------------------
	    # Create status file if not exists
	    if [ ! -e $statusfile ] ; then
		    echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	    fi

	    #-----------------------------------
	    # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	    if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
		    printf "====> "
		    tail -1 $statusfile
		    continue
	    fi

	    #-----------------------------------
	    # nbtries = number of lines from status file (first comment line gives 1 for the first try)
	    nbtries=`wc -l $statusfile | cut -d' ' -f1`		
	    # echo "Nb of tries: $nbtries"

	    #-----------------------------------
	    # Set output file name from list name
	    dirin1=$PATH_SIMU/DEBUG
	    dirin2=${dirin1##${IGCM_DEM}}
	    dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
	    diroutputfile=${OUTPUT_STORE}${dirin3}
 	    fileout1=$( basename $set )
 	    fileout=${fileout1%.list}.tar
	    outputfile=$diroutputfile/${fileout}
	    mkdir -p $diroutputfile
	    echo "Output file to write: $outputfile"

	    #-----------------------------------
	    # Try 1 time before fire "FAILED"
	    # Waiting for CCRT command
	    tarcmd=tar
	    taroptions="--format=posix -W -cf"
    #	taroptions="--format=posix -cf"
	    cd $dirin1
    #	ls

	    startTime=$( getDateMilliSeconds ) # gpdebug : time
	    resCmd=0
	    $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
	    resCmd=$?
	    meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time

	    # resultCmd=$( gives_0_or_1 )
	    # if [ $resultCmd -eq 0 ] ; then
	    if [ $resCmd -eq 0 ] ; then
		sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
		echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
		echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	    else
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "====> FAILED at try #$nbtries" 
		echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile

	    fi

	    #-----------------------------------

    done	

    #------------------------------------------------------
    for set in $( ls $PATH_SIMU/store_cp/*list ) ; do

	    #-----------------------------------
	    echo "#-------------------"
	    echo "Set: $set"
            datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	    datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	    statusfile=${set%%.list}.status
	    logfile=${set%%.list}_${datestr}.log

	    #-----------------------------------
	    # Create status file if not exists
	    if [ ! -e $statusfile ] ; then
		    echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	    fi

	    #-----------------------------------
	    # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	    if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
		    printf "====> "
		    tail -1 $statusfile
		    continue
	    fi

	    #-----------------------------------
	    # nbtries = number of lines from status file (first comment line gives 1 for the first try)
	    nbtries=`wc -l $statusfile | cut -d' ' -f1`		
	    # echo "Nb of tries: $nbtries"

	    #-----------------------------------
	    # Set output file name from first and last files from the current set
	    for file in $( cat $set) ; do

		diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout%"`
		mkdir -p $diroutputfile
		echo "Output file to write: $diroutputfile/$( basename $file )"

	    #-----------------------------------
	    # Try 1 time before fire "FAILED"
	    # Waiting for CCRT command
		cpcmd="cp "
		cpoptions="-rf"
		startTime=$( getDateMilliSeconds ) # gpdebug : time
		resCmd=0
		$cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir
		resCmd=$?
		meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
	    done

	    # resultCmd=$( gives_0_or_1 )
	    # if [ $resultCmd -eq 0 ] ; then
	    if [ $resCmd -eq 0 ] ; then
		sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
		echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
		echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	    else
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "====> FAILED at try #$nbtries" 
		echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile

	    fi

	    #-----------------------------------

    done	

    #------------------------------------------------------
    for set in $( ls $PATH_SIMU/work_cp/*list ) ; do

	    #-----------------------------------
	    echo "#-------------------"
	    echo "Set: $set"
            datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	    datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	    statusfile=${set%%.list}.status
	    logfile=${set%%.list}_${datestr}.log

	    #-----------------------------------
	    # Create status file if not exists
	    if [ ! -e $statusfile ] ; then
		    echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	    fi

	    #-----------------------------------
	    # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	    if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
		    printf "====> "
		    tail -1 $statusfile
		    continue
	    fi

	    #-----------------------------------
	    # nbtries = number of lines from status file (first comment line gives 1 for the first try)
	    nbtries=`wc -l $statusfile | cut -d' ' -f1`		
	    # echo "Nb of tries: $nbtries"

	    #-----------------------------------
	    # Set output file name from first and last files from the current set
	    for file in $( cat $set) ; do

		diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout_work%"`
		mkdir -p $diroutputfile
		echo "Output file to write: $diroutputfile/$( basename $file )"

	    #-----------------------------------
	    # Try 1 time before fire "FAILED"
	    # Waiting for CCRT command
		cpcmd="cp "
		cpoptions="-rf"
		startTime=$( getDateMilliSeconds ) # gpdebug : time
		resCmd=0
		$cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1  # gpdebug : a retablir
		resCmd=$?
		meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : timeS
	    done

	    # resultCmd=$( gives_0_or_1 )
	    # if [ $resultCmd -eq 0 ] ; then
	    if [ $resCmd -eq 0 ] ; then
		sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
		echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
		echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	    else
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "====> FAILED at try #$nbtries" 
		echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile

	    fi

	    #-----------------------------------

    done
fi

for set in $( ls ${PATH_SIMU}/*.list | grep "tar_full_simul" )
do
	# set="${PATH_SIMU}/tar_full_simul.list" # gpdebug : foireux ?
	#-----------------------------------
	echo "#-------------------"
	echo "Set: $set"
        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	statusfile=${set%%.list}.status
	logfile=${set%%.list}_${datestr}.log

	#-----------------------------------
	# Create status file if not exists
	if [ ! -e $statusfile ] ; then
		echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	fi

	#-----------------------------------
	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
		printf "====> "
		tail -1 $statusfile
		continue
	fi
	
	#-----------------------------------
	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
	nbtries=`wc -l $statusfile | cut -d' ' -f1`	    
	# echo "Nb of tries: $nbtries"
	    
	#-----------------------------------
	# Set output file name from list name
	# echo "PATH_SIMU=$PATH_SIMU"
	dirOfOutputFile=`echo $PATH_SIMU | sed "s;${IGCM_DEM};${OUTPUT_STORE};" `
	# echo "dirOfOutputFile=$dirOfOutputFile"
	dirOfOutputFile_Up=$( dirname $dirOfOutputFile )
	PATH_SIMU_NAME=$( basename $dirOfOutputFile )
	fileout=${PATH_SIMU_NAME}.tar
	# echo "dirnameOfListFile=$dirnameOfListFile"
	outputfile=$dirOfOutputFile_Up/${fileout}
	# echo "outputfile=$outputfile"
	# exit 1 # gpdebug : a virer
	mkdir -p $dirOfOutputFile_Up
	echo "Output file to write: $outputfile  tar_full_simul"
	
	# exit 0 # a virer

	#-----------------------------------
	# Try 1 time before fire "FAILED"
	# Waiting for CCRT command
	tarcmd=tar
	taroptions="--format=posix -W -cf"
#	taroptions="--format=posix -cf"

	filefirst=`head -n 1 $set`
	dirinputfile=`dirname $filefirst`
	cd $dirinputfile

	set_local=${set%%.list}.local
	> $set_local
	for file in $( cat $set) ; do
	    basename $file >> $set_local 
	done
#	ls
	
	startTime=$( getDateMilliSeconds ) # gpdebug : time
	resCmd=0
	$tarcmd $taroptions $outputfile --dereference --files-from $set_local > $logfile 2>&1 # gpdebug : a retablir
	resCmd=$?
	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
	
	# resultCmd=$( gives_0_or_1 )
	# if [ $resultCmd -eq 0 ] ; then
	if [ $resCmd -eq 0 ] ; then
	    sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
	    datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	    echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
	    echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
	    echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	else
	    datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	    echo "====> FAILED at try #$nbtries" 
	    echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
	    exit 1
	fi
	
	#-----------------------------------
done