Changeset 257

Timestamp:

10/13/06 00:21:13 (18 years ago)

Author:

nanardon

Message:

• transfert are done for whole directory, not file by file

add -ttest options

• add support of a max size for the repository, old file making this size exceeded are trashed

Location:

trunk/soft/scripts

Files:

• transfert (modified) (3 diffs)
• transfert-config (modified) (1 diff)

trunk/soft/scripts/transfert

@@ -21 +21 @@
 use Pod::Usage;
 use Digest::MD5;
+use File::Find;
+use File::Temp;
 
 # 0 .. 6 => debug, info, report, warn, error, die
@@ -33 +35 @@
     'logfile=s' => \my $logfile,
     'help|h' => sub { pod2usage(-exitval => 0) },
+    't|test' => \my $test,
 ) or pod2usage();
 
@@ -227 +230 @@
     };
 
-    my @files = -d $source ? glob("$source/*") : glob($source);
-    my @datafiles;
-    
-    foreach my $f (@files) {
-        if ($f =~ /.md5sum$/) {
-            loging(0, "%s is md5sum file, skipping", $f);
-            next;
+    # Searching files
+    my @files;
+    find(
+        {
+            wanted => sub {
+                -d $File::Find::name and return;
+                my @stats = stat _ or return;
+                push(@files, {
+                    f => $File::Find::name,
+                    s => $stats[7],
+                    t => $stats[9],
+                });
+            },
+        },
+        $source,
+    );
+
+    my @cmd = (
+        split(/ /, $config->val($entry, 'cmd', 'rsync -avPH')),
+        $source,
+        $dest,
+    );
+    loging(0, "running `%s'", join(' ', @cmd));
+    my $error = 0;
+    while ($error < 3) {
+        my $exit = $test ? 0 : system(@cmd);
+        if ($exit == 0) {
+            # Wahou, success !
+            $error = 0;
+            last;
+        } elsif ($exit == -1) {
+            loging(4, "tranfert failed unable to start `%s'", $cmd[0]);
+            $error++;
+        } elsif ($?) {
+            $error++;
+            $error < 3 and next;
+            loging(4, "tranfert failed, exit code: %d", $? >> 8);
         }
-        
-        if (! -f $f) {
-            loging(0, "%s is not a regular file, skipping", $f);
-            next;
+    }
+    $error and return 0;
+
+    # No error running action:
+    if (my $postrun = $config->val($entry, 'postrun')) {
+        loging(0, "found postrun: %s for %s", $postrun || '(unset)', $entry);
+        my $fpostrun = sprintf($postrun);
+        loging(0, "Running `%s'", $fpostrun);
+        my $postrunres = system($fpostrun);
+        loging(0, "Postrun exit: %d", $postrunres);
+    }
+
+    if (my $maxsize = $config->val($entry, 'maxsize')) {
+        my $size = 0;
+        foreach my $f (sort { ($a->{t} || 0) <=> ($b->{t} || 0) } @files) {
+            if($size + $f->{s} > $maxsize) {
+                loging(0, "Trashing old %s because size exceed", $f->{f});
+                unlink($f->{f});
+            }
+            $size += $f->{s} || 0;
         }
-        my ($fsize, $ftime);
-        {
-            my @st = stat($f);
-            $fsize = $st[7];
-            $ftime = $st[9];
-        }
-        
-        # Does the transfert need
-        if (-f "$f.md5sum") {
-            if ((stat("$f.md5sum"))[9] > $ftime) {
-                loging(0, "%s exists and is newer, assume transfert is not need", "$f.md5sum");
-                next;
-            }
-        }
-        
-        # Creating md5sum file
-        if(open(my $md5h, "> $f.md5sum")) {
-            if (open(my $fh, "< $f")) {
-                loging(0, "Creating %s md5 file", "$f.md5sum");
-                binmode($fh);
-                my $md5 = new Digest::MD5;
-                $md5->addfile($fh);
-                printf $md5h 
-                        "# %s\n# %s (%d octets)\n# %s\n%s  %s\n",
-                        q$Id$,
-                        $f, $fsize,
-                        scalar(localtime),
-                        $md5->hexdigest,
-                        ($f =~ m!(?:.*/)?(.*)$!)[0];
-                close($fh);
-            } else {
-                login(5, "Can't open %s for reading", $f);
-            }
-            close($md5h);
-        } else {
-            login(5, "Can't open %s for writting", "$f.md5sum");
-        }
-
-        my $error = 0;
-        foreach my $filetotranfert (grep { -f $_ } ("$f.md5sum", $f)) {
-            my $cmd = sprintf(
-                "%s %s %s",
-                $config->val($entry, 'cmd', 'rsync -e ssh'),
-                $filetotranfert,
-                $dest,
-            );
-            loging(0, "running `%s'", $cmd);
-            if (system($cmd)) {
-                loging(4, "tranfert failed for %s, exit code: %d", $filetotranfert, $? >> 8);
-                unlink("$f.md5sum"); # ensure transfert will be retried
-                $error++;
-            }
-        }
-        
-        if ($error) {
-            loging(5, "Failed to transfert %s", $f);
-            next;
-        } else {
-            loging(1, "%s (%d) transfer ok", $f, $fsize);
-            push(@datafiles, { file => $f, size => $fsize });
-        }
-
-        # No error running action:
-        if (my $postrun = $config->val($entry, 'postrun')) {
-            loging(0, "found postrun: %s for %s", $postrun || '(unset)', $entry);
-            foreach my $filetotranfert (grep { -f $_ } ("$f.md5sum", $f)) {
-                my $fpostrun = sprintf($postrun, $filetotranfert);
-                loging(0, "Running `%s'", $fpostrun);
-                my $postrunres = system($fpostrun);
-                loging(0, "Postrun exit: %d", $postrunres);
-            }
-        }
-    }
-
-    my $totalsize = 0;
-    $totalsize += $_->{size} foreach (@datafiles);
-    loging(2, "%s: %d file transmit, %d octets", $entry, scalar(@datafiles), $totalsize);
-    
-    return 0;
+    }
+
+    return 1;
 }
 

trunk/soft/scripts/transfert-config

@@ -5 +5 @@
 
 [test]
-source=/home/users/olivier/lidarcvs/soft/scripts/*
+source=/home/users/olivier/myprojects/obsdata/
 dest=/net/tmp/s/
 mailto=thauvin@aerov.jussieu.fr, olivier.thauvin@aero.jussieu.fr
-postrun= cp %s /tmp
+maxsize = 53791040
+#postrun= cp %s /tmp