1 | # $Id$ |
---|
2 | |
---|
3 | package ObsData::Repository; |
---|
4 | |
---|
5 | use strict; |
---|
6 | use warnings; |
---|
7 | use File::Path; |
---|
8 | use POSIX qw(getcwd); |
---|
9 | use ObsData; |
---|
10 | use ObsData::Archive; |
---|
11 | use Config::IniFiles; |
---|
12 | |
---|
13 | our $CVSID = q$Id$; |
---|
14 | our $CVSREV = (q$Revision$ =~ /^Revision: (.*) $/)[0]; |
---|
15 | |
---|
16 | =head1 METHODS |
---|
17 | |
---|
18 | =head2 ObsData::Repository |
---|
19 | |
---|
20 | =head3 new |
---|
21 | |
---|
22 | Create a new ObsData::Repository object |
---|
23 | |
---|
24 | =cut |
---|
25 | |
---|
26 | sub new { |
---|
27 | my ($class, $infos, %param) = @_; |
---|
28 | |
---|
29 | my $or = { %$infos }; |
---|
30 | foreach (keys %param) { |
---|
31 | $or->{$_} = $param{$_}; |
---|
32 | } |
---|
33 | |
---|
34 | # $infos-> |
---|
35 | # obsdata ObsData object |
---|
36 | $or->{obsdata} or die "Error #1de8d015, please insult programmer"; |
---|
37 | # datatype search directly fot it |
---|
38 | # obs station, informationnal only |
---|
39 | # statusfile |
---|
40 | # status Config::IniFile object to store extraction info |
---|
41 | # dir based directory where archive are located |
---|
42 | $or->{dir} or die "No dir given"; |
---|
43 | -d $or->{dir} or return undef; |
---|
44 | # patern glob() over file |
---|
45 | |
---|
46 | $or->{status} ||= new Config::IniFiles( |
---|
47 | -file => (($or->{statusfile} && -f $or->{statusfile}) ? $or->{statusfile} : undef), |
---|
48 | # -default => '.', # Is this a good idea |
---|
49 | ); |
---|
50 | |
---|
51 | bless($or, $class); |
---|
52 | |
---|
53 | $or; |
---|
54 | } |
---|
55 | |
---|
56 | sub DESTROY { |
---|
57 | my ($self) = @_; |
---|
58 | $self->logging(0, "Destroy ObsData::Repository for %s", ref($self)); |
---|
59 | $self->save_status; |
---|
60 | } |
---|
61 | |
---|
62 | =head3 save_status |
---|
63 | |
---|
64 | Save current information about parsed files into status file |
---|
65 | |
---|
66 | =cut |
---|
67 | |
---|
68 | sub save_status { |
---|
69 | my ($self) = @_; |
---|
70 | $self->{statusfile} or return; |
---|
71 | $self->{status}->AddSection('.'); |
---|
72 | $self->{status}->SetSectionComment('.', |
---|
73 | $ObsData::CVSID, |
---|
74 | $ObsData::Repository::CVSID, |
---|
75 | scalar(localtime), |
---|
76 | ); |
---|
77 | |
---|
78 | my $hostname = `hostname`; |
---|
79 | chomp($hostname); |
---|
80 | $self->{status}->newval('.', 'hostname', $hostname); |
---|
81 | $self->{status}->SetParameterComment( |
---|
82 | '.', |
---|
83 | 'hostname', |
---|
84 | 'The computer host name' |
---|
85 | ); |
---|
86 | |
---|
87 | $self->{status}->newval('.', 'directory', $self->{dir}); |
---|
88 | $self->{status}->SetParameterComment( |
---|
89 | '.', |
---|
90 | 'directory', |
---|
91 | 'Directory where archive where located' |
---|
92 | ); |
---|
93 | |
---|
94 | $self->{status}->newval('.', 'runfrom', getcwd()); |
---|
95 | $self->{status}->SetParameterComment( |
---|
96 | '.', |
---|
97 | 'runfrom', |
---|
98 | 'The current directory' |
---|
99 | ); |
---|
100 | |
---|
101 | $self->{status}->WriteConfig($self->{statusfile}); |
---|
102 | } |
---|
103 | |
---|
104 | =head3 logging($level, $msg, ...) |
---|
105 | |
---|
106 | Pass a message for logging to ObsData Object. See L<ObsData> |
---|
107 | |
---|
108 | =cut |
---|
109 | |
---|
110 | sub logging { |
---|
111 | my $self = shift; |
---|
112 | $self->{obsdata}->logging(@_); |
---|
113 | } |
---|
114 | |
---|
115 | =head2 ObsData::Repository::dir |
---|
116 | |
---|
117 | The child class to handle directory |
---|
118 | |
---|
119 | =cut |
---|
120 | |
---|
121 | package ObsData::Repository::dir; |
---|
122 | |
---|
123 | our @ISA = qw(ObsData::Repository); |
---|
124 | |
---|
125 | =head3 new |
---|
126 | |
---|
127 | =cut |
---|
128 | |
---|
129 | sub new { |
---|
130 | my ($class, $infos, %param) = @_; |
---|
131 | |
---|
132 | my $or = ObsData::Repository->new($infos, %param) or return; |
---|
133 | |
---|
134 | bless($or, $class); |
---|
135 | } |
---|
136 | |
---|
137 | =head3 findfile |
---|
138 | |
---|
139 | Return the list of files needing to be checked |
---|
140 | |
---|
141 | =cut |
---|
142 | |
---|
143 | sub findfile { |
---|
144 | my ($self) = @_; |
---|
145 | my @files; |
---|
146 | |
---|
147 | if (!-d $self->{dir}) { |
---|
148 | $self->logging(3, |
---|
149 | "directory %s does not exists", |
---|
150 | $self->{dir}, |
---|
151 | ); |
---|
152 | return; |
---|
153 | } |
---|
154 | |
---|
155 | my $patern = $self->{patern} || '*'; |
---|
156 | foreach my $f (glob("$self->{dir}/$patern")) { |
---|
157 | |
---|
158 | -f $f or next; # skip no regular files |
---|
159 | $f =~ m/obsdata\.ini$/ and next; # avoid default status file |
---|
160 | |
---|
161 | my $bf = $f; |
---|
162 | $bf =~ s!^\Q$self->{dir}\E/*!!; |
---|
163 | push(@files, $bf); |
---|
164 | } |
---|
165 | @files |
---|
166 | } |
---|
167 | |
---|
168 | =head3 process |
---|
169 | |
---|
170 | Process all files in directory |
---|
171 | |
---|
172 | =cut |
---|
173 | |
---|
174 | sub process { |
---|
175 | my ($self) = @_; |
---|
176 | $self->logging(0, "%s() start for %s", (caller(0))[3], $self->{dir}); |
---|
177 | |
---|
178 | foreach my $file ($self->findfile) { |
---|
179 | my $orda = ObsData::Repository::dir::archive->new( |
---|
180 | $self, |
---|
181 | archivefile => $file, |
---|
182 | ); |
---|
183 | if(!$orda->do_continue('do_archive')) { |
---|
184 | $self->logging(0, |
---|
185 | "Archive %s/%s allready parsed, skipping", |
---|
186 | $self->{dir}, |
---|
187 | $file |
---|
188 | ); |
---|
189 | next; |
---|
190 | } |
---|
191 | $orda->process; |
---|
192 | } |
---|
193 | } |
---|
194 | |
---|
195 | =head2 ObsData::Repository::dir::archive |
---|
196 | |
---|
197 | Object to handle archive and to dispatch data |
---|
198 | |
---|
199 | =cut |
---|
200 | |
---|
201 | package ObsData::Repository::dir::archive; |
---|
202 | |
---|
203 | our @ISA = qw(ObsData::Repository); |
---|
204 | |
---|
205 | =head3 new |
---|
206 | |
---|
207 | =cut |
---|
208 | |
---|
209 | sub new { |
---|
210 | my ($class, $infos, %param) = @_; |
---|
211 | |
---|
212 | my $or = ObsData::Repository->new($infos, %param) or return; |
---|
213 | # dir directory to parse |
---|
214 | $or->{archivefile} or die "No archivefile given"; |
---|
215 | |
---|
216 | # $or->{interative_callback} |
---|
217 | $or->{_cache_checks} = {}; |
---|
218 | |
---|
219 | $or->logging(0, |
---|
220 | "Preparing to parse %s/%s", |
---|
221 | $or->{dir}, |
---|
222 | $or->{archivefile} |
---|
223 | ); |
---|
224 | |
---|
225 | bless($or, $class); |
---|
226 | } |
---|
227 | |
---|
228 | =head3 load_archive |
---|
229 | |
---|
230 | Try to read archive, return 1 on success |
---|
231 | |
---|
232 | =cut |
---|
233 | |
---|
234 | sub load_archive { |
---|
235 | my ($self) = @_; |
---|
236 | if ($self->{Oarchive}) { |
---|
237 | return 1; |
---|
238 | } else { |
---|
239 | $self->{Oarchive} = ObsData::Archive->new("$self->{dir}/$self->{archivefile}"); |
---|
240 | if ($self->{Oarchive}) { |
---|
241 | $self->logging(0, |
---|
242 | "Archive %s/%s properly load", |
---|
243 | $self->{dir}, $self->{archivefile} |
---|
244 | ); |
---|
245 | return 1; |
---|
246 | } else { |
---|
247 | $self->logging(4, |
---|
248 | "Can't handle archive %s/%s: %s, skipping", |
---|
249 | $self->{dir}, $self->{archivefile}, |
---|
250 | ObsData::Archive->error |
---|
251 | ); |
---|
252 | return 0; |
---|
253 | } |
---|
254 | } |
---|
255 | } |
---|
256 | |
---|
257 | =head2 getvalue |
---|
258 | |
---|
259 | =cut |
---|
260 | |
---|
261 | sub getvalue { |
---|
262 | my ($self, $var, $default) = @_; |
---|
263 | return $self->{status}->val($self->{archivefile}, $var, $default); |
---|
264 | } |
---|
265 | |
---|
266 | =head2 setvalue |
---|
267 | |
---|
268 | =cut |
---|
269 | |
---|
270 | sub setvalue { |
---|
271 | my ($self, $var, $val, $comment) = @_; |
---|
272 | if (!$self->{status}->SectionExists($self->{archivefile})) { |
---|
273 | $self->{status}->AddSection($self->{archivefile}); |
---|
274 | } |
---|
275 | $self->{status}->newval($self->{archivefile}, $var, $val) if(defined($val)); |
---|
276 | $self->{status}->SetParameterComment( |
---|
277 | $self->{archivefile}, $var, $comment |
---|
278 | ) if(defined($comment)); |
---|
279 | } |
---|
280 | |
---|
281 | =head2 get_data_value |
---|
282 | |
---|
283 | =cut |
---|
284 | |
---|
285 | sub get_data_value { |
---|
286 | my ($self, $datafile, $var, $default) = @_; |
---|
287 | return $self->getvalue( |
---|
288 | sprintf('data_%s_%s', $var, $datafile), |
---|
289 | $var, |
---|
290 | $default |
---|
291 | ); |
---|
292 | } |
---|
293 | |
---|
294 | =head2 set_data_value |
---|
295 | |
---|
296 | =cut |
---|
297 | |
---|
298 | sub set_data_value { |
---|
299 | my ($self, $datafile, $var, $val, $comment) = @_; |
---|
300 | $self->setvalue( |
---|
301 | sprintf('data_%s_%s', $var, $datafile), |
---|
302 | $val, |
---|
303 | $comment |
---|
304 | ); |
---|
305 | } |
---|
306 | |
---|
307 | =head3 checks |
---|
308 | |
---|
309 | =cut |
---|
310 | |
---|
311 | my $checks = { |
---|
312 | archive_exists => sub { |
---|
313 | my ($self) = @_; |
---|
314 | my $res; |
---|
315 | if($self->{status}->SectionExists($self->{archivefile})) { |
---|
316 | $res = 1; |
---|
317 | } else { |
---|
318 | $res = 0; |
---|
319 | } |
---|
320 | return( |
---|
321 | $res, |
---|
322 | $self->logging(0, |
---|
323 | "%s/%s is%s new archive", |
---|
324 | $self->{dir}, |
---|
325 | $self->{archivefile}, |
---|
326 | $res ? " not" : "", |
---|
327 | ) |
---|
328 | ); |
---|
329 | }, |
---|
330 | |
---|
331 | archive_size => sub { |
---|
332 | my ($self) = @_; |
---|
333 | my $res; |
---|
334 | if (defined(my $oldsize = $self->getvalue('size')) && |
---|
335 | defined(my $size = (stat("$self->{dir}/$self->{archivefile}"))[7])) { |
---|
336 | my $res = $size <=> $oldsize; |
---|
337 | return $res, $self->logging(0, |
---|
338 | "Size for %s/%s %s", |
---|
339 | $self->{dir}, |
---|
340 | $self->{archivefile}, |
---|
341 | $res == 0 ? |
---|
342 | sprintf("does not differ (%d)", $size) : |
---|
343 | sprintf("differ %d => %d", $oldsize, $size) |
---|
344 | ); |
---|
345 | } |
---|
346 | }, |
---|
347 | |
---|
348 | archive_mtime => sub { |
---|
349 | my ($self) = @_; |
---|
350 | my $res; |
---|
351 | if (defined(my $oldmtime = $self->getvalue('mtime')) && |
---|
352 | defined(my $mtime = (stat("$self->{dir}/$self->{archivefile}"))[9])) { |
---|
353 | my $res = $mtime <=> $oldmtime; |
---|
354 | return $res, $self->logging(0, |
---|
355 | "Mtime for %s/%s %s", |
---|
356 | $self->{dir}, |
---|
357 | $self->{archivefile}, |
---|
358 | $res == 0 ? |
---|
359 | sprintf("does not differ (%d)", $mtime) : |
---|
360 | sprintf("differ %d => %d", $oldmtime, $mtime) |
---|
361 | ); |
---|
362 | } |
---|
363 | }, |
---|
364 | |
---|
365 | dest_exists => sub { |
---|
366 | my ($self, $datafile, $dest) = @_; |
---|
367 | if (-e $dest) { |
---|
368 | return 1, "$dest exists"; |
---|
369 | } else { |
---|
370 | return 0, "$dest don't exists"; |
---|
371 | } |
---|
372 | }, |
---|
373 | |
---|
374 | data_extracted => sub { |
---|
375 | my ($self, $datafile) = @_; |
---|
376 | my $last_process = $self->gat_data_val($datafile, 'processtime'); |
---|
377 | if ($last_process) { |
---|
378 | $self->logging(0, "%s/%s has been already processed at %s", |
---|
379 | $self->{archivefile}, |
---|
380 | $datafile, |
---|
381 | scalar(localtime($last_process)), |
---|
382 | ); |
---|
383 | } |
---|
384 | $last_process |
---|
385 | }, |
---|
386 | }; |
---|
387 | |
---|
388 | #sub DESTROY { |
---|
389 | # my ($self) = @_; |
---|
390 | # $self->logging(0, "Destroy ObsData::Repository::archive for %s", ref($self)); |
---|
391 | # $self->SUPER::DESTROY(); |
---|
392 | #} |
---|
393 | |
---|
394 | =head3 do_check($check, @args) |
---|
395 | |
---|
396 | Perform $check with @args if exists and return result of check |
---|
397 | |
---|
398 | =cut |
---|
399 | |
---|
400 | sub do_check { |
---|
401 | my ($self, $check, @args) = @_; |
---|
402 | |
---|
403 | if (!$checks->{$check}) { |
---|
404 | die $self->logging(5, |
---|
405 | "check %s does not exists, exiting now", |
---|
406 | $check |
---|
407 | ); |
---|
408 | } |
---|
409 | |
---|
410 | return $checks->{$check}->($self, $check, @args); |
---|
411 | } |
---|
412 | |
---|
413 | =head3 default_checks($why, @args) |
---|
414 | |
---|
415 | Perform defaults checks |
---|
416 | |
---|
417 | =cut |
---|
418 | |
---|
419 | sub default_checks { |
---|
420 | my ($self, $why, @args) = @_; |
---|
421 | |
---|
422 | my ($code, $text); |
---|
423 | |
---|
424 | if ($why eq 'do_archive') { |
---|
425 | ($code, $text) = $self->do_check('archive_size'); |
---|
426 | if ($code) { |
---|
427 | $self->logging(3, $text); |
---|
428 | } |
---|
429 | |
---|
430 | ($code, $text) = $self->do_check('archive_mtime'); |
---|
431 | if ($code) { |
---|
432 | $self->logging(3, $text); |
---|
433 | } |
---|
434 | |
---|
435 | ($code, $text) = $self->do_check('archive_exists'); |
---|
436 | if ($code != 0) { |
---|
437 | $self->logging(0, "%s => skiping", $text); |
---|
438 | return 0; |
---|
439 | } |
---|
440 | } elsif ($why eq 'do_data') { |
---|
441 | |
---|
442 | } elsif ($why eq 'overwrite') { |
---|
443 | ($code, $text) = $self->do_check('dest_exists', @args); |
---|
444 | if ($code) { |
---|
445 | $self->logging(0, "%s", $text); |
---|
446 | return 0; |
---|
447 | } else { |
---|
448 | return 1; |
---|
449 | } |
---|
450 | } |
---|
451 | |
---|
452 | 1; |
---|
453 | } |
---|
454 | |
---|
455 | =head3 do_continue($why, @args) |
---|
456 | |
---|
457 | Perform default or call the interactive callback. |
---|
458 | |
---|
459 | $why is the reason of the call |
---|
460 | @args are argument to passed to callback |
---|
461 | |
---|
462 | =cut |
---|
463 | |
---|
464 | sub do_continue { |
---|
465 | my ($self, $why, @args) = @_; |
---|
466 | |
---|
467 | if ($self->{interative_callback}) { |
---|
468 | return $self->{interative_callback}->($self, $why, @args); |
---|
469 | } else { |
---|
470 | return $self->default_checks($why, @args); |
---|
471 | } |
---|
472 | } |
---|
473 | |
---|
474 | =head3 list_archive |
---|
475 | |
---|
476 | Return archive content |
---|
477 | |
---|
478 | =cut |
---|
479 | |
---|
480 | sub list_archive { |
---|
481 | my ($self) = @_; |
---|
482 | $self->load_archive or return; |
---|
483 | |
---|
484 | return $self->{Oarchive}->ls; |
---|
485 | } |
---|
486 | |
---|
487 | =head3 update_status |
---|
488 | |
---|
489 | Update status information about current archive |
---|
490 | |
---|
491 | =cut |
---|
492 | |
---|
493 | sub update_status { |
---|
494 | my ($self) = @_; |
---|
495 | my @filestats = stat("$self->{dir}/$self->{archivefile}"); |
---|
496 | |
---|
497 | $self->setvalue('mtime', $filestats[9]); |
---|
498 | $self->setvalue('size', $filestats[7]); |
---|
499 | $self->setvalue('configtime', $self->{obsdata}->config_mtime); |
---|
500 | } |
---|
501 | |
---|
502 | # Sub data function: |
---|
503 | |
---|
504 | |
---|
505 | |
---|
506 | =head3 get_data_dest($datafile) |
---|
507 | |
---|
508 | Return the datatype and destination for $datafile |
---|
509 | |
---|
510 | =cut |
---|
511 | |
---|
512 | sub get_data_dest { |
---|
513 | my ($self, $datafile) = @_; |
---|
514 | |
---|
515 | my ($t, $d) = $self->{obsdata}->build_dest_filename( |
---|
516 | $datafile, |
---|
517 | $self->{datatype} |
---|
518 | ); |
---|
519 | $self->logging(0, "%s/%s file, datatype %s, destination %s", |
---|
520 | $self->{archivefile}, |
---|
521 | $datafile, |
---|
522 | $t || '(none)', |
---|
523 | $d || '(none)', |
---|
524 | ); |
---|
525 | if ($t) { |
---|
526 | $self->set_data_value($datafile, 'type', $t); |
---|
527 | } |
---|
528 | return($t, $d); |
---|
529 | } |
---|
530 | |
---|
531 | =head3 extract_data($datafile, $datatype, $destfile) |
---|
532 | |
---|
533 | Extract data file into $destfile, $datafile is informationnal only. |
---|
534 | |
---|
535 | $datatype and $destfile are the result of L<get_data_dest>. |
---|
536 | |
---|
537 | A basic call is: |
---|
538 | $o->extract_data($file, $self->get_data_dest($file)); |
---|
539 | |
---|
540 | Return 1 on success |
---|
541 | |
---|
542 | =cut |
---|
543 | |
---|
544 | sub extract_data { |
---|
545 | # $datatype, informationnal only here |
---|
546 | my ($self, $datafile, $datatype, $destfile) = @_; |
---|
547 | $self->load_archive or return 0; |
---|
548 | my $ltime = time; |
---|
549 | |
---|
550 | $self->set_data_value( |
---|
551 | $datafile, |
---|
552 | 'processtime', |
---|
553 | $ltime, |
---|
554 | scalar(localtime($ltime)) . |
---|
555 | ", $ObsData::VERSION: $ObsData::CVSREV, $ObsData::Repository::CVSREV", |
---|
556 | ); |
---|
557 | $self->set_data_value( |
---|
558 | $datafile, |
---|
559 | 'processversion', |
---|
560 | $ObsData::VERSION, |
---|
561 | ); |
---|
562 | |
---|
563 | if (!$destfile) { |
---|
564 | $self->logging(3, |
---|
565 | "%s/%s has no destination", |
---|
566 | $self->{archivefile}, $datafile, |
---|
567 | ); |
---|
568 | return 0; |
---|
569 | } |
---|
570 | |
---|
571 | if(!$self->do_continue('overwrite', $datafile, $destfile)) { |
---|
572 | $self->logging(3, |
---|
573 | "%s extracted from %s/%s already exists, I won't overwrite it", |
---|
574 | $destfile, |
---|
575 | $self->{archivefile}, $datafile, |
---|
576 | ); |
---|
577 | return 0; |
---|
578 | } |
---|
579 | |
---|
580 | $self->logging(0, |
---|
581 | "Trying to extract %s from %s/%s", |
---|
582 | $destfile, |
---|
583 | $self->{archivefile}, $datafile |
---|
584 | ); |
---|
585 | |
---|
586 | my ($dirname) = $destfile =~ m!(?:(.*)/)?(.*)$!; #! vim syntax sux |
---|
587 | |
---|
588 | if ($dirname && ! -d $dirname) { |
---|
589 | $self->logging(0, "Creating directory %s", $dirname); |
---|
590 | if( ! File::Path::mkpath($dirname)) { |
---|
591 | $self->logging( |
---|
592 | 4, "Can't create directory %s", |
---|
593 | $dirname |
---|
594 | ); |
---|
595 | return 0; |
---|
596 | } |
---|
597 | } |
---|
598 | |
---|
599 | if ($self->{Oarchive}->extract($datafile, $destfile)) { |
---|
600 | $self->set_data_value($datafile, 'dest', $destfile); |
---|
601 | $self->logging( |
---|
602 | 1, "Extraction of %s/%s done as %s", |
---|
603 | $self->{archivefile}, |
---|
604 | $datafile, |
---|
605 | $destfile, |
---|
606 | ); |
---|
607 | |
---|
608 | } else { |
---|
609 | $self->logging( |
---|
610 | 4, "Can't extract %s/%s to %s: %s", |
---|
611 | $self->{archivefile}, $datafile, $destfile, |
---|
612 | $self->{Oarchive}->error, |
---|
613 | ); |
---|
614 | return 0; |
---|
615 | } |
---|
616 | |
---|
617 | return 1; |
---|
618 | } |
---|
619 | |
---|
620 | =head3 process |
---|
621 | |
---|
622 | Process all files in the archive |
---|
623 | |
---|
624 | =cut |
---|
625 | |
---|
626 | sub process { |
---|
627 | my ($self) = @_; |
---|
628 | |
---|
629 | foreach my $data ($self->list_archive) { |
---|
630 | if (!$self->do_continue('do_data', $data)) { |
---|
631 | $self->logging(0, |
---|
632 | "%s/%s has allready processed, skipping", |
---|
633 | $self->{archivefile}, |
---|
634 | $data |
---|
635 | ); |
---|
636 | next; |
---|
637 | } |
---|
638 | $self->extract_data($data, $self->get_data_dest($data)); |
---|
639 | } |
---|
640 | |
---|
641 | $self->update_status; |
---|
642 | |
---|
643 | return 1; |
---|
644 | } |
---|
645 | |
---|
646 | =head1 AUTHOR |
---|
647 | |
---|
648 | Olivier Thauvin <olivier.thauvin@aerov.jussieu.fr> |
---|
649 | |
---|
650 | =cut |
---|
651 | |
---|
652 | 1; |
---|