1 | # $Id$ |
---|
2 | |
---|
3 | package ObsData; |
---|
4 | |
---|
5 | use strict; |
---|
6 | use warnings; |
---|
7 | use Config::IniFiles; |
---|
8 | use POSIX qw(strftime); |
---|
9 | use ObsData::Repository; |
---|
10 | use Mail::Sendmail; |
---|
11 | use MIME::QuotedPrint; |
---|
12 | |
---|
13 | my @loglevel = ( |
---|
14 | 'DEBUG', |
---|
15 | 'INFO', |
---|
16 | 'RESULT', |
---|
17 | 'WARNING', |
---|
18 | 'ERROR', |
---|
19 | 'FATAL', |
---|
20 | ); |
---|
21 | |
---|
22 | our $VERSION = "0.3.0"; |
---|
23 | our $CVSID = q$Id$; |
---|
24 | our $CVSREV = (q$Revision$ =~ /^Revision: (.*) $/)[0]; |
---|
25 | |
---|
26 | =head1 NAME |
---|
27 | |
---|
28 | ObsData - Main object to manage data files |
---|
29 | |
---|
30 | =head1 SYNOPSIS |
---|
31 | |
---|
32 | use ObsData; |
---|
33 | my $conf = "configfile"; |
---|
34 | my $obsdata = ObsData->new($conf); |
---|
35 | |
---|
36 | =head1 METHODS |
---|
37 | |
---|
38 | =head2 new($configfile) |
---|
39 | |
---|
40 | Create a new Obsdata object from $configfile |
---|
41 | |
---|
42 | =cut |
---|
43 | |
---|
44 | sub new { |
---|
45 | my ($class, $configfile, %options) = @_; |
---|
46 | my $obsdata = { |
---|
47 | config => new Config::IniFiles( |
---|
48 | -file => $configfile, |
---|
49 | -default => 'global', |
---|
50 | -allowcontinue => 1 |
---|
51 | ), |
---|
52 | verbose => defined($options{verbose}) ? $options{verbose} : 1, |
---|
53 | logcallback => $options{logcallback}, |
---|
54 | logfile => $options{logfile}, |
---|
55 | dry_run => $options{dry_run}, |
---|
56 | interactive_callback => $options{interactive_callback}, |
---|
57 | processed_lists => [], |
---|
58 | }; |
---|
59 | |
---|
60 | if (!($configfile && -f $configfile && -r _)) { |
---|
61 | return undef; |
---|
62 | } |
---|
63 | |
---|
64 | $obsdata->{config} or return undef; |
---|
65 | |
---|
66 | $obsdata->{logfile} ||= $obsdata->{config}->val('.', 'logfile') || 'obsdata.log'; |
---|
67 | |
---|
68 | bless($obsdata, $class); |
---|
69 | } |
---|
70 | |
---|
71 | sub DESTROY { |
---|
72 | my ($self) = @_; |
---|
73 | |
---|
74 | if ($self->{loghandle}) { |
---|
75 | close($self->{loghandle}); |
---|
76 | $self->{loghandle} = undef; |
---|
77 | } |
---|
78 | } |
---|
79 | |
---|
80 | =head2 load |
---|
81 | |
---|
82 | Prepare the object for usage |
---|
83 | |
---|
84 | =cut |
---|
85 | |
---|
86 | sub load { |
---|
87 | my ($self) = @_; |
---|
88 | |
---|
89 | if (!open($self->{loghandle}, ">> $self->{logfile}")) { |
---|
90 | $self->{loghandle} = undef; |
---|
91 | $self->logging(5, "Can't open log file %s, exiting", "$self->{logfile}"); |
---|
92 | return 0; |
---|
93 | } |
---|
94 | |
---|
95 | return 1; |
---|
96 | } |
---|
97 | |
---|
98 | =head2 logging($level, $message, ...) |
---|
99 | |
---|
100 | Pass a message into the logging process. |
---|
101 | |
---|
102 | $level is the level of message |
---|
103 | $message and or other arguments are printf format string |
---|
104 | |
---|
105 | =cut |
---|
106 | |
---|
107 | sub logging { |
---|
108 | my ($self, $level, $fmt, @val) = @_; |
---|
109 | my $msg = sprintf($fmt, @val); |
---|
110 | my $logh = $self->{loghandle}; |
---|
111 | if ($self->{logcallback}) { |
---|
112 | $self->{logcallback}->($level, $msg); |
---|
113 | } |
---|
114 | if($level >= 0 && $level >= $self->{verbose}) { |
---|
115 | if ($logh) { |
---|
116 | printf $logh |
---|
117 | "%-9s %s %s\n", |
---|
118 | sprintf("[%s]", $self->loglevel($level)), |
---|
119 | strftime("%b %d %H:%M:%S %Y", gmtime), |
---|
120 | $msg; |
---|
121 | } |
---|
122 | } |
---|
123 | $msg |
---|
124 | } |
---|
125 | |
---|
126 | =head2 loglevel($level) |
---|
127 | |
---|
128 | Return the text human readable log level associate to $level number |
---|
129 | |
---|
130 | =cut |
---|
131 | |
---|
132 | sub loglevel { |
---|
133 | my $l = pop(@_); |
---|
134 | defined($l) or $l = pop(@_); |
---|
135 | return $loglevel[ $l ] || "?????"; |
---|
136 | } |
---|
137 | |
---|
138 | =head2 checkconfig() |
---|
139 | |
---|
140 | Check the validity of information contains in configfile. |
---|
141 | |
---|
142 | Notice: This test does not include the syntaxe validity |
---|
143 | |
---|
144 | =cut |
---|
145 | |
---|
146 | sub checkconfig { |
---|
147 | my ($self) = @_; |
---|
148 | my $result = 1; |
---|
149 | foreach my $g ($self->{config}->GroupMembers('Obs')) { |
---|
150 | my ($obs) = $g =~ /\S+\s+(.*)/; |
---|
151 | if (!$self->{config}->SectionExists($obs)) { |
---|
152 | print STDERR "E: '$obs' is listed as Obs but it does not exists\n"; |
---|
153 | next; |
---|
154 | } |
---|
155 | foreach my $param ($self->{config}->Parameters($obs)) { |
---|
156 | } |
---|
157 | foreach my $datatype ($self->list_datatype) { |
---|
158 | foreach my $var (qw(match match_archive)) { |
---|
159 | my $regexp = $self->getvalue($datatype, $var) or next; # next ? are we sure ? |
---|
160 | eval { qr/$regexp/ }; # Many thanks Rafael |
---|
161 | if ($@) { |
---|
162 | $self->logging(4, "error in regexp for %s: '%s': %s", |
---|
163 | $datatype, |
---|
164 | $regexp, |
---|
165 | $@, |
---|
166 | ); |
---|
167 | $result = 0; |
---|
168 | # TODO set this reg unavalable |
---|
169 | } |
---|
170 | } |
---|
171 | } |
---|
172 | } |
---|
173 | return($result); |
---|
174 | } |
---|
175 | |
---|
176 | =head2 getvalue($section, $var, $default) |
---|
177 | |
---|
178 | Return a value from the configuration, $default is assumed if the value is not set |
---|
179 | |
---|
180 | =cut |
---|
181 | |
---|
182 | sub getvalue { |
---|
183 | my ($self, $section, $var, $default) = @_; |
---|
184 | $self->{config}->val($section, $var, $default); |
---|
185 | } |
---|
186 | |
---|
187 | =head2 config_mtime |
---|
188 | |
---|
189 | Return the modification time of config file currently used |
---|
190 | |
---|
191 | =cut |
---|
192 | |
---|
193 | sub config_mtime { |
---|
194 | my ($self) = @_; |
---|
195 | return $self->{configmtime} ||= (stat($self->{config}->GetFileName))[9]; |
---|
196 | } |
---|
197 | |
---|
198 | =head2 list_obs |
---|
199 | |
---|
200 | Return the list of observatories defined in configuration |
---|
201 | |
---|
202 | =cut |
---|
203 | |
---|
204 | sub list_obs { |
---|
205 | my ($self) = @_; |
---|
206 | grep { $self->{config}->SectionExists($_) } |
---|
207 | map { s/^\S+\s+//; $_ } |
---|
208 | $self->{config}->GroupMembers('Obs'); |
---|
209 | } |
---|
210 | |
---|
211 | =head2 is_obs($obsname) |
---|
212 | |
---|
213 | Return True if $obsname is an observatory |
---|
214 | |
---|
215 | =cut |
---|
216 | |
---|
217 | sub is_obs { |
---|
218 | my ($self, $obs) = @_; |
---|
219 | scalar(grep { $_ eq "Obs $obs" } $self->{config}->GroupMembers('Obs')); |
---|
220 | } |
---|
221 | |
---|
222 | =head2 list_obsdatadir($obsname) |
---|
223 | |
---|
224 | Return a hash of data directories per data type for $obsname observatories |
---|
225 | |
---|
226 | =cut |
---|
227 | |
---|
228 | sub list_obsdatadir { |
---|
229 | my ($self, $obs) = @_; |
---|
230 | $self->is_obs($obs) or return undef; |
---|
231 | map { m,^datadir/(.*),; ( ($1 || "") => $self->{config}->val($obs, $_) ) } |
---|
232 | grep { m,^datadir/, || $_ eq 'datadir' } |
---|
233 | $self->{config}->Parameters($obs) |
---|
234 | } |
---|
235 | |
---|
236 | =head2 list_typedatadir($type) |
---|
237 | |
---|
238 | List all directories for $type data |
---|
239 | |
---|
240 | =cut |
---|
241 | |
---|
242 | sub list_typedatadir { |
---|
243 | my ($self, $type) = @_; |
---|
244 | my %dirs; |
---|
245 | foreach my $obs ($self->list_obs) { |
---|
246 | $dirs{$_} = 1 foreach(grep { $_ } $self->get_datadir($obs, $type)); |
---|
247 | } |
---|
248 | keys %dirs; |
---|
249 | |
---|
250 | } |
---|
251 | |
---|
252 | =head2 get_datadir($obs, $type) |
---|
253 | |
---|
254 | Return a list of directories for $type data on $obs observatory |
---|
255 | |
---|
256 | =cut |
---|
257 | |
---|
258 | sub get_datadir { |
---|
259 | my ($self, $obs, $type) = @_; |
---|
260 | $self->is_obs($obs) or return undef; |
---|
261 | grep { defined($_) } ($self->getvalue($obs, "datadir/$type"), $self->getvalue($obs, "datadir")); |
---|
262 | } |
---|
263 | |
---|
264 | =head2 list_datatype |
---|
265 | |
---|
266 | Return a list of datatype handle by config |
---|
267 | |
---|
268 | =cut |
---|
269 | |
---|
270 | sub list_datatype { |
---|
271 | my ($self) = @_; |
---|
272 | grep { $_ ne 'global' } |
---|
273 | grep { $_ !~ /^Obs\s+/ } |
---|
274 | grep { !$self->is_obs($_) } $self->{config}->Sections; |
---|
275 | } |
---|
276 | |
---|
277 | =head2 get_obs_data_handle($obs, $datatype) |
---|
278 | |
---|
279 | Return an ObsData::Repository object about directory for $obs station |
---|
280 | about $datatype data. $datatype can be undefined if you want to get default |
---|
281 | directory |
---|
282 | |
---|
283 | =cut |
---|
284 | |
---|
285 | sub get_obs_data_handle { |
---|
286 | my ($self, $obs, $datatype) = @_; |
---|
287 | |
---|
288 | my $dir = $self->getvalue($obs, ($datatype ? "datadir/$datatype" : "datadir")); |
---|
289 | if (!$dir) { |
---|
290 | $self->logging(4, |
---|
291 | "Can't find data directory for %s, type: %s", |
---|
292 | $obs, $datatype || '(none)' |
---|
293 | ); |
---|
294 | return undef; |
---|
295 | } |
---|
296 | my $or = ObsData::Repository::dir->new( |
---|
297 | { |
---|
298 | obsdata => $self, |
---|
299 | dir => $dir, |
---|
300 | obs => $obs, |
---|
301 | datatype => $datatype, |
---|
302 | dry_run => $self->{dry_run}, # FIXME does this have a better place ? |
---|
303 | patern => $self->getvalue( |
---|
304 | $obs, |
---|
305 | ($datatype ? "searchfiles/$datatype" : "searchfiles") |
---|
306 | ), |
---|
307 | statusfile => $self->getvalue( |
---|
308 | $obs, |
---|
309 | ($datatype ? "index/$datatype" : "index"), |
---|
310 | "$dir/obsdata.ini" |
---|
311 | ), |
---|
312 | } |
---|
313 | ); |
---|
314 | if (!defined($or)) { |
---|
315 | $self->logging(4, "Can't parse %s, check directory exists", $dir); |
---|
316 | return undef; |
---|
317 | } |
---|
318 | |
---|
319 | return($or); |
---|
320 | } |
---|
321 | |
---|
322 | =head2 process_obs($obs) |
---|
323 | |
---|
324 | Process all data handle by $obs station |
---|
325 | |
---|
326 | =cut |
---|
327 | |
---|
328 | sub process_obs { |
---|
329 | my ($self, $obs) = @_; |
---|
330 | my %datadir = $self->list_obsdatadir($obs); |
---|
331 | $self->logging(0, "Starting %s() for %s", (caller(0))[3], $obs); |
---|
332 | |
---|
333 | foreach my $datatype (keys %datadir) { |
---|
334 | my $or = $self->get_obs_data_handle($obs, $datatype); |
---|
335 | $or or next; |
---|
336 | $or->process; |
---|
337 | } |
---|
338 | } |
---|
339 | |
---|
340 | sub processed { |
---|
341 | my ($self, $obs, $archive, $datafile, $datatype, $dest) = @_; |
---|
342 | push(@{$self->{processed_lists}}, |
---|
343 | { |
---|
344 | obs => $obs, |
---|
345 | archive => $archive, |
---|
346 | datafile => $datafile, |
---|
347 | datatype => $datatype, |
---|
348 | destfile => $dest, |
---|
349 | } |
---|
350 | ); |
---|
351 | $self->logging( |
---|
352 | 2, "Extraction of %s/%s done as %s", |
---|
353 | $archive, |
---|
354 | $datafile, |
---|
355 | $dest, |
---|
356 | ); |
---|
357 | } |
---|
358 | |
---|
359 | sub generated_reported { |
---|
360 | my ($self) = @_; |
---|
361 | my $result = { |
---|
362 | all => [], |
---|
363 | users => {}, |
---|
364 | }; |
---|
365 | |
---|
366 | foreach my $entry (@{$self->{processed_lists}}) { |
---|
367 | my @people = split(/\s*,\s*/, $self->getvalue($entry->{datatype}, 'reportto') || ""); |
---|
368 | @people or next; |
---|
369 | foreach my $p (@people) { |
---|
370 | push(@{$result->{users}{$p}{$entry->{datatype}}}, $entry); |
---|
371 | } |
---|
372 | } |
---|
373 | |
---|
374 | foreach my $p (keys(%{$result->{users} || {}})) { |
---|
375 | my %msg = ( |
---|
376 | Subject => 'ObsData Report', |
---|
377 | To => $p, |
---|
378 | 'X-ObsData-Version' => $VERSION, |
---|
379 | 'Content-Type' => "TEXT/PLAIN;\n charset=ISO-8859-1", |
---|
380 | 'Content-Transfer-Encoding' => 'QUOTED-PRINTABLE', |
---|
381 | From => 'ObsData <robot@aerov.jussieu.fr>', |
---|
382 | ); |
---|
383 | my $message = ""; |
---|
384 | foreach my $d (keys(%{$result->{users}{$p} || {}})) { |
---|
385 | $message .= "\nDataType: $d\n"; |
---|
386 | foreach (@{$result->{users}{$p}{$d} || []}) { |
---|
387 | $message .= sprintf("\tfile %s from %s\n\t=> %s\n", |
---|
388 | $_->{datafile}, |
---|
389 | $_->{archive}, |
---|
390 | $_->{destfile}, |
---|
391 | ); |
---|
392 | } |
---|
393 | } |
---|
394 | sendmail( |
---|
395 | %msg, |
---|
396 | smtp => 'mailhost', |
---|
397 | Message => encode_qp($message), |
---|
398 | ) or $self->log(4, "Cannot send mail to %s: %s", |
---|
399 | $msg{To}, $Mail::Sendmail::error, |
---|
400 | ); |
---|
401 | print "OK. Log says:\n", $Mail::Sendmail::log; |
---|
402 | } |
---|
403 | return $result; |
---|
404 | } |
---|
405 | |
---|
406 | 1; |
---|
407 | |
---|
408 | =head1 AUTHOR |
---|
409 | |
---|
410 | Olivier Thauvin <olivier.thauvin@aerov.jussieu.fr> |
---|
411 | |
---|
412 | =cut |
---|