1 | # $Id$ |
---|
2 | |
---|
3 | package ObsData; |
---|
4 | |
---|
5 | use strict; |
---|
6 | use warnings; |
---|
7 | use Config::IniFiles; |
---|
8 | use POSIX qw(strftime); |
---|
9 | use ObsData::Repository; |
---|
10 | |
---|
11 | my @loglevel = ( |
---|
12 | 'DEBUG', |
---|
13 | 'INFO', |
---|
14 | 'RESULT', |
---|
15 | 'WARNING', |
---|
16 | 'ERROR', |
---|
17 | 'FATAL', |
---|
18 | ); |
---|
19 | |
---|
20 | our $VERSION = "0.3.0"; |
---|
21 | our $CVSID = q$Id$; |
---|
22 | our $CVSREV = (q$Revision$ =~ /^Revision: (.*) $/)[0]; |
---|
23 | |
---|
24 | =head1 NAME |
---|
25 | |
---|
26 | ObsData - Main object to manage data files |
---|
27 | |
---|
28 | =head1 SYNOPSIS |
---|
29 | |
---|
30 | use ObsData; |
---|
31 | my $conf = "configfile"; |
---|
32 | my $obsdata = ObsData->new($conf); |
---|
33 | |
---|
34 | =head1 METHODS |
---|
35 | |
---|
36 | =head2 new($configfile) |
---|
37 | |
---|
38 | Create a new Obsdata object from $configfile |
---|
39 | |
---|
40 | =cut |
---|
41 | |
---|
42 | sub new { |
---|
43 | my ($class, $configfile, %options) = @_; |
---|
44 | my $obsdata = { |
---|
45 | config => new Config::IniFiles( |
---|
46 | -file => $configfile, |
---|
47 | -default => 'global', |
---|
48 | -allowcontinue => 1 |
---|
49 | ), |
---|
50 | verbose => defined($options{verbose}) ? $options{verbose} : 1, |
---|
51 | logcallback => $options{logcallback}, |
---|
52 | logfile => $options{logfile}, |
---|
53 | dry_run => $options{dry_run}, |
---|
54 | interactive_callback => $options{interactive_callback}, |
---|
55 | processed_lists => [], |
---|
56 | }; |
---|
57 | |
---|
58 | if (!($configfile && -f $configfile && -r _)) { |
---|
59 | return undef; |
---|
60 | } |
---|
61 | |
---|
62 | $obsdata->{config} or return undef; |
---|
63 | |
---|
64 | $obsdata->{logfile} ||= $obsdata->{config}->val('.', 'logfile') || 'obsdata.log'; |
---|
65 | |
---|
66 | bless($obsdata, $class); |
---|
67 | } |
---|
68 | |
---|
69 | sub DESTROY { |
---|
70 | my ($self) = @_; |
---|
71 | |
---|
72 | if ($self->{loghandle}) { |
---|
73 | close($self->{loghandle}); |
---|
74 | $self->{loghandle} = undef; |
---|
75 | } |
---|
76 | } |
---|
77 | |
---|
78 | =head2 load |
---|
79 | |
---|
80 | Prepare the object for usage |
---|
81 | |
---|
82 | =cut |
---|
83 | |
---|
84 | sub load { |
---|
85 | my ($self) = @_; |
---|
86 | |
---|
87 | if (!open($self->{loghandle}, ">> $self->{logfile}")) { |
---|
88 | $self->{loghandle} = undef; |
---|
89 | $self->logging(5, "Can't open log file %s, exiting", "$self->{logfile}"); |
---|
90 | return 0; |
---|
91 | } |
---|
92 | |
---|
93 | return 1; |
---|
94 | } |
---|
95 | |
---|
96 | =head2 logging($level, $message, ...) |
---|
97 | |
---|
98 | Pass a message into the logging process. |
---|
99 | |
---|
100 | $level is the level of message |
---|
101 | $message and or other arguments are printf format string |
---|
102 | |
---|
103 | =cut |
---|
104 | |
---|
105 | sub logging { |
---|
106 | my ($self, $level, $fmt, @val) = @_; |
---|
107 | my $msg = sprintf($fmt, @val); |
---|
108 | my $logh = $self->{loghandle}; |
---|
109 | if ($self->{logcallback}) { |
---|
110 | $self->{logcallback}->($level, $msg); |
---|
111 | } |
---|
112 | if($level >= 0 && $level >= $self->{verbose}) { |
---|
113 | if ($logh) { |
---|
114 | printf $logh |
---|
115 | "%-9s %s %s\n", |
---|
116 | sprintf("[%s]", $self->loglevel($level)), |
---|
117 | strftime("%b %d %H:%M:%S %Y", gmtime), |
---|
118 | $msg; |
---|
119 | } |
---|
120 | } |
---|
121 | $msg |
---|
122 | } |
---|
123 | |
---|
124 | =head2 loglevel($level) |
---|
125 | |
---|
126 | Return the text human readable log level associate to $level number |
---|
127 | |
---|
128 | =cut |
---|
129 | |
---|
130 | sub loglevel { |
---|
131 | my $l = pop(@_); |
---|
132 | defined($l) or $l = pop(@_); |
---|
133 | return $loglevel[ $l ] || "?????"; |
---|
134 | } |
---|
135 | |
---|
136 | =head2 checkconfig() |
---|
137 | |
---|
138 | Check the validity of information contains in configfile. |
---|
139 | |
---|
140 | Notice: This test does not include the syntaxe validity |
---|
141 | |
---|
142 | =cut |
---|
143 | |
---|
144 | sub checkconfig { |
---|
145 | my ($self) = @_; |
---|
146 | my $result = 1; |
---|
147 | foreach my $g ($self->{config}->GroupMembers('Obs')) { |
---|
148 | my ($obs) = $g =~ /\S+\s+(.*)/; |
---|
149 | if (!$self->{config}->SectionExists($obs)) { |
---|
150 | print STDERR "E: '$obs' is listed as Obs but it does not exists\n"; |
---|
151 | next; |
---|
152 | } |
---|
153 | foreach my $param ($self->{config}->Parameters($obs)) { |
---|
154 | } |
---|
155 | foreach my $datatype ($self->list_datatype) { |
---|
156 | foreach my $var (qw(match match_archive)) { |
---|
157 | my $regexp = $self->getvalue($datatype, $var) or next; # next ? are we sure ? |
---|
158 | eval { qr/$regexp/ }; # Many thanks Rafael |
---|
159 | if ($@) { |
---|
160 | $self->logging(4, "error in regexp for %s: '%s': %s", |
---|
161 | $datatype, |
---|
162 | $regexp, |
---|
163 | $@, |
---|
164 | ); |
---|
165 | $result = 0; |
---|
166 | # TODO set this reg unavalable |
---|
167 | } |
---|
168 | } |
---|
169 | } |
---|
170 | } |
---|
171 | return($result); |
---|
172 | } |
---|
173 | |
---|
174 | =head2 getvalue($section, $var, $default) |
---|
175 | |
---|
176 | Return a value from the configuration, $default is assumed if the value is not set |
---|
177 | |
---|
178 | =cut |
---|
179 | |
---|
180 | sub getvalue { |
---|
181 | my ($self, $section, $var, $default) = @_; |
---|
182 | $self->{config}->val($section, $var, $default); |
---|
183 | } |
---|
184 | |
---|
185 | =head2 config_mtime |
---|
186 | |
---|
187 | Return the modification time of config file currently used |
---|
188 | |
---|
189 | =cut |
---|
190 | |
---|
191 | sub config_mtime { |
---|
192 | my ($self) = @_; |
---|
193 | return $self->{configmtime} ||= (stat($self->{config}->GetFileName))[9]; |
---|
194 | } |
---|
195 | |
---|
196 | =head2 list_obs |
---|
197 | |
---|
198 | Return the list of observatories defined in configuration |
---|
199 | |
---|
200 | =cut |
---|
201 | |
---|
202 | sub list_obs { |
---|
203 | my ($self) = @_; |
---|
204 | grep { $self->{config}->SectionExists($_) } |
---|
205 | map { s/^\S+\s+//; $_ } |
---|
206 | $self->{config}->GroupMembers('Obs'); |
---|
207 | } |
---|
208 | |
---|
209 | =head2 is_obs($obsname) |
---|
210 | |
---|
211 | Return True if $obsname is an observatory |
---|
212 | |
---|
213 | =cut |
---|
214 | |
---|
215 | sub is_obs { |
---|
216 | my ($self, $obs) = @_; |
---|
217 | scalar(grep { $_ eq "Obs $obs" } $self->{config}->GroupMembers('Obs')); |
---|
218 | } |
---|
219 | |
---|
220 | =head2 list_obsdatadir($obsname) |
---|
221 | |
---|
222 | Return a hash of data directories per data type for $obsname observatories |
---|
223 | |
---|
224 | =cut |
---|
225 | |
---|
226 | sub list_obsdatadir { |
---|
227 | my ($self, $obs) = @_; |
---|
228 | $self->is_obs($obs) or return undef; |
---|
229 | map { m,^datadir/(.*),; ( ($1 || "") => $self->{config}->val($obs, $_) ) } |
---|
230 | grep { m,^datadir/, || $_ eq 'datadir' } |
---|
231 | $self->{config}->Parameters($obs) |
---|
232 | } |
---|
233 | |
---|
234 | =head2 list_typedatadir($type) |
---|
235 | |
---|
236 | List all directories for $type data |
---|
237 | |
---|
238 | =cut |
---|
239 | |
---|
240 | sub list_typedatadir { |
---|
241 | my ($self, $type) = @_; |
---|
242 | my %dirs; |
---|
243 | foreach my $obs ($self->list_obs) { |
---|
244 | $dirs{$_} = 1 foreach(grep { $_ } $self->get_datadir($obs, $type)); |
---|
245 | } |
---|
246 | keys %dirs; |
---|
247 | |
---|
248 | } |
---|
249 | |
---|
250 | =head2 get_datadir($obs, $type) |
---|
251 | |
---|
252 | Return a list of directories for $type data on $obs observatory |
---|
253 | |
---|
254 | =cut |
---|
255 | |
---|
256 | sub get_datadir { |
---|
257 | my ($self, $obs, $type) = @_; |
---|
258 | $self->is_obs($obs) or return undef; |
---|
259 | grep { defined($_) } ($self->getvalue($obs, "datadir/$type"), $self->getvalue($obs, "datadir")); |
---|
260 | } |
---|
261 | |
---|
262 | =head2 list_datatype |
---|
263 | |
---|
264 | Return a list of datatype handle by config |
---|
265 | |
---|
266 | =cut |
---|
267 | |
---|
268 | sub list_datatype { |
---|
269 | my ($self) = @_; |
---|
270 | grep { $_ ne 'global' } |
---|
271 | grep { $_ !~ /^Obs\s+/ } |
---|
272 | grep { !$self->is_obs($_) } $self->{config}->Sections; |
---|
273 | } |
---|
274 | |
---|
275 | =head2 get_obs_data_handle($obs, $datatype) |
---|
276 | |
---|
277 | Return an ObsData::Repository object about directory for $obs station |
---|
278 | about $datatype data. $datatype can be undefined if you want to get default |
---|
279 | directory |
---|
280 | |
---|
281 | =cut |
---|
282 | |
---|
283 | sub get_obs_data_handle { |
---|
284 | my ($self, $obs, $datatype) = @_; |
---|
285 | |
---|
286 | my $dir = $self->getvalue($obs, ($datatype ? "datadir/$datatype" : "datadir")); |
---|
287 | if (!$dir) { |
---|
288 | $self->logging(4, |
---|
289 | "Can't find data directory for %s, type: %s", |
---|
290 | $obs, $datatype || '(none)' |
---|
291 | ); |
---|
292 | return undef; |
---|
293 | } |
---|
294 | my $or = ObsData::Repository::dir->new( |
---|
295 | { |
---|
296 | obsdata => $self, |
---|
297 | dir => $dir, |
---|
298 | obs => $obs, |
---|
299 | datatype => $datatype, |
---|
300 | dry_run => $self->{dry_run}, # FIXME does this have a better place ? |
---|
301 | patern => $self->getvalue( |
---|
302 | $obs, |
---|
303 | ($datatype ? "searchfiles/$datatype" : "searchfiles") |
---|
304 | ), |
---|
305 | statusfile => $self->getvalue( |
---|
306 | $obs, |
---|
307 | ($datatype ? "index/$datatype" : "index"), |
---|
308 | "$dir/obsdata.ini" |
---|
309 | ), |
---|
310 | } |
---|
311 | ); |
---|
312 | if (!defined($or)) { |
---|
313 | $self->logging(4, "Can't parse %s, check directory exists", $dir); |
---|
314 | return undef; |
---|
315 | } |
---|
316 | |
---|
317 | return($or); |
---|
318 | } |
---|
319 | |
---|
320 | =head2 process_obs($obs) |
---|
321 | |
---|
322 | Process all data handle by $obs station |
---|
323 | |
---|
324 | =cut |
---|
325 | |
---|
326 | sub process_obs { |
---|
327 | my ($self, $obs) = @_; |
---|
328 | my %datadir = $self->list_obsdatadir($obs); |
---|
329 | $self->logging(0, "Starting %s() for %s", (caller(0))[3], $obs); |
---|
330 | |
---|
331 | foreach my $datatype (keys %datadir) { |
---|
332 | my $or = $self->get_obs_data_handle($obs, $datatype); |
---|
333 | $or or next; |
---|
334 | $or->process; |
---|
335 | } |
---|
336 | } |
---|
337 | |
---|
338 | sub processed { |
---|
339 | my ($self, $obs, $archive, $datatype, $dest) = @_; |
---|
340 | push(@{$self->{processed_lists}}, |
---|
341 | { |
---|
342 | obs => $obs, |
---|
343 | archive => $archive, |
---|
344 | datatype => $datatype, |
---|
345 | destfile => $dest, |
---|
346 | } |
---|
347 | ); |
---|
348 | } |
---|
349 | |
---|
350 | 1; |
---|
351 | |
---|
352 | =head1 AUTHOR |
---|
353 | |
---|
354 | Olivier Thauvin <olivier.thauvin@aerov.jussieu.fr> |
---|
355 | |
---|
356 | =cut |
---|