[Chicago-talk] Script comments
Young, Darren
Darren.Young at ChicagoGSB.edu
Tue Jan 17 11:13:43 PST 2006
Finally have come close to finishing my parallel backup script. Thanks
for the Parallel::ForkManager idea, works great. Love to get some review
and comments on the script from those that are better than me at this
(before it gets shoved into production).
Thanks in advance,
>>>
1 #!/usr/local/bin/perl
2 #
========================================================================
======
3 #
4 # backup.pl
5 #
6 # Perform backup of the mailstore(s) in parallel
7 #
8 # $Id: backup.pl,v 1.6 2006/01/17 19:04:04 dyoung2 Exp $
9 #
10 #
========================================================================
======
11 #
12 # ChangeLog
13 #
14 # $Log: backup.pl,v $
15 # Revision 1.6 2006/01/17 19:04:04 dyoung2
16 # * Cleaned up tabs (converted to whitespace)
17 #
18 # Revision 1.5 2006/01/17 18:54:25 dyoung2
19 # * Removed constants.
20 # * Added --smtphost and --smtpfrom options.
21 #
22 # Revision 1.4 2006/01/17 18:45:44 dyoung2
23 # * Added sending of report email.
24 #
25 # Revision 1.3 2006/01/17 17:50:29 dyoung2
26 # * Added stats tracking.
27 # * Added --testmode.
28 # * Added dup of STDIN, STDOUT and STDERR to the logfile.
29 #
30 # Revision 1.2 2006/01/10 21:08:58 dyoung2
31 # * First version for testing.
32 #
33 # Revision 1.1 2006/01/09 20:38:17 dyoung2
34 # * Initial version.
35 #
36 #
========================================================================
======
37 #
38 # Notes:
39 #
40 # - Doesn't account for multiple instances of message server on
the same box.
41 # - Need to check the backup-groups file to ensure groups exist
there.
42 #
43 #
========================================================================
======
44
45 my $cvsid = '$Id: backup.pl,v 1.6 2006/01/17 19:04:04 dyoung2
Exp $';
46 my @cvsinfo = split( ' ', $cvsid );
47 our $NAME = File::Basename::basename($0);
48 our $VERSION = $cvsinfo[2];
49
50
51 #
------------------------------------------------------------------------
------
52 # B E G I N
53 #
------------------------------------------------------------------------
------
54
55 # Pragmas
56 use strict;
57
58 # "Standard" modules we use
59 use FindBin;
60 use Getopt::Long qw(GetOptions);
61 use File::Basename ();
62 use Sys::Hostname;
63 use Mail::Sender;
64 use Parallel::ForkManager;
65 use File::stat;
66
67 use POSIX qw(strftime);
68
69
70 #
------------------------------------------------------------------------
------
71 # G L O B A L V A R I A B L E S
72 #
------------------------------------------------------------------------
------
73 my $DEBUG = 0;
74 my $DEBUG_LEVEL = 1;
75 my $VERBOSE = 0;
76 my $LOGFILE = "$FindBin::Bin/$NAME.log";
77
78 my $TESTMODE = 0;
79 my $TYPE = "full";
80 my $REPTDEST = "systems@\lists.chicagogsb.edu";
81 my $SMTPHOST = "localhost";
82 my $SMTPFROM = "IMS_Backup";
83
84 my $STARTLETTER = "A";
85 my $ENDLETTER = "Z";
86
87 my $REGFILE = "/etc/msgregistry.inf";
88 my $HOSTNAME = hostname;
89
90 my $IMSROOT = get_ims_root($REGFILE);
91 my $BKPFILE = $IMSROOT . "/" . "msg-$HOSTNAME" . "/" .
"backup-groups.conf";
92 my $IMSBKPCMD = $IMSROOT . "/bin/msg/store/bin/imsbackup";
93
94 my $BKPDEST = "/mnt/imsbkvol";
95
96 my $NUMPROCS = 4;
97
98
99 # used for statistics
100 my %JOB_START;
101 my %JOB_END;
102 my %JOB_SIZE;
103
104
105 #
------------------------------------------------------------------------
------
106 # C O M M A N D L I N E O P T I O N S
107 #
------------------------------------------------------------------------
------
108
109 our $USAGE = "
110 Usage:
111
112 $NAME <options>
113 - Perform backup of the mailstore(s) in parallel
114
115 Options:
116 --help : this help screen
117 --version : prints the version
118
119 --debug : enables debugging (default off)
120 --debug_level : debug level (1-4) (default: $DEBUG_LEVEL
121 --verbose : passes a verbose flag to all commands
called
122
123 --logfile : location of the logfile (default: $LOGFILE)
124
125 --testmode : only run in test, don't actually do
anything (default off)
126
127 --type : type of backup
128 valid options are: full (the default) or
incr (incremental)
129
130 --numprocs : number of children to keep running
(default: $NUMPROCS)
131 --bkpdest : destination for the backup files (default:
$BKPDEST)
132
133 --startletter : group letter to start with (default:
$STARTLETTER)
134 --endletter : group letter to end with (default:
$ENDLETTER)
135
136 --smtphost : host to use for sending report email
(default: $SMTPHOST)
137 --smtpfrom : from address for report email (default:
$SMTPHOST)
138
139 ";
140
141
142 GetOptions(
143 "help" => sub { print_usage(); },
144 "version" => sub { print_version(); },
145
146 "debug" => \$DEBUG,
147 "debug_level" => \$DEBUG_LEVEL,
148 "verbose" => \$VERBOSE,
149
150 "logfile=s" => \$LOGFILE,
151
152 "testmode" => \$TESTMODE,
153
154 "type=s" => \$TYPE,
155
156 "numprocs=i" => \$NUMPROCS,
157 "bkpdest=s" => \$BKPDEST,
158
159 "startletter=s" => \$STARTLETTER,
160 "endletter=s" => \$ENDLETTER,
161
162 "smtphost=s" => \$SMTPHOST,
163 "smtpfrom=s" => \$SMTPFROM
164 );
165
166
167 #
------------------------------------------------------------------------
------
168 # V A R I A B L E S
169 #
------------------------------------------------------------------------
------
170
171 # upper case the start and end letters
172 $STARTLETTER = uc($STARTLETTER);
173 $ENDLETTER = uc($ENDLETTER);
174
175 my $name = "main()";
176
177 # the group letters to run imsbackup against
178 # what happens if ENDLETTER comes *before* STARTLETTER (e.g. type)
179 my @letters = ("$STARTLETTER" .. "$ENDLETTER");
180
181
182 #
------------------------------------------------------------------------
------
183 # M A I N L O O P
184 #
------------------------------------------------------------------------
------
185
186 # Flush everything
187 select(STDIN); $| = 1;
188 select(STDERR); $| = 1;
189 select(STDOUT); $| = 1;
190
191 unless ( $TESTMODE ) {
192 open( STDOUT, ">$LOGFILE" ) or die "Unable to dup STDOUT:
($!)\n";
193 open( STDIN, "/dev/null" ) or die "Can't read /dev/null:
($!)";
194 open( STDERR, ">>&STDOUT" ) or die "Can't dup stdout: ($!)";
195 }
196
197 if ( $TESTMODE ) {
198 logmsg("$name: ** Running in test mode **");
199 }
200
201 # Avoid broken pipe messages
202 $SIG{PIPE} = 'IGNORE';
203
204 logmsg("$NAME: version $VERSION starting");
205 logmsg("$name: performing a $TYPE backup");
206
207 # dump our settings if debug was requested
208 debug("$name: START -> $STARTLETTER");
209 debug("$name: END -> $ENDLETTER");
210 debug("$name: IMSROOT -> $IMSROOT");
211 debug("$name: BKPFILE -> $BKPFILE");
212 debug("$name: BKPCMD -> $IMSBKPCMD");
213 debug("$name: NUMPROCS -> $NUMPROCS");
214 debug("$name: BKPDEST -> $BKPDEST");
215 debug("$name: SMTPHOST -> $SMTPHOST");
216 debug("$name: SMTPFROM -> $SMTPFROM");
217
218
219
220 # create the fork manager object
221 my $forker = new Parallel::ForkManager($NUMPROCS);
222
223 # callback for each child start
224 $forker->run_on_start( sub {
225 my ($pid, $group) = @_;
226 logmsg("$group: Started $group,
pid = $pid");
227 $JOB_START{$group} = time();
228 });
229
230 # callback for each child finish
231 $forker->run_on_finish( sub {
232 my ($pid, $exit_code, $group) =
@_;
233 logmsg("$group: Finished $group,
pid = $pid, exit code = $exit_code");
234 $JOB_END{$group} = time();
235
236 # if the dump file is present save
the file size
237 # otherwise use 0 as the size
(file isn't there)
238 if ( -f "$BKPDEST/$group" ) {
239 my $f =
stat("$BKPDEST/$group");
240 $JOB_SIZE{$group} = $f->size;
241 logmsg("$group: size => " .
$f->size);
242 } else {
243 $JOB_SIZE{$group} = 0;
244 }
245 });
246
247
248
249 ###
250 # Start children, keep specified number of children running
251 ###
252 foreach my $letter (@letters) {
253 my $group = "group" . $letter;
254 logmsg("forker(): working on letter $letter");
255
256 my $pid = $forker->start($group) and next;
257
258 my $cmd;
259 $cmd = $IMSBKPCMD;
260 $cmd .= " " . "-f-";
261 $cmd .= " " . "/$HOSTNAME/$group";
262 $cmd .= " " . ">";
263 $cmd .= " " . "$BKPDEST" . "/" . $group;
264
265 debug("$name: $cmd");
266
267 # only run the system() call outside of testmode
268 if ( ! $TESTMODE ) {
269 $rc = system($cmd);
270 $rc = $rc / 256;
271 } else {
272 $rc = 0;
273 }
274
275 logmsg("forker(): system call returned : $rc");
276
277 $forker->finish($rc, $group); # closes process
278 }
279
280
281 # wait for all children to complete (avoids zombies)
282 logmsg("$name: waiting for all children to finish");
283 $forker->wait_all_children();
284
285
286 ###
287 # Print statistics for each group
288 ###
289 logmsg("$name: all done, dumping stats\n");
290 foreach my $letter (@letters) {
291 my $group = "group" . $letter;
292
293 my $start = strftime "%m-0.000000e+00-%Y %H:%M:%S",
localtime($JOB_START{$group});
294 my $end = strftime "%m-0.000000e+00-%Y %H:%M:%S",
localtime($JOB_END{$group});
295 my $elapsed = $JOB_END{$group} = $JOB_START{$group};
296
297 print "$group " . $start . " " . $end . " " .
$JOB_SIZE{$group} . "\n";
298 }
299 print "\n";
300
301
302
303 ###
304 # Generate and send report email
305 ###
306 logmsg("$name: Sending report email");
307 open(LOGFILE, "<$LOGFILE") or die "Unable to open logfile $LOGFILE
for read ($!)\n";
308
309 if ( $DEBUG ) {
310 $sender = new Mail::Sender {
311 smtp => "$SMTPHOST",
312 from => "$SMTPFROM",
313 debug => \*STDOUT,
314 debug_level => $DEBUG_LEVEL,
315 } or die $Mail::Sender::Error . "\n";
316 } else {
317 $sender = new Mail::Sender {
318 smtp => "$SMTPHOST",
319 from => "$SMTPFROM",
320 } or die $Mail::Sender::Error . "\n";
321 }
322
323 logmsg("$name: opening smtp connection to $SMTPHOST");
324 $sender->Open ({
325 to => 'darren.young at gsb.uchicago.edu',
326 subject => "IMS backup report for " .
localtime(time()),
327 }) or die $Mail::Sender::Error . "\n";
328
329 while(<LOGFILE>) {
330 $sender->SendEx($_);
331 };
332
333 $sender->Close();
334 close(LOGFILE);
335
336
337 # all done
338 exit(0);
339
340
341
342
343 #
------------------------------------------------------------------------
------
344 # F U N C T I O N S
345 #
------------------------------------------------------------------------
------
346
347 sub print_version {
348 print "$NAME version $VERSION\n";
349 exit(0);
350 }
351
352 sub print_usage {
353 print "$USAGE";
354 exit(0);
355 }
356
357 sub logmsg {
358 my $message = shift;
359 if ( ! $message ) {
360 print "logmsg: you must supply a message!\n";
361 exit(127);
362 }
363 my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
localtime(time);
364
365 $mon += 1;
366 $year += 1900;
367 $mon = sprintf("00", $mon);
368 $mday = sprintf("00", $mday);
369 $hour = sprintf("00", $hour);
370 $min = sprintf("00", $min);
371 $sec = sprintf("00", $sec);
372
373 print "$mon-$mday-$year $hour:$min:$sec: $message\n";
374
375 return(1);
376 }
377
378 sub debug {
379 my $message = shift;
380 if ( $DEBUG ) {
381 logmsg("DEBUG: $message");
382 return(1);
383 } else {
384 return(1);
385 }
386 }
387
388 sub get_ims_root {
389 my ($regfile) = @_;
390 my $rootpath = "";
391
392 open(REGFILE, "<$regfile") or die "Failed to open $regfile
($!)\n";
393 while(<REGFILE>) {
394 next if /^$/;
395 chomp;
396 if ( /RootPath/ ) {
397 ($key, $rootpath) = split('=', $_);
398 $rootpath =~ s/^\s+//;
399 $rootpath =~ s/\s+$//;
400 }
401 }
402 close(REGFILE);
403
404 return($rootpath);
405 }
406
407 __END__
More information about the Chicago-talk
mailing list