<div dir="ltr">Thanks guys. I appreciate the input. I don't want to derail the list much though, as this is supposed to be more BIND than Cisco :)<div><br></div><div style>At this point my BIND installation seems to be stable, so we'll call it case closed. We do plan on replacing our firewalls in the near future, so hopefully we won't need to put much more effort into it. But again appreciate all the help and suggestions, it definitely pushed me in the right direction for finding the problem.</div>
<div style><br></div><div style>Jason</div><div class="gmail_extra"><br><br><div class="gmail_quote">On Wed, Mar 26, 2014 at 12:56 PM, CARTWRIGHT, CORY C <span dir="ltr"><<a href="mailto:cc3283@att.com" target="_blank">cc3283@att.com</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Here is a script I wrote to log and sent traps. I'm sure you'll have to make a lot of changes but hopefully it can help you get started monitoring the FWSM. You can use this as a template to expand upon.<br>
<br>
#!/usr/bin/perl<br>
<br>
use strict;<br>
use Expect;<br>
use Net::Telnet;<br>
use Data::Dumper;<br>
use POSIX qw(tzset);<br>
use Data::Dumper;<br>
use lib qw( /usr/local/rrdtool-1.2.13/lib/perl );<br>
use RRDs;<br>
use File::Copy;<br>
use Net::SNMP qw(:asn1);<br>
<br>
## quick fix for gathering codec data<br>
## not very robust !!!<br>
## author: Cory Cartwright <a href="mailto:corycartwright@sbcglobal.net">corycartwright@sbcglobal.net</a><br>
##<br>
## grab cisco FWSM cpu information for RRD graphing and SNMP trap generation<br>
##<br>
<br>
$ENV{TZ} = 'EDT';<br>
POSIX::tzset();<br>
<br>
my $createRRD = shift || 'false';<br>
<br>
my $host = "MY6500|7600 host";<br>
my $user = "router username";<br>
my $pass = "router passwd";<br>
my $fwUser = "FWSM username";<br>
my $fwPasswd = "FWSM password";<br>
my $comunity = "FWSM comunity string";<br>
my $monitor = 'trap monitor IP'; # source that set and sent the trap<br>
my @trapCatchers = qw(array of trap catchers);<br>
<br>
my $filename = "/var/voip/fwsm_logger.txt"; #dump file<br>
my $DBfile = '/var/voip/codecDump.csv';<br>
<br>
my $trapThreshold = '60'; #'60'; #five sec thresh > send trap%<br>
my $procThreshold = '30'; #'30' ; #threshhold before we capture sh proc<br>
<br>
my %meas_hash = ( 'fiveSec' => 'fiveSec',<br>
'oneMin' => 'oneMin',<br>
'fiveMin' => 'fiveMin',<br>
);<br>
my $rrd = '/usr/voip/bin/fwcpuRRD.rrd';<br>
<br>
if (! -e $rrd) { $createRRD = 'true'; }<br>
<br>
my $hashRef = doExec();<br>
<br>
if($hashRef->{'fiveSec'} >= $trapThreshold) {<br>
#send trap<br>
print "Sending trap\n";<br>
sendTrap($hashRef->{'fiveSec'});<br>
}<br>
<br>
createRRD($rrd,\%meas_hash) if($createRRD eq 'true');<br>
updateRRD($rrd,\%meas_hash,$hashRef);<br>
print " struct\n" . Dumper(%meas_hash);<br>
print " data\n" . Dumper($hashRef);<br>
copy($rrd,"/var/www/voipdata/fwcpuRRD.rrd");<br>
<br>
sub doExec {<br>
<br>
<br>
my $exp = new Expect;<br>
#$exp->log_stdout(1);<br>
$exp->log_file("$filename");<br>
<br>
my $command = "ssh -l $fwUser $host";<br>
<br>
$exp->spawn($command) or die "Could not spawn $command $!";<br>
<br>
my $string = qr/passwd/;<br>
my $return = $exp->expect(3, "$string");<br>
<br>
$exp->send("$pass\n");<br>
<br>
$return = $exp->expect(3, '7604-nh1>');<br>
$exp->send("session slot 3 pro 1\n");<br>
<br>
$return = $exp->expect(3, /Password:/);<br>
$exp->send("x1c2v3\n");<br>
<br>
$return = $exp->expect(3, 'sipsfw>');<br>
$exp->send("enable\n");<br>
$return = $exp->expect(3, "$string");<br>
$exp->send("$fwPasswd\n");<br>
<br>
$return = $exp->expect(3, 'sipsfw#');<br>
$exp->send("sh cpu\n");<br>
$exp->expect(2);<br>
my $cpu = $exp->before();<br>
$cpu = $exp->before();<br>
my %cpu = ();<br>
if($cpu =~ /\d\sseconds\s=\s(\d+)\%\;\s\d\sminute\:\s(\d+)\%\;\s\d\sminutes\:\s(\d+)\%/g) {<br>
$cpu{'fiveSec'} = $1;<br>
$cpu{'oneMin'} = $2;<br>
$cpu{'fiveMin'} = $3;<br>
print Dumper(%cpu);<br>
}<br>
if($cpu{'fiveSec'} >= $procThreshold) {<br>
my $timestamp = "\nBEGIN: TIME: " . time . " !! " . localtime(time) . "\n### CPU 5 sec " . $cpu{'fiveSec'} . "\n";<br>
$exp->print_log_file($timestamp);<br>
$exp->send("no pager\n");<br>
$exp->send("sh proc\n");<br>
$exp->send("sh conn\n");<br>
$exp->send("sh resource usage\n");<br>
$exp->expect(3,'sipsfw#');<br>
}<br>
$exp->send("exit\n"); #exit enable<br>
$exp->expect(1);<br>
$exp->send("exit\n"); #exit fw<br>
$exp->expect(1);<br>
$exp->send("exit\n"); #exit switch<br>
$exp->expect(1);<br>
$exp->print_log_file("\nEND\n");<br>
$exp->soft_close();<br>
<br>
return(\%cpu);<br>
} #end doExec<br>
<br>
<br>
sub updateRRD {<br>
my ($rrd,$meas_hashRef,$dataHashRef) = @_;<br>
my $epoc = time;<br>
my $data_string = '';<br>
foreach my $cust (sort keys %$meas_hashRef) {<br>
my $data = $$dataHashRef{$$meas_hashRef{$cust}} || 0;<br>
print "Cust $cust: $data \n";<br>
$data_string = $data_string . "$data:";<br>
}<br>
<br>
$data_string =~ s/:$//g;<br>
print "rrdtool update $rrd $epoc:$data_string\n";<br>
RRDs::updatev $rrd, $epoc .":" . $data_string;<br>
if (my $ERROR = RRDs::error) {<br>
warn "$0: unable to update $rrd : $ERROR";<br>
}<br>
} #end sub<br>
<br>
sub createRRD {<br>
my $starttime = time;<br>
my $step = (5 * 60);<br>
my ($rrd,$meas_hashRef) = @_;<br>
print Dumper($meas_hashRef);<br>
print "In createRRD: ($starttime,$rrd,$step,$meas_hashRef)\n";<br>
my $DS_string = "$rrd --start $starttime --step $step ";<br>
foreach(sort keys %{$meas_hashRef}) {<br>
print "Key: $_\n";<br>
$DS_string = $DS_string . "DS:$_:GAUGE:$step:U:U ";<br>
}<br>
$DS_string = $DS_string .<br>
"RRA:AVERAGE:0.5:1:3000 " .<br>
"RRA:MIN:0.5:1:3000 " .<br>
"RRA:MAX:0.5:1:3000 " .<br>
"RRA:LAST:0.5:1:3000 ";<br>
<br>
print "$DS_string\n";<br>
my $return = `/usr/voip/bin/rrdtool create $DS_string`;<br>
print $return;<br>
my $ERROR = RRDs::error;<br>
die "$0: unable to create $rrd : $ERROR" if $ERROR;<br>
<br>
}<br>
<br>
sub sendTrap {<br>
my $cpu = shift;<br>
foreach my $host (@trapCatchers) {<br>
my ($session, $error) = Net::SNMP->session(<br>
-hostname => $host,<br>
-community => $comunity,<br>
-version => 'snmpv1',<br>
-port => 162<br>
);<br>
<br>
if (!defined($session)) {<br>
printf("ERROR: %s.\n", $error);<br>
exit 1;<br>
}<br>
<br>
my $svSvcName = '1.3.6.1.4.1.77.1.2.3.1.1';<br>
my $message = "FWSM CPU TOO HIGH $cpu%";<br>
my @oids = ($svSvcName, OCTET_STRING, $message);<br>
#my @oids;<br>
my $result = $session->trap(<br>
-agentaddr => $monitor,<br>
-varbindlist => \@oids<br>
#-varbindlist => [$svSvcName, OCTET_STRING, "$message"]<br>
);<br>
<br>
if (!defined($result)) {<br>
printf("ERROR: %s.\n", $session->error);<br>
$session->close;<br>
<br>
exit 1;<br>
<br>
}<br>
<br>
$session->close;<br>
print "Sent Trap \"$message\" to $host\n";<br>
} #end foreach<br>
} #end sub<br>
<div class="im HOEnZb"><br></div></blockquote></div><div><br></div>-- <br><div>Jason K. Brandt</div><div>Systems Administrator</div><div><br></div>
</div></div>