High recursive client counts

CARTWRIGHT, CORY C cc3283 at att.com
Wed Mar 26 17:56:22 UTC 2014


Here is a script I wrote to log and sent traps.  I'm sure you'll have to make a lot of changes but hopefully it can help you get started monitoring the FWSM.  You can use this as a template to expand upon.

#!/usr/bin/perl

use strict;
use Expect;
use Net::Telnet;
use Data::Dumper;
use POSIX qw(tzset);
use Data::Dumper;
use lib qw( /usr/local/rrdtool-1.2.13/lib/perl );
use RRDs;
use File::Copy;
use Net::SNMP qw(:asn1);

##  quick fix for gathering codec data
## not very robust !!!
## author: Cory Cartwright corycartwright at sbcglobal.net
##
## grab cisco FWSM cpu information for RRD graphing and SNMP trap generation
##

$ENV{TZ} = 'EDT';
POSIX::tzset();

my $createRRD = shift || 'false';

my $host = "MY6500|7600 host";
my $user = "router username";
my $pass = "router passwd";
my $fwUser = "FWSM username";
my $fwPasswd = "FWSM password";
my $comunity = "FWSM comunity string";
my $monitor = 'trap monitor IP';   # source that set and sent the trap
my @trapCatchers = qw(array of trap catchers);

my $filename = "/var/voip/fwsm_logger.txt"; #dump file
my $DBfile = '/var/voip/codecDump.csv';

my $trapThreshold = '60'; #'60'; #five sec thresh > send trap%
my $procThreshold = '30'; #'30' ; #threshhold before we capture sh proc

my %meas_hash = (       'fiveSec' => 'fiveSec',
                        'oneMin' => 'oneMin',
                        'fiveMin' => 'fiveMin',
                 );
my $rrd = '/usr/voip/bin/fwcpuRRD.rrd';

if (! -e $rrd) { $createRRD = 'true'; }

my $hashRef = doExec();

if($hashRef->{'fiveSec'} >= $trapThreshold) {
        #send trap
        print "Sending trap\n";
        sendTrap($hashRef->{'fiveSec'});
}

createRRD($rrd,\%meas_hash) if($createRRD eq 'true');
updateRRD($rrd,\%meas_hash,$hashRef);
print " struct\n" . Dumper(%meas_hash);
print "  data\n" . Dumper($hashRef);
copy($rrd,"/var/www/voipdata/fwcpuRRD.rrd");

sub doExec {


my $exp = new Expect;
#$exp->log_stdout(1);
$exp->log_file("$filename");

my $command = "ssh -l $fwUser $host";

$exp->spawn($command) or die "Could not spawn $command $!";

my $string = qr/passwd/;
my $return = $exp->expect(3, "$string");

$exp->send("$pass\n");

$return = $exp->expect(3, '7604-nh1>');
$exp->send("session slot 3 pro 1\n");

$return = $exp->expect(3, /Password:/);
$exp->send("x1c2v3\n");

$return = $exp->expect(3, 'sipsfw>');
$exp->send("enable\n");
$return = $exp->expect(3, "$string");
$exp->send("$fwPasswd\n");

$return = $exp->expect(3, 'sipsfw#');
$exp->send("sh cpu\n");
$exp->expect(2);
my $cpu = $exp->before();
$cpu = $exp->before();
my %cpu = ();
if($cpu =~ /\d\sseconds\s=\s(\d+)\%\;\s\d\sminute\:\s(\d+)\%\;\s\d\sminutes\:\s(\d+)\%/g) {
$cpu{'fiveSec'} = $1;
$cpu{'oneMin'} = $2;
$cpu{'fiveMin'} = $3;
print Dumper(%cpu);
}
if($cpu{'fiveSec'} >= $procThreshold) {
        my $timestamp = "\nBEGIN: TIME: " . time . " !! " . localtime(time) . "\n### CPU 5 sec " . $cpu{'fiveSec'} . "\n";
        $exp->print_log_file($timestamp);
        $exp->send("no pager\n");
        $exp->send("sh proc\n");
        $exp->send("sh conn\n");
        $exp->send("sh resource usage\n");
        $exp->expect(3,'sipsfw#');
}
$exp->send("exit\n"); #exit enable
$exp->expect(1);
$exp->send("exit\n"); #exit fw
$exp->expect(1);
$exp->send("exit\n"); #exit switch
$exp->expect(1);
$exp->print_log_file("\nEND\n");
$exp->soft_close();

return(\%cpu);
} #end doExec


sub updateRRD {
        my ($rrd,$meas_hashRef,$dataHashRef) = @_;
        my $epoc = time;
        my $data_string = '';
        foreach my $cust (sort keys %$meas_hashRef) {
                my $data = $$dataHashRef{$$meas_hashRef{$cust}} || 0;
               print "Cust $cust: $data \n";
                $data_string = $data_string . "$data:";
        }

        $data_string =~ s/:$//g;
        print "rrdtool update $rrd $epoc:$data_string\n";
        RRDs::updatev $rrd, $epoc .":" . $data_string;
                if (my $ERROR = RRDs::error) {
                        warn "$0: unable to update $rrd : $ERROR";
                }
} #end sub

sub createRRD {
        my $starttime = time;
        my $step = (5 * 60);
        my ($rrd,$meas_hashRef) = @_;
        print Dumper($meas_hashRef);
        print "In createRRD: ($starttime,$rrd,$step,$meas_hashRef)\n";
        my $DS_string = "$rrd --start $starttime --step $step ";
        foreach(sort keys %{$meas_hashRef}) {
                print "Key: $_\n";
                $DS_string = $DS_string . "DS:$_:GAUGE:$step:U:U ";
        }
        $DS_string = $DS_string .
                "RRA:AVERAGE:0.5:1:3000 " .
                "RRA:MIN:0.5:1:3000 " .
                "RRA:MAX:0.5:1:3000 " .
                "RRA:LAST:0.5:1:3000 ";

        print "$DS_string\n";
        my $return = `/usr/voip/bin/rrdtool create $DS_string`;
        print $return;
        my $ERROR = RRDs::error;
        die "$0: unable to create $rrd : $ERROR" if $ERROR;

}

sub sendTrap {
my $cpu = shift;
foreach my $host (@trapCatchers) {
my ($session, $error) = Net::SNMP->session(
                                           -hostname    => $host,
                                           -community   => $comunity,
                                           -version     => 'snmpv1',
                                           -port        => 162
);

if (!defined($session)) {
    printf("ERROR: %s.\n", $error);
    exit 1;
}

my $svSvcName = '1.3.6.1.4.1.77.1.2.3.1.1';
my $message = "FWSM CPU TOO HIGH $cpu%";
my @oids = ($svSvcName, OCTET_STRING, $message);
#my @oids;
        my $result = $session->trap(
                            -agentaddr    => $monitor,
                            -varbindlist  => \@oids
                            #-varbindlist  => [$svSvcName, OCTET_STRING, "$message"]
        );

        if (!defined($result)) {
                printf("ERROR: %s.\n", $session->error);
        $session->close;

        exit 1;

        }

        $session->close;
print "Sent Trap \"$message\" to $host\n";
} #end foreach
} #end sub

-----Original Message-----
From: bind-users-bounces+cc3283=att.com at lists.isc.org [mailto:bind-users-bounces+cc3283=att.com at lists.isc.org] On Behalf Of Sam Wilson
Sent: Wednesday, March 26, 2014 1:29 PM
To: comp-protocols-dns-bind at isc.org
Subject: Re: High recursive client counts

In article <mailman.2540.1395835774.20661.bind-users at lists.isc.org>,
 Jason Brandt <jbrandt at fsmail.bradley.edu> wrote:

> The code on our FWSMs isn't the latest release, so that could be part of
> the issue, but it's been about 16 hours now since I shut it off, and so far
> so good.  I would say though with the other load on our firewalls, it's
> highly possible that they were being overloaded.  Unfortunately our MRTG
> isn't setup to track firewall CPU, so I can't say for sure.

Logging into your FWSM and doing 'show cpu usage' when things are going 
badly might be an option, but if you've got MRTG monitoring the 6500 
that the FWSM is in you could also have a look at the traffic on the 
virtual ethernets that connect to the FWSM.  Whilst they don't show up 
on 'show int' they and a 6 Gbps portchannel are visible to SNMP and in 
'show firewall module X traffic' (or 'show firewall switch X module Y 
traffic' in a VSS setup)..

Sam

-- 
The University of Edinburgh is a charitable body, registered in
Scotland, with registration number SC005336.
_______________________________________________
Please visit https://lists.isc.org/mailman/listinfo/bind-users to unsubscribe from this list

bind-users mailing list
bind-users at lists.isc.org
https://lists.isc.org/mailman/listinfo/bind-users


More information about the bind-users mailing list