INN commit: trunk (4 files)
INN Commit
Russ_Allbery at isc.org
Wed Nov 22 08:39:05 UTC 2006
Date: Wednesday, November 22, 2006 @ 00:39:04
Author: eagle
Revision: 7592
Track statistics separately for every connection in innreport and only
sum them together at the end of log processing. This allows us to also
track checkpoints and add checkpoint numbers to the totals at the end
of log processing if no intervening close was seen. Based on work by
Jim Dutton and Julien ?\195?\137LIE.
Modified:
trunk/CONTRIBUTORS
trunk/samples/innreport.conf.in
trunk/scripts/innreport.in
trunk/scripts/innreport_inn.pm
---------------------------+
CONTRIBUTORS | 2
samples/innreport.conf.in | 122 ++++++++++++++++++------------------
scripts/innreport.in | 4 -
scripts/innreport_inn.pm | 146 ++++++++++++++++++++++++++++++++++----------
4 files changed, 179 insertions(+), 95 deletions(-)
Modified: CONTRIBUTORS
===================================================================
--- CONTRIBUTORS 2006-11-22 07:20:46 UTC (rev 7591)
+++ CONTRIBUTORS 2006-11-22 08:39:04 UTC (rev 7592)
@@ -261,4 +261,4 @@
Christopher P. Lindsey, Winfried Szukalski, Edvard Tuinder,
Frank McConnell, Ilya Kovalenko, Steve Youngs, Jacek Konieczny,
Ilya Voronin, Sergey Babitch, WATANABE Katsuhiro, F. Senault,
-Chris Caputo, Thomas Parleman, Adam J. Richter
+Chris Caputo, Thomas Parleman, Adam J. Richter, Jim Dutton, Julien ÉLIE
Modified: samples/innreport.conf.in
===================================================================
--- samples/innreport.conf.in 2006-11-22 07:20:46 UTC (rev 7591)
+++ samples/innreport.conf.in 2006-11-22 08:39:04 UTC (rev 7592)
@@ -1,12 +1,12 @@
-##########################################################
+######################################################################
# Configuration file for innreport (3.*).
#
# Sample file for INN.
# Tested with INN 2.3, 2.1, 1.7.2 and 1.5.1.
#
# (c) 1997, 1998, 1999 by Fabien Tassin <fta at sofaraway.org>
-# version 3.0.2
-##########################################################
+# version 3.0.5
+######################################################################
# Default parameters
section default {
@@ -508,8 +508,8 @@
section innd_connect {
title "Incoming Feeds (INN):";
- data "%innd_seconds";
- sort "$innd_accepted{$b} <=> $innd_accepted{$a}";
+ data "%innd_seconds_sum";
+ sort "$innd_accepted_sum{$b} <=> $innd_accepted_sum{$a}";
numbering true;
column {
name "Server";
@@ -523,79 +523,81 @@
name "Connects";
format_name "%5s";
format "%5d";
- value "$innd_connect{$key}";
- total "total(%innd_connect)";
+ value "$innd_connect_sum{$key}";
+ total "total(%innd_connect_sum)";
};
column {
name "Offered";
format_name "%8s";
format "%8d";
- value "$innd_offered{$key}";
- total "total(%innd_offered)";
+ value "$innd_offered_sum{$key}";
+ total "total(%innd_offered_sum)";
};
column {
name "Taken";
format_name "%7s";
format "%7d";
- value "$innd_accepted{$key}";
- total "total(%innd_accepted)";
+ value "$innd_accepted_sum{$key}";
+ total "total(%innd_accepted_sum)";
};
column {
name "Refused";
format_name "%7s";
format "%7d";
- value "$innd_refused{$key}";
- total "total(%innd_refused)";
+ value "$innd_refused_sum{$key}";
+ total "total(%innd_refused_sum)";
};
column {
name "Reject";
format_name "%7s";
format "%7d";
- value "$innd_rejected{$key}";
- total "total(%innd_rejected)";
+ value "$innd_rejected_sum{$key}";
+ total "total(%innd_rejected_sum)";
};
column {
name "%Accpt";
format_name "%6s";
format "%4d%%";
- value "$innd_offered{$key} == 0 ? 0 :
- $innd_accepted{$key} / $innd_offered{$key} * 100";
- total "total(%innd_offered) == 0 ? 0 :
- total(%innd_accepted) / total(%innd_offered) * 100";
+ value "$innd_offered_sum{$key} == 0 ? 0 :
+ ($innd_accepted_sum{$key} /
+ $innd_offered_sum{$key} * 100)";
+ total "total(%innd_offered_sum) == 0 ? 0 :
+ (total(%innd_accepted_sum) /
+ total(%innd_offered_sum)) * 100";
};
column {
name "Elapsed";
format_name "%8s";
format "%9s";
- value "time($innd_seconds{$key})";
- total "time(total(%innd_seconds))";
+ value "time($innd_seconds_sum{$key})";
+ total "time(total(%innd_seconds_sum))";
};
graph {
title "Articles received by server";
type histo3d;
- sort "%innd_accepted";
+ sort "%innd_accepted_sum";
data {
name "Articles accepted";
color "#0000FF";
- value "%innd_accepted";
+ value "%innd_accepted_sum";
};
data {
name "Articles refused";
color "#FFAF00";
- value "%innd_refused";
+ value "%innd_refused_sum";
};
data {
name "Articles rejected";
color "#FF0000";
- value "%innd_rejected";
+ value "%innd_rejected_sum";
};
};
};
section innd_incoming_vol {
title "Incoming Volume (INN):";
- data "%innd_seconds";
- sort "$innd_stored_size{$b} <=> $innd_stored_size{$a}";
+ data "%innd_seconds_sum";
+ sort "$innd_stored_size_sum{$b} <=> $innd_stored_size_sum{$a}";
numbering true;
column {
name "Server";
@@ -607,78 +609,80 @@
column {
name "AcceptVol";
format "%9s";
- value "bytes($innd_stored_size{$key})";
- total "bytes(total(%innd_stored_size))";
+ value "bytes($innd_stored_size_sum{$key})";
+ total "bytes(total(%innd_stored_size_sum))";
};
column {
name "DupVol";
format "%9s";
- value "bytes($innd_duplicated_size{$key})";
- total "bytes(total(%innd_duplicated_size))";
+ value "bytes($innd_duplicated_size_sum{$key})";
+ total "bytes(total(%innd_duplicated_size_sum))";
};
column {
name "RejVol";
format "%9s";
- value "bytes($innd_rejected_size{$key})";
- total "bytes(total(%innd_rejected_size))";
+ value "bytes($innd_rejected_size_sum{$key})";
+ total "bytes(total(%innd_rejected_size_sum))";
};
column {
name "TotalVol";
format "%9s";
- value "bytes($innd_stored_size{$key} +
- $innd_duplicated_size{$key} +
- $innd_rejected_size{$key})";
- total "bytes(total(%innd_stored_size) +
- total(%innd_duplicated_size) +
- total(%innd_rejected_size))";
+ value "bytes($innd_stored_size_sum{$key} +
+ $innd_duplicated_size_sum{$key} +
+ $innd_rejected_size_sum{$key})";
+ total "bytes(total(%innd_stored_size_sum) +
+ total(%innd_duplicated_size_sum) +
+ total(%innd_rejected_size_sum))";
};
column {
name "%Acc";
format_name "%4s";
format "%3d%%";
- value "$innd_offered_size{$key} == 0 ? 0 :
- $innd_stored_size{$key} / $innd_offered_size{$key} * 100";
- total "total(%innd_offered_size) == 0 ? 0 :
- total(%innd_stored_size) / total(%innd_offered_size) * 100";
+ value "$innd_offered_size_sum{$key} == 0 ? 0 :
+ $innd_stored_size_sum{$key}
+ / $innd_offered_size_sum{$key} * 100";
+ total "total(%innd_offered_size_sum) == 0 ? 0 :
+ (total(%innd_stored_size_sum) /
+ total(%innd_offered_size_sum)) * 100";
};
column {
name "Vol/Art";
format "%9s";
- value "bytes(($innd_stored_size{$key} +
- $innd_duplicated_size{$key} +
- $innd_rejected_size{$key}) /
- ($innd_accepted{$key} +
- $innd_rejected{$key}))";
- total "bytes((total(%innd_stored_size) +
- total(%innd_duplicated_size) +
- total(%innd_rejected_size)) /
- (total(%innd_accepted) +
- total(%innd_rejected)))";
+ value "bytes(($innd_stored_size_sum{$key} +
+ $innd_duplicated_size_sum{$key} +
+ $innd_rejected_size_sum{$key}) /
+ ($innd_accepted_sum{$key} +
+ $innd_rejected_sum{$key}))";
+ total "bytes((total(%innd_stored_size_sum) +
+ total(%innd_duplicated_size_sum) +
+ total(%innd_rejected_size_sum)) /
+ (total(%innd_accepted_sum) +
+ total(%innd_rejected_sum)))";
};
column {
name "Elapsed";
format "%9s";
- value "time($innd_seconds{$key})";
- total "time(total(%innd_seconds))";
+ value "time($innd_seconds_sum{$key})";
+ total "time(total(%innd_seconds_sum))";
};
graph {
title "Incoming Volume received by server";
type histo3d;
- sort "%innd_stored_size";
+ sort "%innd_stored_size_sum";
data {
name "Accepted Volume";
color "#0000FF";
- value "%innd_stored_size";
+ value "%innd_stored_size_sum";
};
data {
name "Duplicated Volume";
color "#FFAF00";
- value "%innd_duplicated_size";
+ value "%innd_duplicated_size_sum";
};
data {
name "Rejected Volume";
color "#FF0000";
- value "%innd_rejected_size";
+ value "%innd_rejected_size_sum";
};
};
};
Modified: scripts/innreport.in
===================================================================
--- scripts/innreport.in 2006-11-22 07:20:46 UTC (rev 7591)
+++ scripts/innreport.in 2006-11-22 08:39:04 UTC (rev 7592)
@@ -6,7 +6,7 @@
# innreport: Perl script to summarize news log files
# (with optional HTML output and graphs).
#
-# version: 3.0.2
+# version: 3.0.5
#
# Copyright (c) 1996-1999, Fabien Tassin (fta at sofaraway.org).
#
@@ -143,7 +143,7 @@
## THERE'S NOTHING TO CHANGE AFTER THIS LINE ##
###############################################
-my $version = "3.0.2";
+my $version = "3.0.5";
my %output; # content of the configuration file.
my $DEBUG = 0; # set to 1 to verify the structure/content of the conf file.
my $start_time = time;
Modified: scripts/innreport_inn.pm
===================================================================
--- scripts/innreport_inn.pm 2006-11-22 07:20:46 UTC (rev 7591)
+++ scripts/innreport_inn.pm 2006-11-22 08:39:04 UTC (rev 7592)
@@ -1,11 +1,12 @@
-##########################################################
+##########################################################################
# INN module for innreport (3.*).
#
# Sample file tested with INN 2.3, 2.2, 1.7.2 and 1.5.1
#
# (c) 1997-1999 by Fabien Tassin <fta at sofaraway.org>
-# version 3.0.2
-##########################################################
+# version 3.0.5
+#
+##########################################################################
# TODO: add the map file.
@@ -94,6 +95,16 @@
}
$nnrpd_time_times = 0; # ...
+my %notice_state;
+my %notice_state_ever_closed;
+my %innd_cp_accepted;
+my %innd_cp_refused;
+my %innd_cp_rejected;
+my %innd_cp_seconds;
+my %innd_cp_stored_size;
+my %innd_cp_duplicated_size;
+my %innd_cp_rejected_size;
+
# collect: Used to collect the data.
sub collect {
my ($day, $hour, $prog, $res, $left, $CASE_SENSITIVE) = @_;
@@ -333,30 +344,73 @@
return 1;
}
# connected
- if ($left =~ /^(\S+) connected \d+/o) {
- my $server = $1;
+ #
+ # Record <server>:<channel> instead of just <server> as otherwise we may
+ # miss some persistent connection newsfeeds that in any given innreport
+ # reporting period may not record any connect entries. We'll accumulate
+ # these into totals at the end of processing.
+ if ($left =~ /^(\S+) connected (\d+)/o) {
+ my $server = "$1:$2";
$server = lc $server unless $CASE_SENSITIVE;
$innd_connect{$server}++;
return 1;
}
- # closed (with times)
- if ($left =~ /(\S+):\d+ closed seconds (\d+) accepted (\d+) refused (\d+) rejected (\d+) duplicate (\d+) accepted size (\d+) duplicate size (\d+) rejected size (\d+)$/o) {
- my ($server, $seconds, $accepted, $refused, $rejected, $duplicate, $accptsize, $dupsize, $rjctsize) =
- ($1, $2, $3, $4, $5, $6, $7, $8, $9);
+
+ # closed/checkpoint (with times)
+ #
+ # For checkpoints, store the current checkpoint numbers; if we still have
+ # checkpoint numbers left over at the end of the run, we'll add them to
+ # our totals. On a closed, clear any checkpoint numbers and add the close
+ # numbers to the totals. Basically, we want to ignore checkpoints unless
+ # we don't get a close before the end of the log.
+ if ($left =~ /(\S+:\d+) (checkpoint|closed) seconds (\d+) accepted (\d+) refused (\d+) rejected (\d+) duplicate (\d+) accepted size (\d+) duplicate size (\d+) rejected size (\d+)$/o) {
+ my ($server, $status, $seconds, $accepted, $refused, $rejected, $duplicate, $accptsize, $dupsize, $rjctsize) =
+ ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);
$server = lc $server unless $CASE_SENSITIVE;
- $innd_seconds{$server} += $seconds;
- $innd_accepted{$server} += $accepted;
- $innd_refused{$server} += $refused;
- $innd_rejected{$server} += $rejected;
- $innd_stored_size{$server} += $accptsize;
- $innd_duplicated_size{$server} += $dupsize;
- $innd_rejected_size{$server} += $rjctsize;
+ $notice_state{$server} = $status;
+
+ if ($status eq 'checkpoint') {
+ $innd_cp_accepted{$server} = $accepted;
+ $innd_cp_refused{$server} = $refused;
+ $innd_cp_rejected{$server} = $rejected;
+ $innd_cp_seconds{$server} = $seconds;
+ $innd_cp_stored_size{$server} = $accptsize;
+ $innd_cp_duplicated_size{$server} = $dupsize;
+ $innd_cp_rejected_size{$server} = $rjctsize;
+ elsif ($status eq "closed") {
+ $innd_cp_accepted{$server} = 0;
+ $innd_cp_refused{$server} = 0;
+ $innd_cp_rejected{$server} = 0;
+ $innd_cp_seconds{$server} = 0;
+ $innd_cp_stored_size{$server} = 0;
+ $innd_cp_duplicated_size{$server} = 0;
+ $innd_cp_rejected_size{$server} = 0;
+
+ $notice_state_ever_closed{$server} = "YES";
+
+ $innd_seconds{$server} += $seconds;
+ $innd_accepted{$server} += $accepted;
+ $innd_refused{$server} += $refused;
+ $innd_rejected{$server} += $rejected;
+ $innd_stored_size{$server} += $accptsize;
+ $innd_duplicated_size{$server} += $dupsize;
+ $innd_rejected_size{$server} += $rjctsize;
+ }
return 1;
- } elsif ($left =~ /(\S+):\d+ closed seconds (\d+) accepted (\d+) refused (\d+) rejected (\d+)$/o) {
- # closed (with times)
+ # closed (with times)
+ } elsif ($left =~ /(\S+:\d+) closed seconds (\d+) accepted (\d+) refused (\d+) rejected (\d+)$/o) {
my ($server, $seconds, $accepted, $refused, $rejected) =
($1, $2, $3, $4, $5);
$server = lc $server unless $CASE_SENSITIVE;
+
+ $innd_cp_accepted{$server} = 0;
+ $innd_cp_refused{$server} = 0;
+ $innd_cp_rejected{$server} = 0;
+ $innd_cp_seconds{$server} = 0;
+ $innd_cp_stored_size{$server} = 0;
+ $innd_cp_duplicated_size{$server} = 0;
+ $innd_cp_rejected_size{$server} = 0;
+
$innd_seconds{$server} += $seconds;
$innd_accepted{$server} += $accepted;
$innd_refused{$server} += $refused;
@@ -365,20 +419,6 @@
}
# closed (without times (?))
return 1 if $left =~ m/\S+ closed$/o;
- # checkpoint
- return 1 if $left =~ m/^\S+:\d+ checkpoint /o;
- # if ($left =~ /(\S+):\d+ checkpoint seconds (\d+) accepted (\d+)
- # refused (\d+) rejected (\d+)$/) {
- # # Skipped...
- # my ($server, $seconds, $accepted, $refused, $rejected) =
- # ($1, $2, $3, $4, $5);
- # $innd_seconds{$server} += $seconds;
- # $innd_accepted{$server} += $accepted;
- # $innd_refused{$server} += $refused;
- # $innd_rejected{$server} += $rejected;
- # return 1;
- # }
-
# flush
if ($left =~ /(\S+) flush$/o) {
$innd_control{"flush"}++;
@@ -1792,7 +1832,31 @@
# Fill some hashes
{
- my $key;
+ my ($key, $hostname, $channel);
+
+ # If there are any checkpoint counts left over at the end of the file,
+ # transfer them to the totals for that server.
+ #
+ # Also, since the checkpoint counts include entries for all server
+ # connections, check to see if any checkpoint server entries are not also
+ # in %innd_connect. Add any missing servers (persistant servers with no
+ # connected log lines) to %innd_connect so that incoming totals will be
+ # properly computed.
+ foreach $server (keys (%innd_cp_accepted)) {
+ if (! defined($innd_connect{$server})) {
+ $innd_connect{$server} = 0;
+ }
+ if ($notice_state{$server} eq "checkpoint") {
+ $innd_accepted{$server} += $innd_cp_accepted{$server};
+ $innd_refused{$server} += $innd_cp_refused{$server};
+ $innd_rejected{$server} += $innd_cp_rejected{$server};
+ $innd_seconds{$server} += $innd_cp_seconds{$server};
+ $innd_stored_size{$server} += $innd_cp_stored_size{$server};
+ $innd_duplicated_size{$server} += $innd_cp_duplicated_size{$server};
+ $innd_rejected_size{$server} += $innd_cp_rejected_size{$server};
+ }
+ }
+
foreach $key (keys (%innd_connect)) {
$innd_offered{$key} = ($innd_accepted{$key} || 0)
+ ($innd_refused{$key} || 0)
@@ -1801,6 +1865,22 @@
+ ($innd_duplicated_size{$key} || 0) + ($innd_rejected_size{$key} || 0);
}
+ # Sum all incoming traffic for each full server.
+ foreach $key (keys (%innd_connect)) {
+ ($hostname, $channel) = split(':', $key);
+ if (defined($innd_seconds_sum{$hostname})) {
+ $innd_seconds_sum{$hostname} += $innd_seconds{$key};
+ $innd_connect_sum{$hostname} += $innd_connect{$key};
+ $innd_offered_sum{$hostname} += $innd_offered{$key};
+ $innd_accepted_sum{$hostname} += $innd_accepted{$key};
+ $innd_refused_sum{$hostname} += $innd_refused{$key};
+ $innd_rejected_sum{$hostname} += $innd_rejected{$key};
+ $innd_stored_size_sum{$hostname} += $innd_stored_size{$key};
+ $innd_duplicated_size_sum{$hostname} += $innd_duplicated_size{$key};
+ $innd_offered_size_sum{$hostname} += $innd_offered_size{$key};
+ $innd_rejected_size_sum{$hostname} += $innd_rejected_size_sum{$key};
+ }
+ }
# adjust min/max of innd timer stats.
if (%innd_time_min) {
More information about the inn-committers
mailing list