INN commit: branches/2.5 (8 files)
INN Commit
rra at isc.org
Sun Sep 4 09:22:23 UTC 2011
Date: Sunday, September 4, 2011 @ 02:22:23
Author: iulius
Revision: 9372
add documentation for procbatch
New documentation written by Florian Schlichting, with the help of a memo
by Russ Allbery.
Add a link to this new documentation in innfeed and news.daily documentation.
Use "$INN::Config::pathtmp" instead of "INN::Config::$spooltmp" in procbatch. (The two variables are the same; the first one is more wide-spread.)
Add a warning if -s is used without -c, and if -t is used without -m.
Added:
branches/2.5/doc/pod/procbatch.pod
(from rev 9371, trunk/doc/pod/procbatch.pod)
Modified:
branches/2.5/MANIFEST
branches/2.5/doc/man/ (properties)
branches/2.5/doc/man/Makefile
branches/2.5/doc/pod/Makefile
branches/2.5/doc/pod/innfeed.pod
branches/2.5/doc/pod/news.daily.pod
branches/2.5/innfeed/procbatch.in
------------------------+
MANIFEST | 2
doc/man/Makefile | 2
doc/pod/Makefile | 3
doc/pod/innfeed.pod | 9 +-
doc/pod/news.daily.pod | 2
doc/pod/procbatch.pod | 144 +++++++++++++++++++++++++++++++++++++++++++++++
innfeed/procbatch.in | 39 ++++++------
7 files changed, 175 insertions(+), 26 deletions(-)
Modified: MANIFEST
===================================================================
--- MANIFEST 2011-09-04 09:21:43 UTC (rev 9371)
+++ MANIFEST 2011-09-04 09:22:23 UTC (rev 9372)
@@ -200,6 +200,7 @@
doc/man/passwd.nntp.5 Manpage for passwd.nntp config file
doc/man/perl-nocem.8 Manpage for perl-nocem
doc/man/pgpverify.1 Manpage for pgpverify
+doc/man/procbatch.8 Manpage for procbatch
doc/man/prunehistory.8 Manpage for prunehistory
doc/man/pullnews.1 Manpage for pullnews
doc/man/qio.3 Manpage for fast I/O file routines
@@ -301,6 +302,7 @@
doc/pod/ovdb_stat.pod Master file for ovdb_stat.8
doc/pod/overchan.pod Master file for overchan.8
doc/pod/passwd.nntp.pod Master file for passwd.nntp.5
+doc/pod/procbatch.pod Master file for procbatch.8
doc/pod/prunehistory.pod Master file for prunehistory.8
doc/pod/pullnews.pod Master file for pullnews.1
doc/pod/qio.pod Master file for qio.3
Property changes on: branches/2.5/doc/man
___________________________________________________________________
Modified: svn:ignore
- active.5
active.times.5
actsync.8
archive.8
auth_krb5.8
batcher.8
buffchan.8
buffindexed.conf.5
ckpasswd.8
cnfsheadconf.8
cnfsstat.8
control.ctl.5
controlchan.8
convdate.1
ctlinnd.8
cvtbatch.8
cycbuff.conf.5
distrib.pats.5
distributions.5
docheckgroups.8
domain.8
expire.ctl.5
expire.8
expireover.8
expirerm.8
fastrm.1
getlist.1
grephistory.1
ident.8
incoming.conf.5
inews.1
inn.conf.5
INN__Config.3pm
INN__Utils__Shlock.3pm
innbind.8
inncheck.8
innconfval.1
innd.8
inndf.8
innfeed.conf.5
innfeed.8
innmail.1
innupgrade.8
innxmit.8
libauth.3
libinnhist.3
libstorage.3
list.3
mailpost.8
makedbz.8
makehistory.8
mod-active.8
moderators.5
motd.news.5
newsfeeds.5
news.daily.8
news2mail.8
newslog.5
newsgroups.5
ninpaths.8
nnrpd.8
nntpsend.8
nntpsend.ctl.5
ovdb.5
ovdb_init.8
ovdb_monitor.8
ovdb_server.8
ovdb_stat.8
overchan.8
passwd.nntp.5
perl-nocem.8
pgpverify.1
prunehistory.8
pullnews.1
qio.3
radius.8
radius.conf.5
rc.news.8
readers.conf.5
rnews.1
sasl.conf.5
scanlogs.8
scanspool.8
send-uucp.8
sendinpaths.8
shlock.1
simpleftp.1
sm.1
storage.conf.5
subscriptions.5
tally.control.8
tdx-util.8
tinyleaf.8
tst.3
uwildmat.3
+ active.5
active.times.5
actsync.8
archive.8
auth_krb5.8
batcher.8
buffchan.8
buffindexed.conf.5
ckpasswd.8
cnfsheadconf.8
cnfsstat.8
control.ctl.5
controlchan.8
convdate.1
ctlinnd.8
cvtbatch.8
cycbuff.conf.5
distrib.pats.5
distributions.5
docheckgroups.8
domain.8
expire.ctl.5
expire.8
expireover.8
expirerm.8
fastrm.1
getlist.1
grephistory.1
ident.8
incoming.conf.5
inews.1
inn.conf.5
INN__Config.3pm
INN__Utils__Shlock.3pm
innbind.8
inncheck.8
innconfval.1
innd.8
inndf.8
innfeed.conf.5
innfeed.8
innmail.1
innupgrade.8
innxmit.8
libauth.3
libinnhist.3
libstorage.3
list.3
mailpost.8
makedbz.8
makehistory.8
mod-active.8
moderators.5
motd.news.5
newsfeeds.5
news.daily.8
news2mail.8
newslog.5
newsgroups.5
ninpaths.8
nnrpd.8
nntpsend.8
nntpsend.ctl.5
ovdb.5
ovdb_init.8
ovdb_monitor.8
ovdb_server.8
ovdb_stat.8
overchan.8
passwd.nntp.5
perl-nocem.8
pgpverify.1
procbatch.8
prunehistory.8
pullnews.1
qio.3
radius.8
radius.conf.5
rc.news.8
readers.conf.5
rnews.1
sasl.conf.5
scanlogs.8
scanspool.8
send-uucp.8
sendinpaths.8
shlock.1
simpleftp.1
sm.1
storage.conf.5
subscriptions.5
tally.control.8
tdx-util.8
tinyleaf.8
tst.3
uwildmat.3
Modified: doc/man/Makefile
===================================================================
--- doc/man/Makefile 2011-09-04 09:21:43 UTC (rev 9371)
+++ doc/man/Makefile 2011-09-04 09:22:23 UTC (rev 9372)
@@ -29,7 +29,7 @@
innupgrade.8 innwatch.8 innxbatch.8 innxmit.8 mailpost.8 makedbz.8 \
makehistory.8 mod-active.8 news.daily.8 news2mail.8 ninpaths.8 \
nnrpd.8 nntpsend.8 ovdb_init.8 ovdb_monitor.8 ovdb_server.8 \
- ovdb_stat.8 overchan.8 perl-nocem.8 prunehistory.8 radius.8 \
+ ovdb_stat.8 overchan.8 perl-nocem.8 procbatch.8 prunehistory.8 radius.8 \
rc.news.8 scanlogs.8 scanspool.8 send-nntp.8 send-uucp.8 sendinpaths.8 \
tally.control.8 tdx-util.8 tinyleaf.8 writelog.8
Modified: doc/pod/Makefile
===================================================================
--- doc/pod/Makefile 2011-09-04 09:21:43 UTC (rev 9371)
+++ doc/pod/Makefile 2011-09-04 09:22:23 UTC (rev 9372)
@@ -42,7 +42,7 @@
../man/nnrpd.8 ../man/nntpsend.8 \
../man/ovdb_init.8 ../man/ovdb_monitor.8 ../man/ovdb_server.8 \
../man/ovdb_stat.8 ../man/overchan.8 \
- ../man/prunehistory.8 ../man/radius.8 \
+ ../man/procbatch.8 ../man/prunehistory.8 ../man/radius.8 \
../man/rc.news.8 ../man/scanlogs.8 ../man/scanspool.8 \
../man/sendinpaths.8 \
../man/tally.control.8 ../man/tdx-util.8 \
@@ -148,6 +148,7 @@
../man/ovdb_server.8: ovdb_server.pod ; $(POD2MAN) -s 8 $? > $@
../man/ovdb_stat.8: ovdb_stat.pod ; $(POD2MAN) -s 8 $? > $@
../man/overchan.8: overchan.pod ; $(POD2MAN) -s 8 $? > $@
+../man/procbatch.8: procbatch.pod ; $(POD2MAN) -s 8 $? > $@
../man/prunehistory.8: prunehistory.pod ; $(POD2MAN) -s 8 $? > $@
../man/radius.8: radius.pod ; $(POD2MAN) -s 8 $? > $@
../man/rc.news.8: rc.news.pod ; $(POD2MAN) -s 8 $? > $@
Modified: doc/pod/innfeed.pod
===================================================================
--- doc/pod/innfeed.pod 2011-09-04 09:21:43 UTC (rev 9371)
+++ doc/pod/innfeed.pod 2011-09-04 09:22:23 UTC (rev 9372)
@@ -128,9 +128,10 @@
If B<innfeed> is told to send an article to a host it is not managing,
then the article information will be put into a file matching the pattern
F<innfeed-dropped.*>, with part of the file name matching the pid of the
-B<innfeed> process that is writing to it. B<innfeed> will not process
-this file except to write to it. If nothing is written to the file,
-then it will be removed if B<innfeed> exits normally.
+B<innfeed> process that is writing to it. B<innfeed> will not process this
+file except to write to it. If nothing is written to the file, then it
+will be removed if B<innfeed> exits normally. Otherwise, the file remains,
+and B<procbatch> can be invoked to process it afterwards.
=head1 SIGNALS
@@ -467,6 +468,6 @@
=head1 SEE ALSO
-ctlinnd(8), inn.conf(5), innfeed.conf(5), innd(8).
+ctlinnd(8), inn.conf(5), innfeed.conf(5), innd(8), procbatch(8).
=cut
Modified: doc/pod/news.daily.pod
===================================================================
--- doc/pod/news.daily.pod 2011-09-04 09:21:43 UTC (rev 9371)
+++ doc/pod/news.daily.pod 2011-09-04 09:22:23 UTC (rev 9372)
@@ -248,6 +248,6 @@
=head1 SEE ALSO
active(5), ctlinnd(8), expire(8), expire.ctl(5), expireover(8), expirerm(8), fastrm(8),
-inn.conf(5), innstat(8), newslog(5), scanlogs(8), shlock(1).
+inn.conf(5), innstat(8), newslog(5), procbatch(8), scanlogs(8), shlock(1).
=cut
Copied: branches/2.5/doc/pod/procbatch.pod (from rev 9371, trunk/doc/pod/procbatch.pod)
===================================================================
--- doc/pod/procbatch.pod (rev 0)
+++ doc/pod/procbatch.pod 2011-09-04 09:22:23 UTC (rev 9372)
@@ -0,0 +1,144 @@
+=head1 NAME
+
+procbatch - Process an INN funnel file or innfeed-dropped file
+
+=head1 SYNOPSIS
+
+B<procbatch> [B<-hquv>] [B<-c> [B<-s> I<spooldir>]] [B<-d> I<outdir>]
+[B<-e> I<peer>] [B<-m> [B<-t> I<backlogdir>]] I<batchfile>
+
+=head1 DESCRIPTION
+
+B<procbatch> will take an INN funnel or F<innfeed-dropped> file and split
+it up by host for direct processing with B<innfeed> or B<innxmit>.
+
+While funnel files will normally only be of interest after a crash,
+B<innfeed> may drop articles for a variety of reasons, not all of which
+indicate a fundamental problem. For example, B<innfeed> may drop articles
+when started and stopped in quick succession while INN processes a large
+number of control messages (a temporary indisposition), or when INN feeds
+it articles for a site that isn't in F<innfeed.conf> (a misconfiguration).
+
+Every running B<innfeed> opens a file named F<< innfeed-dropped.<id>
+>>, which should normally always be zero length and deleted on exit.
+If there are non-zero length files, B<innfeed> has dropped some articles,
+and those dropped article files have to be processed or those articles will
+never be sent to peers. Though B<news.daily> automatically process these
+files (invoking B<procbatch>), it is also possible to do that manually.
+First make sure that the file doesn't correspond to a currently running
+B<innfeed>, for example by calling C<ctlinnd flush innfeed!>.
+
+An INN funnel file, or an F<innfeed-dropped> file, will usually be of
+the format:
+
+ pathname message-id peer1 peer2 peer3 ...
+
+where I<pathname> can alternatively be a storage API token.
+
+B<procbatch> will break this file up into files named F<peer1.tmp>,
+F<peer2.tmp>, F<peer3.tmp>, etc. of the format:
+
+ pathname message-id
+
+These tape files will be sorted and stripped of duplicate entries. Simply
+renaming them to F<peer1>, F<peer2>, F<peer3>, etc. in a running B<innfeed>'s
+backlog directory will cause them to be picked up automatically by B<innfeed>
+every I<backlog-newfile-period> seconds (as set in in F<innfeed.conf>).
+Use the B<-m> flag to have B<procbatch> rename and move them into place
+automatically.
+
+After running B<procbatch>, you may want to make sure that every generated
+backlog file has a corresponding C<.lock> file. Otherwise, B<innfeed>
+doesn't have that site configured as a peer, meaning your F<innfeed.conf>
+may need fixing.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<-c>
+
+Check for the existence of an article before writing references to it to
+the tape files. Currently, this option can only be used with a tradspool
+article store. Using it with any other spool format will lead to all
+articles being dropped.
+
+=item B<-d> I<outdir>
+
+Put the output file(s) into the directory I<outdir> (defaults to I<pathtmp>
+as set in F<inn.conf>). When also specifying B<-m>, the temporary output
+files will still be put here before being moved into their final location.
+
+=item B<-e> I<peer>
+
+Only process entries for I<peer>.
+
+=item B<-h>
+
+Display a short help screen.
+
+=item B<-m>
+
+Directly append to tape files in B<innfeed>'s backlog directory.
+
+=item B<-q>
+
+Quiet mode: only display error messages.
+
+=item B<-s> I<spooldir>
+
+Use I<spooldir> as the location of the article spool (defaults to
+I<patharticles> as set in F<inn.conf>). This option has an effect only
+on a tradspool storage system when also specifying B<-c>.
+
+=item B<-t> I<backlogdir>
+
+Use I<backlogdir> as the location of the B<innfeed> backlog directory
+(the directory where the created tape files will be put). Defaults to
+I<pathspool>/innfeed. This option has an effect only when also specifying
+B<-m>.
+
+=item B<-u>
+
+Unlink the input batchfile after processing.
+
+=item B<-v>
+
+Add verbosity.
+
+=back
+
+=head1 EXAMPLES
+
+Take the file F<innfeed-dropped.B012345>, process its lines appending to
+tapefiles in B<innfeed>'s backlog directory, and remove it when done.
+Be verbose while doing so:
+
+ procbatch -umv innfeed-dropped.B012345
+
+Go through F<innfeed-dropped.B012345> saving entries for C<peer1>
+in F</tmp/peer1.tmp>, but only if the articles are still available
+in the local tradspool:
+
+ procbatch -e peer1 -d /tmp -c innfeed-dropped.B012345
+
+=head1 BUGS
+
+B<procbatch> should be able to check for the existence of articles
+with any kind of article store, possibly using B<grephistory>.
+
+=head1 HISTORY
+
+B<procbatch> was written by James Brister and improved for speed by
+Clayton O'Neill.
+
+This manual page was written by Florian Schlichting, with the help of a
+memo by Russ Allbery.
+
+$Id$
+
+=head1 SEE ALSO
+
+filechan(8), innfeed(8), innxmit(8), news.daily(8).
+
+=cut
Modified: innfeed/procbatch.in
===================================================================
--- innfeed/procbatch.in 2011-09-04 09:21:43 UTC (rev 9371)
+++ innfeed/procbatch.in 2011-09-04 09:22:23 UTC (rev 9372)
@@ -3,16 +3,14 @@
# Author: James Brister <brister at vix.com> -- berkeley-unix --
# Start Date: Thu May 16 10:32:02 1996 +0200
-# Project: INN -- innfeed
-# File: procbatch.pl
# RCSId: $Id$
#
# Description: Take a file of the form generated by innd in the outgoing
-# directory ("Wnm*") and separate it into tape files for INN.
+# directory ("Wnm*") and separate it into tape files for innfeed.
#
# Thanks to Clayton O'Neill <coneill at premier.net> for serious speed
-# improvements.
-#
+# improvements.
+#
#
# Hmm, perhaps we should try to read "backlog-directory"
@@ -20,7 +18,7 @@
#
use strict;
my $tapeDir = $INN::Config::pathspool . "/innfeed";
-my $destDir = $INN::Config::spooltemp;
+my $destDir = $INN::Config::pathtmp;
my $spoolArts = $INN::Config::patharticles;
my $outGoing = $INN::Config::pathoutgoing;
@@ -32,38 +30,39 @@
use Getopt::Std;
-my $usage = "$0 [ -q ][ -v ][ -u ][ -e host ][ -d dir ][ -c [ -s dir ]][ -m [-t dir ]] inn-batchfile\n
+my $usage = "$0 [-hquv] [-c [-s dir]] [-d dir] [-e host] [-m [-t dir]] inn-batchfile\n
+ -c to check pathnames of articles before storing them
+ -d dir to put the output file(s) in that directory ($destDir)
-e host to process on entries for only that host
- -d dir to put the output file(s) in that directory ($destDir)
- -c to check pathnames of articles before storing them
+ -h display a short help screen
+ -m to have $0 move the new files to the backlog directory
+ -q quiet mode: only display error messages; good for cron jobs
-s dir to specify where the news articles are
($spoolArts)
- -m to have $0 move the new files to the backlog directory
-t dir to specify the backlog directory ($tapeDir)
-u to unlink the input files when finished
-v for verbosity
- -q quiet mode: only display error messages; good for cron jobs
$0 will take an INN funnel file (normally a file in
-$outGoing), or an innfeed ``dropped'' file,
+$outGoing), or an innfeed \"dropped\" file,
which is presumed to be of the format:
pathname message-id peer1 peer2 peer3 ...
-and will break it up into files peer1.tmp peer2.tmp peer3.tmp... Each of
+and will break it up into files peer1.tmp, peer2.tmp, peer3.tmp... Each of
these files will be of the format:
pathname message-id
that is the same as innfeed's backlog file format. Simply rename these files
-to peer1 peer2 peer3 in a running innfeed's backlog directory and they will be
-picked up automatically and processed by innfeed. Use the '-m' flag and
-they'll be moved automatically.
+to peer1, peer2, peer3... in a running innfeed's backlog directory and they
+will be picked up automatically and processed by innfeed. Use the '-m' flag
+and they'll be moved automatically.
";
my (%opt, %hosts);
my $missing = 0;
-getopts ("he:t:s:d:cvumq", \%opt) || die $usage;
+getopts ("cd:e:hmqs:t:uv", \%opt) || die $usage;
die $usage if $opt{'h'};
die "Cannot specify both -q and -v\n\n" . $usage if ($opt{'q'} && $opt{'v'});
@@ -81,6 +80,8 @@
die "No such directory: $spoolArts\n\n" . $usage if ( ! -d $spoolArts && $opt{'c'} );
die "No such directory: $destDir\n\n" . $usage if ( ! -d $destDir );
die "No such directory: $tapeDir\n\n" . $usage if ( ! -d $tapeDir && $opt{'m'} );
+warn "Specifying -s without -c has no effect!" if $opt{'s'} and not $opt{'c'};
+warn "Specifying -t without -m has no effect!" if $opt{'t'} and not $opt{'m'};
print "Using $inputFile\n" if $opt{'v'};
open (my $INPUT, '<', $inputFile) || die "$0: open ($inputFile): $!\n";
@@ -89,7 +90,7 @@
chop;
my @F = split;
- # Check the format of the line vigorously
+ # Check the format of the line vigorously
next unless (m!^\S+/\d+ <.+ at .+> \S+! || m!^@[0-9A-F]+@ <.+ at .+> \S+!);
if ( $opt{'c'} ) {
@@ -104,7 +105,7 @@
my $host = $F[$i];
next if ($opt{'e'} && $opt{'e'} ne $host);
- # Keep out host names with any funny characters (from
+ # Keep out host names with any funny characters (from
# corrupted files)
if ($host !~ /^[-\._0-9A-Za-z]+$/) {
warn "$0: bad site name ignored: \"$host\"\n";
More information about the inn-committers
mailing list