INN commit: trunk (36 files)
INN Commit
Russ_Allbery at isc.org
Sat Apr 11 22:13:42 UTC 2009
Date: Saturday, April 11, 2009 @ 15:13:41
Author: iulius
Revision: 8409
* Add support for list values in inn.conf.
* Remove the overview.fmt file and use two new inn.conf
parameters: extraoverviewadvertised and extraoverviewhidden.
They allow the news administrator to choose the overview
fields he wants to advertise to readers.
* Add a new argument to overview_extra_fields: this function
returns either the additional fields to advertise (with ":full")
or all the additional fields to generate.
* Xref is forced as the eighth field of the overview database.
* Be consistent with the responses of other LIST commands:
no full stop at the end of the initial response line.
* Updated documentation, samples and test suite.
close #101
Added:
trunk/tests/data/upgrade/overview.fmt
Modified:
trunk/MANIFEST
trunk/doc/man/Makefile
trunk/doc/man/inncheck.8
trunk/doc/pod/ctlinnd.pod
trunk/doc/pod/getlist.pod
trunk/doc/pod/inn.conf.pod
trunk/doc/pod/makehistory.pod
trunk/doc/pod/news.pod
trunk/doc/pod/nnrpd.pod
trunk/expire/Makefile
trunk/expire/makehistory.c
trunk/include/inn/innconf.h
trunk/include/inn/ov.h
trunk/include/inn/overview.h
trunk/include/inn/paths.h.in
trunk/innd/Makefile
trunk/innd/art.c
trunk/lib/innconf.c
trunk/nnrpd/list.c
trunk/nnrpd/nnrpd.c
trunk/samples/inn.conf.in
trunk/site/ (properties)
trunk/site/Makefile
trunk/storage/Makefile
trunk/storage/expire.c
trunk/storage/overdata.c
trunk/storage/tradindexed/tdx-util.c
trunk/support/mkmanifest
trunk/tests/data/etc/inn.conf
trunk/tests/data/upgrade/inn.conf.ok
trunk/tests/innd/artparse-t.c
trunk/tests/lib/innconf-t.c
Deleted:
trunk/doc/man/overview.fmt.5
trunk/samples/overview.fmt
trunk/tests/data/etc/overview.fmt
---------------------------------+
MANIFEST | 4
doc/man/Makefile | 2
doc/man/inncheck.8 | 1
doc/man/overview.fmt.5 | 58 ----------
doc/pod/ctlinnd.pod | 2
doc/pod/getlist.pod | 9 +
doc/pod/inn.conf.pod | 97 +++++++++++++++++
doc/pod/makehistory.pod | 2
doc/pod/news.pod | 36 +++++-
doc/pod/nnrpd.pod | 5
expire/Makefile | 2
expire/makehistory.c | 90 +++++++---------
include/inn/innconf.h | 4
include/inn/ov.h | 2
include/inn/overview.h | 2
include/inn/paths.h.in | 1
innd/Makefile | 5
innd/art.c | 76 ++++++-------
lib/innconf.c | 207 ++++++++++++++++++++++++++++++++++++--
nnrpd/list.c | 6 -
nnrpd/nnrpd.c | 2
samples/inn.conf.in | 2
samples/overview.fmt | 16 --
site/Makefile | 9 -
storage/Makefile | 10 -
storage/expire.c | 65 ++++-------
storage/overdata.c | 96 ++++++-----------
storage/tradindexed/tdx-util.c | 2
support/mkmanifest | 1
tests/data/etc/inn.conf | 17 +--
tests/data/etc/overview.fmt | 12 --
tests/data/upgrade/inn.conf.ok | 7 -
tests/data/upgrade/overview.fmt | 18 +++
tests/innd/artparse-t.c | 7 -
tests/lib/innconf-t.c | 5
35 files changed, 535 insertions(+), 345 deletions(-)
Modified: MANIFEST
===================================================================
--- MANIFEST 2009-04-11 17:49:27 UTC (rev 8408)
+++ MANIFEST 2009-04-11 22:13:41 UTC (rev 8409)
@@ -197,7 +197,6 @@
doc/man/ovdb_server.8 Manpage for ovdb_server
doc/man/ovdb_stat.8 Manpage for ovdb_stat
doc/man/overchan.8 Manpage for overchan backend
-doc/man/overview.fmt.5 Manpage for overview.fmt config file
doc/man/passwd.nntp.5 Manpage for passwd.nntp config file
doc/man/perl-nocem.8 Manpage for perl-nocem
doc/man/pgpverify.1 Manpage for pgpverify
@@ -636,7 +635,6 @@
samples/nntpsend.ctl Outgoing nntpsend feed configuration
samples/nocem.ctl Config file for perl-nocem
samples/ovdb.conf Berkeley DB overview configuration
-samples/overview.fmt Format of news overview database
samples/passwd.nntp Passwords for remote connections
samples/radius.conf Sample config for RADIUS authentication
samples/readers.conf Reader connection configuration
@@ -807,7 +805,6 @@
tests/data/etc/inn-bfx.conf Basic inn.conf for testing buffindexed
tests/data/etc/inn-tdx.conf Basic inn.conf for testing tradindexed
tests/data/etc/inn.conf Basic inn.conf file for testing
-tests/data/etc/overview.fmt Overview configuration for testing
tests/data/etc/passwd Password data for ckpasswd tests
tests/data/etc/storage.conf Storage configuration for testing
tests/data/overview Test overview data (Directory)
@@ -824,6 +821,7 @@
tests/data/upgrade/inn.conf.ok Fixed inn.conf file
tests/data/upgrade/newsfeeds newsfeeds file that needs fixing
tests/data/upgrade/newsfeeds.ok Fixed newsfeeds file
+tests/data/upgrade/overview.fmt Obsolete overview.fmt config file
tests/data/upgrade/sasl.conf Obsolete sasl.conf config file
tests/innd Test suite for innd (Directory)
tests/innd/artparse-t.c Tests for ARTparse in innd
Modified: doc/man/Makefile
===================================================================
--- doc/man/Makefile 2009-04-11 17:49:27 UTC (rev 8408)
+++ doc/man/Makefile 2009-04-11 22:13:41 UTC (rev 8409)
@@ -17,7 +17,7 @@
cycbuff.conf.5 distrib.pats.5 distributions.5 expire.ctl.5 history.5 incoming.conf.5 \
inn.conf.5 innfeed.conf.5 innwatch.ctl.5 moderators.5 motd.news.5 \
newsfeeds.5 newsgroups.5 newslog.5 nnrpd.track.5 nntpsend.ctl.5 ovdb.5 \
- overview.fmt.5 passwd.nntp.5 radius.conf.5 readers.conf.5 \
+ passwd.nntp.5 radius.conf.5 readers.conf.5 \
storage.conf.5 subscriptions.5
SEC8 = actsync.8 archive.8 batcher.8 buffchan.8 ckpasswd.8 \
Modified: doc/man/inncheck.8
===================================================================
--- doc/man/inncheck.8 2009-04-11 17:49:27 UTC (rev 8408)
+++ doc/man/inncheck.8 2009-04-11 22:13:41 UTC (rev 8409)
@@ -58,7 +58,6 @@
inn.conf
moderators
newsfeeds
- overview.fmt
nntpsend.ctl
passwd.nntp
readers.conf
Deleted: doc/man/overview.fmt.5
===================================================================
--- doc/man/overview.fmt.5 2009-04-11 17:49:27 UTC (rev 8408)
+++ doc/man/overview.fmt.5 2009-04-11 22:13:41 UTC (rev 8409)
@@ -1,58 +0,0 @@
-.\" $Revision$
-.TH OVERVIEW.FMT 5
-.SH NAME
-overview.fmt \- format of news overview database
-.SH DESCRIPTION
-The file
-.I <pathetc in inn.conf>/overview.fmt
-specifies the organization of the news overview database.
-Blank lines and lines beginning with a number sign (``#'') are ignored.
-The order of lines in this file is important; it determines the order
-in which the fields will appear in the database.
-.PP
-Most lines will consist of an article header name, optionally
-followed by a colon.
-A trailing set of lines can have the word ``full'' appear after the
-colon; this indicates that the header should appear as well as its value.
-.PP
-If this file is changed, new overview records will be constructed with
-the modified format. If it is desired that existing records be updated to
-the new format, it is necessary to rebuild the overview database using
-.IR makehistory (8)
-after removing all existing overview files.
-.PP
-\&``Xref:full'' must be included, or
-.IR innd (8)
-and other programs which utilize overview method cannot start.
-.PP
-The default file, show below, is compatible with Geoff Collyer's ``nov''
-package:
-.PP
-.RS
-.nf
-Subject:
-From:
-Date:
-Message-ID:
-References:
-Bytes:
-Lines:
-Xref:full
-.fi
-.RE
-.PP
-Usually the only modifications which should be made to the default file
-are additions of new fields to the end of the file; all such fields
-should have ``full'' after the colon. The first eight fields should
-remain in the same order as above.
-.SH HISTORY
-Written by Rich $alz <rsalz at uunet.uu.net> for InterNetNews.
-Intended to be compatible with the
-.I nov
-package written by Geoff Collyer <geoff at world.std.com>.
-.de R$
-This is revision \\$3, dated \\$4.
-..
-.R$ $Id$
-.SH "SEE ALSO"
-inn.conf(5)
Modified: doc/pod/ctlinnd.pod
===================================================================
--- doc/pod/ctlinnd.pod 2009-04-11 17:49:27 UTC (rev 8408)
+++ doc/pod/ctlinnd.pod 2009-04-11 22:13:41 UTC (rev 8409)
@@ -496,6 +496,6 @@
=head1 SEE ALSO
active(5), active.times(5), buffchan(8), incoming.conf(5), innd(8),
-inndcomm(3), inn.conf(5), newsfeeds(5), nnrpd(8), overview.fmt(5).
+inndcomm(3), inn.conf(5), newsfeeds(5), nnrpd(8).
=cut
Modified: doc/pod/getlist.pod
===================================================================
--- doc/pod/getlist.pod 2009-04-11 17:49:27 UTC (rev 8408)
+++ doc/pod/getlist.pod 2009-04-11 22:13:41 UTC (rev 8409)
@@ -41,7 +41,10 @@
may also be usable for some of these listing files.
For more information on the formats of these files, see distrib.pats(5),
-moderators(5), motd.news(5), overview.fmt(5) and subscriptions(5).
+moderators(5), motd.news(5) and subscriptions(5). The overview fields
+obtained with C<overview.fmt> are the ones for which the overview database
+is consistent (see I<extraoverviewadvertised> in F<inn.conf> for more
+information).
=head1 OPTIONS
@@ -100,7 +103,7 @@
=head1 SEE ALSO
active(5), active.times(5), distrib.pats(5), distributions(5), inn.conf(5),
-moderators(5), motd.news(5), newsgroups(5), nnrpd(8), overview.fmt(5),
-passwd.nntp(5), subscriptions(5), uwildmat(3).
+moderators(5), motd.news(5), newsgroups(5), nnrpd(8), passwd.nntp(5),
+subscriptions(5), uwildmat(3).
=cut
Modified: doc/pod/inn.conf.pod
===================================================================
--- doc/pod/inn.conf.pod 2009-04-11 17:49:27 UTC (rev 8408)
+++ doc/pod/inn.conf.pod 2009-04-11 22:13:41 UTC (rev 8409)
@@ -387,6 +387,101 @@
true, I<ovmethod> must also be set. This is a boolean value and the
default is true.
+=item I<extraoverviewadvertised>
+
+Besides the seven standard overview fields (which are in order C<Subject:>,
+C<From:>, C<Date:>, C<Message-ID:>, C<References:>, C<:bytes> and
+C<:lines>) and the eighth C<Xref:full> field required by INN in order
+to handle crossposts, it is possible to add other fields in the
+overview database. This parameter expects a list of such header
+names. Overview data for these additional headers will be generated
+for each new article at the time of arrival. For instance, if you specify:
+
+ extraoverviewadvertised: [ Path NNTP-Hosting-Host ]
+
+it implies that B<nnrpd> will advertise C<Path:full> and
+C<NNTP-Hosting-Host:full> as the ninth and tenth fields in response
+to LIST OVERVIEW.FMT and that these two headers will be stored in the
+overview database for each new article.
+
+The default value is an empty list (no additional fields are stored).
+Owing to optimizations when B<innd> parses the articles it receives,
+it is possible that all the values in the list are not recognized by
+B<innd> as standard headers. In such cases, B<innd> will log an error
+in F<news.err> at startup and the unrecognized fields will be discarded.
+
+You should advertise only fields for which the overview database
+is consistent, that is to say it records the content or absence
+of these fields for I<all> articles, including those already existing
+in the news spool. Consequently, if you decide to add or remove a field
+from your overview database, you should either modify
+I<extraoverviewadvertised> and rebuild your overview database with
+makehistory(8) after removing all existing overview files, or implement
+a transition period by first using I<extraoverviewhidden> as described below.
+
+Use of a transition period can accommodate most overview reconfigurations,
+but certain drastic changes may still require a complete overview rebuild.
+
+If for instance you want to store the content of the To: header in addition
+to the fields already stored above, you should use:
+
+ extraoverviewadvertised: [ Path NNTP-Hosting-Host ]
+ extraoverviewhidden: [ To ]
+
+This way, C<To:full> will not be advertised by B<nnrpd> but will be stored
+for each new article. Once you know that all articles in your overview
+database record the content or absence of that new field (if expire.ctl(5) is
+parametered so that all your articles expire within S<30 days>, you can assume
+the database is in such a state after S<30 days> S<-- however>, note that time
+to expiration can be unpredictable with CNFS and you then have to use
+C<cnfsstat -a> for checking on when buffers have rolled over), you should put:
+
+ extraoverviewadvertised: [ Path NNTP-Hosting-Host To ]
+ extraoverviewhidden: [ ]
+
+The C<To> value must be added at the end of the list because
+order matters and fields mentioned in I<extraoverviewhidden>
+are generated after those mentioned in I<extraoverviewadvertised>.
+B<nnrpd> will now advertise C<To:full> in response to the LIST
+OVERVIEW.FMT command (C<full> indicates that the header appears
+followed by its value).
+
+Now suppose you want to remove the content of the NNTP-Hosting-Host:
+header from the overview. As order matters, the overview database will
+no longer be consistent for the To: header. Therefore, you need to specify:
+
+ extraoverviewadvertised: [ Path ]
+ extraoverviewhidden: [ To ]
+
+And once overview data is accurate for all articles, you should use:
+
+ extraoverviewadvertised: [ Path To ]
+ extraoverviewhidden: [ ]
+
+Note that you have to restart B<nnrpd> if it runs as a daemon
+whenever you change the value of I<extraoverviewadvertised>; a mere
+C<ctlinnd xexec innd> is not enough.
+
+=item I<extraoverviewhidden>
+
+This parameter should be used in conjunction with
+I<extraoverviewadvertised> (see above for more details). It expects
+a list of headers names. Overview data for these headers will be
+generated for each new article at the time of arrival but, contrary
+to the fields mentioned in I<extraoverviewadvertised>, B<nnrpd> will
+not advertise them in response to the LIST OVERVIEW.FMT command. It
+also implies that B<nnrpd> will not look in the overview database
+for fields mentioned in I<extraoverviewhidden> when it handles HDR,
+XHDR and XPAT requests; B<nnrpd> will have to parse the headers
+of the requested articles in the news spool, which is slower than
+directly querying the overview database.
+
+The default value is an empty list (no additional fields are stored).
+Owing to optimizations when B<innd> parses the articles it receives,
+it is possible that all the values in the list are not recognized by
+B<innd> as standard headers. In such cases, B<innd> will log an error
+in F<news.err> at startup and the unrecognized fields will be discarded.
+
=item I<groupbaseexpiry>
Whether to enable newsgroup-based expiry. If set to false, article expiry
@@ -1331,7 +1426,7 @@
=head1 SEE ALSO
-inews(1), innd(8), innwatch(8), nnrpd(8), rnews(1).
+inews(1), innd(8), innwatch(8), makehistory(8), nnrpd(8), rnews(1).
Nearly every program in INN uses this file to one degree or another. The
above are just the major and most frequently mentioned ones.
Modified: doc/pod/makehistory.pod
===================================================================
--- doc/pod/makehistory.pod 2009-04-11 17:49:27 UTC (rev 8408)
+++ doc/pod/makehistory.pod 2009-04-11 22:13:41 UTC (rev 8409)
@@ -191,6 +191,6 @@
=head1 SEE ALSO
active(5), ctlinnd(8), dbz(3), history(5), inn.conf(5), innd(8),
-makedbz(8), ovdb_init(8), overchan(8), overview.fmt(5).
+makedbz(8), ovdb_init(8), overchan(8).
=cut
Modified: doc/pod/news.pod
===================================================================
--- doc/pod/news.pod 2009-04-11 17:49:27 UTC (rev 8408)
+++ doc/pod/news.pod 2009-04-11 22:13:41 UTC (rev 8409)
@@ -16,7 +16,7 @@
or you can install the module directly from CPAN (C<MIME-tools> in
F<modules/by-module/MIME/>, for instance on ftp.perl.org).
-Perl 5.8.0 or later is recommended for INN. If you are using an earlier
+S<Perl 5.8.0> or later is recommended for INN. If you are using an earlier
version, you will also need the C<Encode> module for correct processing of
control messages. (It is included with Perl itself in 5.8.0 and later.)
@@ -30,6 +30,16 @@
=item *
+The F<overview.fmt> file is no longer used by INN. Two new parameters
+have been added to F<inn.conf>: I<extraoverviewadvertised> and
+I<extraoverviewhidden>. Although B<innupgrade> takes care of the
+change during C<make update>, you should make sure that your overview
+database is consistent with all the fields declared in F<overview.fmt>
+because they will all be advertised. See the inn.conf(5) man page
+for more information about these parameters.
+
+=item *
+
The B<innreport> configuration file has slightly changed. The new
F<innreport.conf> file shipped with INN should be used and your possible
changes backported to this new version.
@@ -160,7 +170,7 @@
described in USEPRO and can handle character set conversions of newsgroup
descriptions. The C<MIME::Parser> and C<Encode> modules are used.
Processing control messages has been greatly improved, especially
-checkgroups: the F<active> and F<newsgroups> files are now properly
+checkgroups: the F<active> and F<newsgroups> files are now properly
updated when they are processed.
A new F<control.ctl.local> file has also been added in I<pathetc>. Rules
@@ -232,15 +242,33 @@
allows to set the news user and the news group under which the news server
runs. Thanks to Ivan Shmakov for this feature.
-New other options have been added to configuration files: I<ignore> in
+New other options have been added to configuration files: I<ignore> in
F<incoming.conf>, I<logstats> and I<nnrpdflags> in F<inn.conf>, and
I<log-time-format> in F<innfeed.conf>.
The B<--with-http-dir> option has also been added to C<configure> to set
I<pathhttp> in F<inn.conf>.
+The I<nntpactsync> parameter has been renamed to I<incominglogfrequency>
+in F<inn.conf>.
+
=item *
+The F<sasl.conf> file has been removed in favour of new parameters in
+F<inn.conf> to deal with TLS support: I<tlscafile>, I<tlscapath>,
+I<tlscertfile> and I<tlskeyfile>.
+
+The F<overview.fmt> file has been removed in favour of new parameters
+in F<inn.conf> to deal with transition periods to accommodate
+overview reconfigurations. It is now possible to specify on the one
+hand the fields that should be advertised by B<nnrpd> in response to
+LIST OVERVIEW.FMT and used for HDR, XHDR and XPAT requests (see the new
+I<extraoverviewadvertised> parameter) and on the other hand the
+additional fields that should be silently generated (see the new
+I<extraoverviewhidden> parameter).
+
+=item *
+
Support for S<Berkeley DB> versions prior to 4.3 has been dropped. You
will have to use at least S<Berkeley DB 4.4>; the recommended version is
4.7.
@@ -261,7 +289,7 @@
=item *
-A lot of work has been done on documentation: improvements of existing
+A lot of work has been done on documentation: improvements of existing
documents, new documentation, and proof-reading. Sample configuration
files are also more detailed.
Modified: doc/pod/nnrpd.pod
===================================================================
--- doc/pod/nnrpd.pod 2009-04-11 17:49:27 UTC (rev 8408)
+++ doc/pod/nnrpd.pod 2009-04-11 22:13:41 UTC (rev 8409)
@@ -230,8 +230,9 @@
where created, a file specifying default distribution patterns, a list
of valid distributions, the moderators list, the message of the day information
for readers, a one-per-line description of the current set of newsgroups,
-a listing of the F<overview.fmt> file, or a list of the automatic group
-subscriptions.
+a listing of the overview fields for which the overview database is consistent
+(see I<extraoverviewadvertised> in F<inn.conf> for more information), or
+a list of the automatic group subscriptions.
The command C<list active> is equivalent to the C<list> command. This
is a common extension.
Modified: expire/Makefile
===================================================================
--- expire/Makefile 2009-04-11 17:49:27 UTC (rev 8408)
+++ expire/Makefile 2009-04-11 22:13:41 UTC (rev 8409)
@@ -133,7 +133,7 @@
../include/inn/newsuser.h ../include/config.h ../include/clibrary.h \
../include/inn/ov.h ../include/inn/storage.h ../include/inn/history.h \
../include/inn/paths.h ../include/inn/qio.h ../include/inn/storage.h \
- ../include/inn/wire.h
+ ../include/inn/vector.h ../include/inn/wire.h
prunehistory.o: prunehistory.c ../include/config.h \
../include/inn/defines.h ../include/inn/system.h \
../include/inn/options.h ../include/clibrary.h ../include/config.h \
Modified: expire/makehistory.c
===================================================================
--- expire/makehistory.c 2009-04-11 17:49:27 UTC (rev 8408)
+++ expire/makehistory.c 2009-04-11 22:13:41 UTC (rev 8409)
@@ -21,6 +21,7 @@
#include "inn/paths.h"
#include "inn/qio.h"
#include "inn/storage.h"
+#include "inn/vector.h"
#include "inn/wire.h"
static const char usage[] = "\
@@ -54,7 +55,6 @@
#define DEFAULT_SEGSIZE 10000;
bool NukeBadArts;
-char *SchemaPath = NULL;
char *ActivePath = NULL;
char *HistoryPath = NULL;
struct history *History;
@@ -77,7 +77,8 @@
static char LINES[] = "Lines";
static char MESSAGEID[] = "Message-ID";
static char XREF[] = "Xref";
-static ARTOVERFIELD *ARTfields; /* overview fields listed in overview.fmt */
+static ARTOVERFIELD *ARTfields; /* Overview fields for which overview data
+ * is generated. */
static size_t ARTfieldsize;
static ARTOVERFIELD *Bytesp = (ARTOVERFIELD *)NULL;
static ARTOVERFIELD *Datep = (ARTOVERFIELD *)NULL;
@@ -85,9 +86,8 @@
static ARTOVERFIELD *Linesp = (ARTOVERFIELD *)NULL;
static ARTOVERFIELD *Msgidp = (ARTOVERFIELD *)NULL;
static ARTOVERFIELD *Xrefp = (ARTOVERFIELD *)NULL;
-static ARTOVERFIELD *Missfields; /* Header fields not listed in
- * overview.fmt, but ones that we need
- * (e.g. Message-ID:). */
+static ARTOVERFIELD *Missfields; /* Header fields not used in overview
+ * but that we need (e.g. Expires:). */
static size_t Missfieldsize = 0;
static void OverAddAllNewsgroups(void);
@@ -393,39 +393,23 @@
static void
ARTreadschema(bool Overview)
{
- FILE *F;
- char *p;
+ const struct cvector *standardoverview;
+ const struct vector *extraoverview;
ARTOVERFIELD *fp;
- int i;
- char buff[SMBUF];
- bool foundxreffull = false;
+ unsigned int i;
if (Overview) {
- /* Open file, count lines. */
- if ((F = fopen(SchemaPath, "r")) == NULL)
- sysdie("cannot open %s", SchemaPath);
- for (i = 0; fgets(buff, sizeof buff, F) != NULL; i++)
- continue;
- fseeko(F, 0, SEEK_SET);
- ARTfields = xmalloc((i + 1) * sizeof(ARTOVERFIELD));
+ /* Count the number of overview fields and allocate ARTfields. */
+ standardoverview = overview_fields();
+ extraoverview = overview_extra_fields(true);
+ ARTfields = xmalloc((standardoverview->count + extraoverview->count + 1)
+ * sizeof(ARTOVERFIELD));
/* Parse each field. */
- for (fp = ARTfields; fgets(buff, sizeof buff, F) != NULL; ) {
- /* Ignore blank and comment lines. */
- if ((p = strchr(buff, '\n')) != NULL)
- *p = '\0';
- if ((p = strchr(buff, '#')) != NULL)
- *p = '\0';
- if (buff[0] == '\0')
- continue;
- if ((p = strchr(buff, ':')) != NULL) {
- *p++ = '\0';
- fp->NeedHeadername = (strcmp(p, "full") == 0);
- }
- else
- fp->NeedHeadername = false;
- fp->Headername = xstrdup(buff);
- fp->HeadernameLength = strlen(buff);
+ for (i = 0, fp = ARTfields; i < standardoverview->count; i++) {
+ fp->NeedHeadername = false;
+ fp->Headername = xstrdup(standardoverview->strings[i]);
+ fp->HeadernameLength = strlen(standardoverview->strings[i]);
fp->Header = (char *)NULL;
fp->HasHeader = false;
fp->HeaderLength = 0;
@@ -433,24 +417,37 @@
* because we will need them afterwards; we will then not
* have to compare fp->Headername to both "Bytes" and "Lines"
* for each fp. */
- if (strncasecmp(buff, BYTES, strlen(BYTES)) == 0)
+ if (strncasecmp(standardoverview->strings[i],
+ BYTES, strlen(BYTES)) == 0)
Bytesp = fp;
- if (strncasecmp(buff, DATE, strlen(DATE)) == 0)
+ if (strncasecmp(standardoverview->strings[i],
+ DATE, strlen(DATE)) == 0)
Datep = fp;
- if (strncasecmp(buff, EXPIRES, strlen(EXPIRES)) == 0)
- Expp = fp;
- if (strncasecmp(buff, LINES, strlen(LINES)) == 0)
+ if (strncasecmp(standardoverview->strings[i],
+ LINES, strlen(LINES)) == 0)
Linesp = fp;
- if (strncasecmp(buff, MESSAGEID, strlen(MESSAGEID)) == 0)
+ if (strncasecmp(standardoverview->strings[i],
+ MESSAGEID, strlen(MESSAGEID)) == 0)
Msgidp = fp;
- if (strncasecmp(buff, XREF, strlen(XREF)) == 0) {
- Xrefp = fp;
- foundxreffull = fp->NeedHeadername;
- }
fp++;
}
+ for (i = 0; i < extraoverview->count; i++) {
+ fp->NeedHeadername = true;
+ fp->Headername = xstrdup(extraoverview->strings[i]);
+ fp->HeadernameLength = strlen(extraoverview->strings[i]);
+ fp->Header = (char *)NULL;
+ fp->HasHeader = false;
+ fp->HeaderLength = 0;
+ if (strncasecmp(extraoverview->strings[i],
+ XREF, strlen(XREF)) == 0)
+ Xrefp = fp;
+ if (strncasecmp(extraoverview->strings[i],
+ EXPIRES, strlen(EXPIRES)) == 0)
+ Expp = fp;
+ fp++;
+ }
+
ARTfieldsize = fp - ARTfields;
- fclose(F);
}
if (Bytesp == (ARTOVERFIELD *)NULL)
Missfieldsize++;
@@ -462,8 +459,6 @@
Missfieldsize++;
if (Msgidp == (ARTOVERFIELD *)NULL)
Missfieldsize++;
- if (Overview && (Xrefp == (ARTOVERFIELD *)NULL || !foundxreffull))
- die("Xref:full must be included in %s", SchemaPath);
if (Missfieldsize > 0) {
Missfields = xmalloc(Missfieldsize * sizeof(ARTOVERFIELD));
fp = Missfields;
@@ -834,7 +829,6 @@
HistoryPath = concatpath(innconf->pathdb, INN_PATH_HISTORY);
ActivePath = concatpath(innconf->pathdb, INN_PATH_ACTIVE);
TmpDir = innconf->pathtmp;
- SchemaPath = concatpath(innconf->pathetc, INN_PATH_SCHEMA);
RebuiltflagPath = concatpath(innconf->pathrun, INN_PATH_REBUILDOVERVIEW);
OverTmpSegSize = DEFAULT_SEGSIZE;
@@ -914,7 +908,7 @@
ensure_news_user_grp(true, true);
}
- /* Read in the overview schema */
+ /* Read in the overview schema. */
ARTreadschema(DoOverview);
if (DoOverview && !WriteStdout) {
Modified: include/inn/innconf.h
===================================================================
--- include/inn/innconf.h 2009-04-11 17:49:27 UTC (rev 8408)
+++ include/inn/innconf.h 2009-04-11 22:13:41 UTC (rev 8409)
@@ -59,6 +59,10 @@
/* Article Storage */
long cnfscheckfudgesize; /* Additional CNFS integrity checking */
bool enableoverview; /* Store overview info for articles? */
+ struct vector
+ *extraoverviewadvertised; /* Extra overview fields for LIST OVERVIEW.FMT */
+ struct vector
+ *extraoverviewhidden; /* Extra overview fields silently generated */
bool groupbaseexpiry; /* Do expiry by newsgroup? */
bool mergetogroups; /* Refile articles from to.* into to */
bool nfswriter; /* Use NFS writer functionality */
Modified: include/inn/ov.h
===================================================================
--- include/inn/ov.h 2009-04-11 17:49:27 UTC (rev 8408)
+++ include/inn/ov.h 2009-04-11 22:13:41 UTC (rev 8409)
@@ -50,7 +50,7 @@
/* Overview data manipulation functions. */
const struct cvector *overview_fields(void);
-struct vector *overview_extra_fields(void);
+struct vector *overview_extra_fields(bool hidden);
struct buffer *overview_build(ARTNUM number, const char *article,
size_t length, const struct vector *extra,
struct buffer *);
Modified: include/inn/overview.h
===================================================================
--- include/inn/overview.h 2009-04-11 17:49:27 UTC (rev 8408)
+++ include/inn/overview.h 2009-04-11 22:13:41 UTC (rev 8409)
@@ -176,7 +176,7 @@
/* Overview data manipulation functions. */
const struct cvector *overview_fields(void);
-struct vector *overview_extra_fields(void);
+struct vector *overview_extra_fields(bool hidden);
struct buffer *overview_build(ARTNUM number, const char *article,
size_t length, const struct vector *extra,
struct buffer *);
Modified: include/inn/paths.h.in
===================================================================
--- include/inn/paths.h.in 2009-04-11 17:49:27 UTC (rev 8408)
+++ include/inn/paths.h.in 2009-04-11 22:13:41 UTC (rev 8409)
@@ -68,7 +68,6 @@
#define INN_PATH_NNTPPASS "passwd.nntp"
#define INN_PATH_NNRPACCESS "readers.conf"
#define INN_PATH_EXPIRECTL "expire.ctl"
-#define INN_PATH_SCHEMA "overview.fmt"
#define INN_PATH_MOTD "motd.news"
#define INN_PATH_STORAGECTL "storage.conf"
#define INN_PATH_RADIUS_CONFIG "radius.conf"
Modified: innd/Makefile
===================================================================
--- innd/Makefile 2009-04-11 17:49:27 UTC (rev 8408)
+++ innd/Makefile 2009-04-11 22:13:41 UTC (rev 8409)
@@ -77,8 +77,9 @@
../include/inn/system.h ../include/inn/options.h ../include/clibrary.h \
../include/config.h ../include/inn/innconf.h ../include/inn/defines.h \
../include/inn/md5.h ../include/inn/ov.h ../include/inn/storage.h \
- ../include/inn/history.h ../include/inn/storage.h ../include/inn/wire.h \
- innd.h ../include/portable/time.h ../include/config.h \
+ ../include/inn/history.h ../include/inn/storage.h \
+ ../include/inn/vector.h ../include/inn/wire.h innd.h \
+ ../include/portable/time.h ../include/config.h \
../include/portable/socket.h ../include/portable/getaddrinfo.h \
../include/portable/getnameinfo.h ../include/inn/buffer.h \
../include/inn/history.h ../include/inn/messages.h \
Modified: innd/art.c
===================================================================
--- innd/art.c 2009-04-11 17:49:27 UTC (rev 8408)
+++ innd/art.c 2009-04-11 22:13:41 UTC (rev 8409)
@@ -11,6 +11,7 @@
#include "inn/md5.h"
#include "inn/ov.h"
#include "inn/storage.h"
+#include "inn/vector.h"
#include "inn/wire.h"
#include "innd.h"
@@ -134,71 +135,60 @@
bool
ARTreadschema(void)
{
- static char *SCHEMA = NULL;
- FILE *F;
- int i;
- char *p;
- ARTOVERFIELD *fp;
- const ARTHEADER *hp;
- bool ok;
- char buff[SMBUF];
- bool foundxref = false;
- bool foundxreffull = false;
+ const struct cvector *standardoverview;
+ const struct vector *extraoverview;
+ unsigned int i;
+ ARTOVERFIELD *fp;
+ const ARTHEADER *hp;
+ bool ok = true;
if (ARTfields != NULL) {
free(ARTfields);
ARTfields = NULL;
}
- /* Open file, count lines. */
- if (SCHEMA == NULL)
- SCHEMA = concatpath(innconf->pathetc, INN_PATH_SCHEMA);
- if ((F = Fopen(SCHEMA, "r", TEMPORARYOPEN)) == NULL)
- return false;
- for (i = 0; fgets(buff, sizeof buff, F) != NULL; i++)
- continue;
- fseeko(F, 0, SEEK_SET);
- ARTfields = xmalloc((i + 1) * sizeof(ARTOVERFIELD));
+ /* Count the number of overview fields and allocate ARTfields. */
+ standardoverview = overview_fields();
+ extraoverview = overview_extra_fields(true);
+ ARTfields = xmalloc((standardoverview->count + extraoverview->count + 1)
+ * sizeof(ARTOVERFIELD));
/* Parse each field. */
- for (ok = true, fp = ARTfields ; fgets(buff, sizeof buff, F) != NULL ;) {
- /* Ignore blank and comment lines. */
- if ((p = strchr(buff, '\n')) != NULL)
- *p = '\0';
- if ((p = strchr(buff, '#')) != NULL)
- *p = '\0';
- if (buff[0] == '\0')
- continue;
- if ((p = strchr(buff, ':')) != NULL) {
- *p++ = '\0';
- fp->NeedHeader = (strcmp(p, "full") == 0);
- } else
- fp->NeedHeader = false;
- if (strcasecmp(buff, "Xref") == 0) {
- foundxref = true;
- foundxreffull = fp->NeedHeader;
- }
+ for (i = 0, fp = ARTfields; i < standardoverview->count; i++) {
+ fp->NeedHeader = false;
for (hp = ARTheaders; hp < ARRAY_END(ARTheaders); hp++) {
- if (strcasecmp(buff, hp->Name) == 0) {
+ if (strcasecmp(standardoverview->strings[i], hp->Name) == 0) {
fp->Header = hp;
break;
}
}
if (hp == ARRAY_END(ARTheaders)) {
syslog(L_ERROR, "%s bad_schema unknown header \"%s\"",
- LogName, buff);
+ LogName, standardoverview->strings[i]);
ok = false;
continue;
}
fp++;
}
+ for (i = 0; i < extraoverview->count; i++) {
+ fp->NeedHeader = true;
+ for (hp = ARTheaders; hp < ARRAY_END(ARTheaders); hp++) {
+ if (strcasecmp(extraoverview->strings[i], hp->Name) == 0) {
+ fp->Header = hp;
+ break;
+ }
+ }
+ if (hp == ARRAY_END(ARTheaders)) {
+ syslog(L_ERROR, "%s bad_schema unknown header \"%s\"",
+ LogName, extraoverview->strings[i]);
+ ok = false;
+ continue;
+ }
+ fp++;
+ }
+
fp->Header = NULL;
- Fclose(F);
- if (!foundxref || !foundxreffull) {
- syslog(L_FATAL, "%s 'Xref:full' must be included in %s", LogName, SCHEMA);
- exit(1);
- }
return ok;
}
Modified: lib/innconf.c
===================================================================
--- lib/innconf.c 2009-04-11 17:49:27 UTC (rev 8408)
+++ lib/innconf.c 2009-04-11 22:13:41 UTC (rev 8409)
@@ -43,7 +43,8 @@
enum type {
TYPE_BOOLEAN,
TYPE_NUMBER,
- TYPE_STRING
+ TYPE_STRING,
+ TYPE_LIST
};
struct config {
@@ -54,6 +55,7 @@
bool boolean;
long integer;
const char *string;
+ const struct vector *list;
} defaults;
};
@@ -62,14 +64,16 @@
#define K(name) (#name), offsetof(struct innconf, name)
-#define BOOL(def) TYPE_BOOLEAN, { (def), 0, NULL }
-#define NUMBER(def) TYPE_NUMBER, { 0, (def), NULL }
-#define STRING(def) TYPE_STRING, { 0, 0, (def) }
+#define BOOL(def) TYPE_BOOLEAN, { (def), 0, NULL, NULL }
+#define NUMBER(def) TYPE_NUMBER, { 0, (def), NULL, NULL }
+#define STRING(def) TYPE_STRING, { 0, 0, (def), NULL }
+#define LIST(def) TYPE_LIST, { 0, 0, NULL, (def) }
/* Accessor macros to get a pointer to a value inside a struct. */
-#define CONF_BOOL(conf, offset) (bool *) (void *)((char *) (conf) + (offset))
-#define CONF_LONG(conf, offset) (long *) (void *)((char *) (conf) + (offset))
-#define CONF_STRING(conf, offset) (char **)(void *)((char *) (conf) + (offset))
+#define CONF_BOOL(conf, offset) (bool *) (void *)((char *) (conf) + (offset))
+#define CONF_LONG(conf, offset) (long *) (void *)((char *) (conf) + (offset))
+#define CONF_STRING(conf, offset) (char **) (void *)((char *) (conf) + (offset))
+#define CONF_LIST(conf, offset) (struct vector **)(void *)((char *) (conf) + (offset))
/* Special notes:
@@ -104,6 +108,8 @@
const struct config config_table[] = {
{ K(domain), STRING (NULL) },
{ K(enableoverview), BOOL (true) },
+ { K(extraoverviewadvertised), LIST (NULL) },
+ { K(extraoverviewhidden), LIST (NULL) },
{ K(fromhost), STRING (NULL) },
{ K(groupbaseexpiry), BOOL (true) },
{ K(mailcmd), STRING (NULL) },
@@ -348,6 +354,13 @@
if (innconf->mailcmd == NULL)
innconf->mailcmd = concatpath(innconf->pathbin, "innmail");
+ /* Create empty vectors of extra overview fields if they haven't already
+ * been created. */
+ if (innconf->extraoverviewadvertised == NULL)
+ innconf->extraoverviewadvertised = vector_new();
+ if (innconf->extraoverviewhidden == NULL)
+ innconf->extraoverviewhidden = vector_new();
+
/* Defaults used only if TLS (SSL) is supported. */
#ifdef HAVE_SSL
if (innconf->tlscertfile == NULL)
@@ -367,11 +380,13 @@
static struct innconf *
innconf_parse(struct config_group *group)
{
- unsigned int i;
+ unsigned int i, j;
bool *bool_ptr;
long *long_ptr;
const char *char_ptr;
char **string;
+ const struct vector *vector_ptr;
+ struct vector **list;
struct innconf *config;
config = xmalloc(sizeof(struct innconf));
@@ -393,6 +408,25 @@
string = CONF_STRING(config, config_table[i].location);
*string = (char_ptr == NULL) ? NULL : xstrdup(char_ptr);
break;
+ case TYPE_LIST:
+ /* vector_ptr contains the value taken from inn.conf or the
+ * default value from config_table; *list points to the inn.conf
+ * structure in memory for this parameter.
+ * We have to do a deep copy of vector_ptr because, like char_ptr,
+ * it is freed by config_free() called by other parts of INN. */
+ if (!config_param_list(group, config_table[i].name, &vector_ptr))
+ vector_ptr = config_table[i].defaults.list;
+ list = CONF_LIST(config, config_table[i].location);
+ *list = vector_new();
+ if (vector_ptr != NULL && vector_ptr->strings != NULL) {
+ vector_resize(*list, vector_ptr->count);
+ for (j = 0; j < vector_ptr->count; j++) {
+ if (vector_ptr->strings[j] != NULL) {
+ vector_add(*list, vector_ptr->strings[j]);
+ }
+ }
+ }
+ break;
default:
die("internal error: invalid type in row %u of config table", i);
break;
@@ -432,6 +466,7 @@
warn("ovmethod must be set in inn.conf if enableoverview is true");
okay = false;
}
+
threshold = innconf->datamovethreshold;
if (threshold <= 0 || threshold > 1024 * 1024) {
config_error_param(group, "datamovethreshold",
@@ -553,13 +588,20 @@
{
unsigned int i;
char *p;
+ struct vector *q;
- for (i = 0; i < ARRAY_SIZE(config_table); i++)
+ for (i = 0; i < ARRAY_SIZE(config_table); i++) {
if (config_table[i].type == TYPE_STRING) {
p = *CONF_STRING(config, config_table[i].location);
if (p != NULL)
free(p);
}
+ if (config_table[i].type == TYPE_LIST) {
+ q = *CONF_LIST(config, config_table[i].location);
+ if (q != NULL)
+ vector_free(q);
+ }
+ }
free(config);
}
@@ -679,6 +721,94 @@
/*
+** Print a single list value with appropriate quoting.
+*/
+static void
+print_list(FILE *file, const char *key, const struct vector *value,
+ enum innconf_quoting quoting)
+{
+ char *upper, *p;
+ const char *letter;
+ unsigned int i;
+ static const char tcl_unsafe[] = "$[]{}\"\\";
+
+ switch (quoting) {
+ case INNCONF_QUOTE_NONE:
+ fprintf(file, "[ ");
+ if (value != NULL && value->strings != NULL) {
+ for (i = 0; i < value->count; i++) {
+ /* No separation between strings. */
+ fprintf(file, "%s ",
+ value->strings[i] != NULL ? value->strings[i] : "");
+ }
+ }
+ fprintf(file, "]\n");
+ break;
+ case INNCONF_QUOTE_SHELL:
+ upper = xstrdup(key);
+ for (p = upper; *p != '\0'; p++)
+ *p = toupper(*p);
+ fprintf(file, "%s=( ", upper);
+ if (value != NULL && value->strings != NULL) {
+ for (i = 0; i < value->count; i++) {
+ fprintf(file, "'");
+ for (letter = value->strings[i]; letter != NULL
+ && *letter != '\0'; letter++) {
+ if (*letter == '\'')
+ fputs("'\\''", file);
+ else if (*letter == '\\')
+ fputs("\\\\", file);
+ else
+ fputc(*letter, file);
+ }
+ fprintf(file, "' ");
+ }
+ }
+ fprintf(file, "); export %s;\n", upper);
+ free(upper);
+ break;
+ case INNCONF_QUOTE_PERL:
+ fprintf(file, "@%s = ( ", key);
+ if (value != NULL && value->strings != NULL) {
+ for (i = 0; i < value->count; i++) {
+ fprintf(file, "'");
+ for (letter = value->strings[i]; letter != NULL
+ && *letter != '\0'; letter++) {
+ if (*letter == '\'' || *letter == '\\')
+ fputc('\\', file);
+ fputc(*letter, file);
+ }
+ if (i == value->count - 1) {
+ fprintf(file, "' ");
+ } else {
+ fprintf(file, "', ");
+ }
+ }
+ }
+ fprintf(file, ");\n");
+ break;
+ case INNCONF_QUOTE_TCL:
+ fprintf(file, "set inn_%s { ", key);
+ if (value != NULL && value->strings != NULL) {
+ for (i = 0; i < value->count; i++) {
+ fprintf(file, "\"");
+ for (letter = value->strings[i]; letter != NULL
+ && *letter != '\0'; letter++) {
+ if (strchr(tcl_unsafe, *letter) != NULL)
+ fputc('\\', file);
+ fputc(*letter, file);
+ }
+ fprintf(file, "\" ");
+ }
+ }
+ fprintf(file, "}\n");
+ break;
+ }
+}
+
+
+
+/*
** Print a single parameter to the given file. Take an index into the table
** specifying the attribute to print and the quoting.
*/
@@ -688,6 +818,7 @@
bool bool_val;
long long_val;
const char *string_val;
+ const struct vector *list_val;
switch (config_table[i].type) {
case TYPE_BOOLEAN:
@@ -702,6 +833,10 @@
string_val = *CONF_STRING(innconf, config_table[i].location);
print_string(file, config_table[i].name, string_val, quoting);
break;
+ case TYPE_LIST:
+ list_val = *CONF_LIST(innconf, config_table[i].location);
+ print_list(file, config_table[i].name, list_val, quoting);
+ break;
default:
die("internal error: invalid type in row %lu of config table",
(unsigned long) i);
@@ -750,10 +885,11 @@
bool
innconf_compare(struct innconf *conf1, struct innconf *conf2)
{
- unsigned int i;
+ unsigned int i, j;
bool bool1, bool2;
long long1, long2;
const char *string1, *string2;
+ const struct vector *list1, *list2;
bool okay = true;
for (i = 0; i < ARRAY_SIZE(config_table); i++)
@@ -795,6 +931,57 @@
}
}
break;
+ case TYPE_LIST:
+ list1 = *CONF_LIST(conf1, config_table[i].location);
+ list2 = *CONF_LIST(conf2, config_table[i].location);
+ /* Vectors are not resized when created in inn.conf.
+ * Therefore, we can compare their length. */
+ if ( (list1 == NULL && list2 != NULL)
+ || (list1 != NULL && list2 == NULL)) {
+ warn("list variable %s differs: one is NULL",
+ config_table[i].name);
+ okay = false;
+ } else if (list1 != NULL && list2 != NULL) {
+ if ( (list1->strings == NULL && list2->strings != NULL)
+ || (list1->strings != NULL && list2->strings == NULL)) {
+ warn("list strings variable %s differs: one is NULL",
+ config_table[i].name);
+ okay = false;
+ } else if (list1->strings != NULL && list2->strings != NULL) {
+ if (list1->count != list2->count) {
+ warn("list variable %s differs in length: %lu != %lu",
+ config_table[i].name, (unsigned long) list1->count,
+ (unsigned long) list2->count);
+ okay = false;
+ } else {
+ for (j = 0; j < list1->count; j++) {
+ if (list1->strings[j] == NULL
+ && list2->strings[j] != NULL) {
+ warn("list variable %s differs: NULL != %s",
+ config_table[i].name, list2->strings[j]);
+ okay = false;
+ break;
+ } else if (list1->strings[j] != NULL
+ && list2->strings[j] == NULL) {
+ warn("list variable %s differs: %s != NULL",
+ config_table[i].name, list1->strings[j]);
+ okay = false;
+ break;
+ } else if (list1->strings[j] != NULL
+ && list2->strings[j] != NULL) {
+ if (strcmp(list1->strings[j], list2->strings[j]) != 0) {
+ warn("list variable %s differs at element %u: %s != %s",
+ config_table[i].name, j+1,
+ list1->strings[j], list2->strings[j]);
+ okay = false;
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ break;
default:
die("internal error: invalid type in row %d of config table", i);
break;
Modified: nnrpd/list.c
===================================================================
--- nnrpd/list.c 2009-04-11 17:49:27 UTC (rev 8408)
+++ nnrpd/list.c 2009-04-11 22:13:41 UTC (rev 8409)
@@ -87,7 +87,7 @@
const struct cvector *standard;
unsigned int i;
- Reply("%d %s.\r\n", NNTP_OK_LIST, lp->Format);
+ Reply("%d %s\r\n", NNTP_OK_LIST, lp->Format);
standard = overview_fields();
for (i = 0; i < standard->count; ++i) {
Printf("%s:\r\n", standard->strings[i]);
@@ -234,9 +234,7 @@
if (qp == NULL) {
Reply("%d No list of %s available\r\n",
NNTP_ERR_UNAVAILABLE, lp->Items);
- /* Only the active, newsgroups and overview.fmt files are required
- * (but the last one has already called cmd_list_schema).
- * LIST HEADERS has also called its own function cmd_list_headers. */
+ /* Only the active and newsgroups files are required. */
if (lp->Required || errno != ENOENT) {
/* %m outputs strerror(errno). */
syslog(L_ERROR, "%s can't fopen %s %m", Client.host, lp->File);
Modified: nnrpd/nnrpd.c
===================================================================
--- nnrpd/nnrpd.c 2009-04-11 17:49:27 UTC (rev 8408)
+++ nnrpd/nnrpd.c 2009-04-11 22:13:41 UTC (rev 8409)
@@ -799,7 +799,7 @@
Reply("%d NNTP server unavailable. Try later!\r\n", NNTP_FAIL_TERMINATING);
ExitWithStats(1, true);
}
- OVextra = overview_extra_fields();
+ OVextra = overview_extra_fields(false);
if (OVextra == NULL) {
/* Overview_extra_fields should already have logged something
* useful. */
Modified: samples/inn.conf.in
===================================================================
--- samples/inn.conf.in 2009-04-11 17:49:27 UTC (rev 8408)
+++ samples/inn.conf.in 2009-04-11 22:13:41 UTC (rev 8409)
@@ -63,6 +63,8 @@
cnfscheckfudgesize: 0
enableoverview: true
+extraoverviewadvertised: [ ]
+extraoverviewhidden: [ ]
groupbaseexpiry: true
mergetogroups: false
nfswriter: false
Deleted: samples/overview.fmt
===================================================================
--- samples/overview.fmt 2009-04-11 17:49:27 UTC (rev 8408)
+++ samples/overview.fmt 2009-04-11 22:13:41 UTC (rev 8409)
@@ -1,16 +0,0 @@
-## $Revision$
-## overview.fmt - format of news overview database
-## Format
-## <header>
-## <header>:full
-## header is a news article header, known by innd. If ":full" appears,
-## then header name will be prepended. Order of lines is important!
-Subject:
-From:
-Date:
-Message-ID:
-References:
-Bytes:
-Lines:
-Xref:full
-#Keywords:full
Property changes on: trunk/site
___________________________________________________________________
Name: svn:ignore
- INN.py
active.minimal
actsync.cfg
actsync.ign
buffindexed.conf
config
control.ctl
control.ctl.local
cycbuff.conf
distrib.pats
distributions
expire.ctl
filter.tcl
filter_innd.pl
filter_innd.py
filter_nnrpd.pl
incoming.conf
inn.conf
innfeed.conf
innreport.conf
innreport.css
innwatch.ctl
localgroups
moderators
motd.news
news2mail.cf
newsfeeds
newsgroups.minimal
nnrpd.py
nnrpd.track
nnrpd_access.pl
nnrpd_auth.pl
nnrpd_auth.py
nnrpd_access.py
nnrpd_dynamic.py
nntpsend.ctl
nocem.ctl
ovdb.conf
overview.fmt
passwd.nntp
radius.conf
readers.conf
sasl.conf
send-uucp.cf
startup.tcl
startup_innd.pl
storage.conf
subscriptions
update
+ INN.py
active.minimal
actsync.cfg
actsync.ign
buffindexed.conf
config
control.ctl
control.ctl.local
cycbuff.conf
distrib.pats
distributions
expire.ctl
filter.tcl
filter_innd.pl
filter_innd.py
filter_nnrpd.pl
incoming.conf
inn.conf
innfeed.conf
innreport.conf
innreport.css
innwatch.ctl
localgroups
moderators
motd.news
news2mail.cf
newsfeeds
newsgroups.minimal
nnrpd.py
nnrpd.track
nnrpd_access.pl
nnrpd_auth.pl
nnrpd_auth.py
nnrpd_access.py
nnrpd_dynamic.py
nntpsend.ctl
nocem.ctl
ovdb.conf
passwd.nntp
radius.conf
readers.conf
sasl.conf
send-uucp.cf
startup.tcl
startup_innd.pl
storage.conf
subscriptions
update
Modified: site/Makefile
===================================================================
--- site/Makefile 2009-04-11 17:49:27 UTC (rev 8408)
+++ site/Makefile 2009-04-11 22:13:41 UTC (rev 8409)
@@ -36,7 +36,6 @@
PATH_NEWSFEEDS = ${PATHETC}/newsfeeds
PATH_READERSCONF = ${PATHETC}/readers.conf
PATH_NNRPDTRACK = ${PATHETC}/nnrpd.track
-PATH_SCHEMA = ${PATHETC}/overview.fmt
PATH_NNTPPASS = ${PATHETC}/passwd.nntp
PATH_CTLWATCH = ${PATHETC}/innwatch.ctl
PATH_ACTSYNC_IGN = ${PATHETC}/actsync.ign
@@ -60,7 +59,7 @@
REST = \
newsfeeds incoming.conf nnrpd.track passwd.nntp \
inn.conf moderators innreport.conf innreport.css localgroups \
- control.ctl control.ctl.local expire.ctl nntpsend.ctl overview.fmt \
+ control.ctl control.ctl.local expire.ctl nntpsend.ctl \
innwatch.ctl distrib.pats distributions actsync.cfg actsync.ign \
motd.news storage.conf cycbuff.conf buffindexed.conf \
innfeed.conf startup_innd.pl filter_innd.pl filter_nnrpd.pl \
@@ -81,10 +80,10 @@
$D$(PATHETC)/nntpsend.ctl \
$D$(PATHETC)/innreport.conf $D$(PATHHTTP)/innreport.css \
$D$(PATHETC)/localgroups \
- $D$(PATH_CTLWATCH) $D$(PATH_DISTPATS) $D$(PATH_SCHEMA) \
+ $D$(PATH_CTLWATCH) $D$(PATH_DISTPATS) \
$D$(PATH_ACTSYNC_CFG) $D$(PATH_ACTSYNC_IGN) \
$D$(PATH_MOTD) $D$(PATH_STORAGECONF) \
- $D$(PATH_OVERVIEWCTL) $D$(PATH_CYCBUFFCONFIG) $D$(PATH_BUFFINDEXED) \
+ $D$(PATH_CYCBUFFCONFIG) $D$(PATH_BUFFINDEXED) \
$D$(PATH_INNFEEDCTL) $D$(PATH_PERL_STARTUP_INND) \
$D$(PATH_PERL_FILTER_INND) $D$(PATH_PERL_FILTER_NNRPD) \
$D$(PATH_PYTHON_FILTER_INND) $D$(PATH_PYTHON_INN_MODULE) \
@@ -182,7 +181,6 @@
$D$(PATH_READERSCONF): readers.conf ; $(COPY_RPUB) $? $@
$D$(PATH_RADIUS_CONF): radius.conf ; $(COPY_RPRI) $? $@
$D$(PATH_NNRPDTRACK): nnrpd.track ; $(COPY_RPUB) $? $@
-$D$(PATH_SCHEMA): overview.fmt ; $(COPY_RPUB) $? $@
$D$(PATH_CONTROLCTL): control.ctl ; $(COPY_RPUB) $? $@
$D$(PATH_CONTROLCTLLOCAL): control.ctl.local ; $(COPY_RPUB) $? $@
$D$(PATH_CTLWATCH): innwatch.ctl ; $(COPY_RPUB) $? $@
@@ -268,7 +266,6 @@
nnrpd.track: ../samples/nnrpd.track ; $(COPY) $? $@
nntpsend.ctl: ../samples/nntpsend.ctl ; $(COPY) $? $@
nocem.ctl: ../samples/nocem.ctl ; $(COPY) $? $@
-overview.fmt: ../samples/overview.fmt ; $(COPY) $? $@
parsecontrol: ../samples/parsecontrol ; $(COPY) $? $@
passwd.nntp: ../samples/passwd.nntp ; $(COPY) $? $@
readers.conf: ../samples/readers.conf ; $(COPY) $? $@
Modified: storage/Makefile
===================================================================
--- storage/Makefile 2009-04-11 17:49:27 UTC (rev 8408)
+++ storage/Makefile 2009-04-11 22:13:41 UTC (rev 8409)
@@ -104,7 +104,7 @@
../include/config.h ../include/inn/innconf.h ../include/inn/defines.h \
../include/inn/libinn.h ../include/inn/ov.h ../include/inn/storage.h \
../include/inn/history.h ovinterface.h ../include/inn/history.h \
- ../include/inn/storage.h ../include/inn/paths.h
+ ../include/inn/storage.h ../include/inn/paths.h ../include/inn/vector.h
interface.o: interface.c ../include/config.h ../include/inn/defines.h \
../include/inn/system.h ../include/inn/options.h ../include/clibrary.h \
../include/config.h ../include/conffile.h ../include/inn/innconf.h \
@@ -128,11 +128,11 @@
overdata.o: overdata.c ../include/config.h ../include/inn/defines.h \
../include/inn/system.h ../include/inn/options.h ../include/clibrary.h \
../include/config.h ../include/inn/buffer.h ../include/inn/defines.h \
- ../include/inn/innconf.h ../include/inn/messages.h ../include/inn/qio.h \
+ ../include/inn/innconf.h ../include/inn/messages.h \
../include/inn/wire.h ../include/inn/vector.h ../include/inn/libinn.h \
- ../include/inn/ov.h ../include/inn/storage.h ../include/inn/history.h \
- ovinterface.h ../include/inn/history.h ../include/inn/storage.h \
- ../include/inn/paths.h
+ ovinterface.h ../include/inn/history.h ../include/inn/ov.h \
+ ../include/inn/storage.h ../include/inn/history.h \
+ ../include/inn/storage.h
overview.o: overview.c ../include/config.h ../include/inn/defines.h \
../include/inn/system.h ../include/inn/options.h ../include/clibrary.h \
../include/config.h ../include/inn/buffer.h ../include/inn/defines.h \
Modified: storage/expire.c
===================================================================
--- storage/expire.c 2009-04-11 17:49:27 UTC (rev 8408)
+++ storage/expire.c 2009-04-11 22:13:41 UTC (rev 8409)
@@ -19,6 +19,7 @@
#include "ovinterface.h"
#include "inn/paths.h"
#include "inn/storage.h"
+#include "inn/vector.h"
enum KRP {Keep, Remove, Poison};
@@ -545,56 +546,34 @@
static void
ARTreadschema(void)
{
- FILE *F;
- char *p;
- char *path;
+ const struct cvector *standardoverview;
+ const struct vector *extraoverview;
ARTOVERFIELD *fp;
- int i;
- char buff[SMBUF];
- bool foundxref = false;
- bool foundxreffull = false;
+ unsigned int i;
- /* Open file, count lines. */
- path = concatpath(innconf->pathetc, INN_PATH_SCHEMA);
- F = fopen(path, "r");
- if (F == NULL)
- return;
- for (i = 0; fgets(buff, sizeof buff, F) != NULL; i++)
- continue;
- fseeko(F, 0, SEEK_SET);
- ARTfields = xmalloc((i + 1) * sizeof(ARTOVERFIELD));
+ /* Count the number of overview fields and allocate ARTfields. */
+ standardoverview = overview_fields();
+ extraoverview = overview_extra_fields(true);
+ ARTfields = xmalloc((standardoverview->count + extraoverview->count + 1)
+ * sizeof(ARTOVERFIELD));
/* Parse each field. */
- for (fp = ARTfields; fgets(buff, sizeof buff, F) != NULL; ) {
- /* Ignore blank and comment lines. */
- if ((p = strchr(buff, '\n')) != NULL)
- *p = '\0';
- if ((p = strchr(buff, '#')) != NULL)
- *p = '\0';
- if (buff[0] == '\0')
- continue;
- if ((p = strchr(buff, ':')) != NULL) {
- *p++ = '\0';
- fp->NeedsHeader = (strcmp(p, "full") == 0);
- }
- else
- fp->NeedsHeader = false;
+ for (i = 0, fp = ARTfields; i < standardoverview->count; i++) {
+ fp->NeedsHeader = false;
fp->HasHeader = false;
- fp->Header = xstrdup(buff);
- fp->Length = strlen(buff);
- if (strcasecmp(buff, "Xref") == 0) {
- foundxref = true;
- foundxreffull = fp->NeedsHeader;
- }
+ fp->Header = xstrdup(standardoverview->strings[i]);
+ fp->Length = strlen(standardoverview->strings[i]);
fp++;
}
- ARTfieldsize = fp - ARTfields;
- fclose(F);
- if (!foundxref || !foundxreffull) {
- fprintf(stderr, "'Xref:full' must be included in %s", path);
- exit(1);
+ for (i = 0; i < extraoverview->count; i++) {
+ fp->NeedsHeader = true;
+ fp->HasHeader = false;
+ fp->Header = xstrdup(extraoverview->strings[i]);
+ fp->Length = strlen(extraoverview->strings[i]);
+ fp++;
}
- free(path);
+
+ ARTfieldsize = fp - ARTfields;
}
/*
@@ -652,7 +631,7 @@
}
/*
-** Read overview.fmt and find index for headers
+** Read overview schema and find index for headers.
*/
static void
OVfindheaderindex(void)
Modified: storage/overdata.c
===================================================================
--- storage/overdata.c 2009-04-11 17:49:27 UTC (rev 8408)
+++ storage/overdata.c 2009-04-11 22:13:41 UTC (rev 8409)
@@ -13,23 +13,20 @@
#include "inn/buffer.h"
#include "inn/innconf.h"
#include "inn/messages.h"
-#include "inn/qio.h"
#include "inn/wire.h"
#include "inn/vector.h"
#include "inn/libinn.h"
-#include "inn/ov.h"
#include "ovinterface.h"
-#include "inn/paths.h"
-/* The standard overview fields. */
+/* The standard overview fields. The order of these fields is important. */
static const char * const fields[] = {
"Subject", "From", "Date", "Message-ID", "References", "Bytes", "Lines"
};
/*
-** Return a vector of the standard overview fields. Note there is no
+** Return a vector of the standard overview fields. Note there is no
** way to free up the resulting data structure.
*/
const struct cvector *
@@ -51,71 +48,50 @@
}
/*
-** Parse the overview schema and return a vector of the additional fields
-** over the standard ones. Caller is responsible for freeing the vector.
+** Return a vector of the additional fields over the standard ones.
+** The order of these fields is important.
+**
+** Xref: is mandatory for INN and we make it the first extra field
+** after the seven overview fields defined in RFC 3977.
+**
+** Caller is responsible for freeing the vector.
*/
struct vector *
-overview_extra_fields(void)
+overview_extra_fields(bool hidden)
{
struct vector *list = NULL;
- struct vector *result = NULL;
- char *schema = NULL;
- char *line, *p;
- QIOSTATE *qp = NULL;
- unsigned int field;
- bool full = false;
+ unsigned int i;
- schema = concatpath(innconf->pathetc, INN_PATH_SCHEMA);
- qp = QIOopen(schema);
- if (qp == NULL) {
- syswarn("cannot open %s", schema);
- goto done;
- }
list = vector_new();
- for (field = 0, line = QIOread(qp); line != NULL; line = QIOread(qp)) {
- while (ISWHITE(*line))
- line++;
- p = strchr(line, '#');
- if (p != NULL)
- *p = '\0';
- p = strchr(line, '\n');
- if (p != NULL)
- *p = '\0';
- if (*line == '\0')
- continue;
- p = strchr(line, ':');
- if (p != NULL) {
- *p++ = '\0';
- full = (strcmp(p, "full") == 0);
+
+ if (hidden) {
+ vector_resize(list, innconf->extraoverviewadvertised->count
+ + innconf->extraoverviewhidden->count + 1);
+ } else {
+ vector_resize(list, innconf->extraoverviewadvertised->count + 1);
+ }
+
+ vector_add(list, "Xref");
+
+ if (innconf->extraoverviewadvertised->strings != NULL) {
+ for (i = 0; i < innconf->extraoverviewadvertised->count; i++) {
+ if (innconf->extraoverviewadvertised->strings[i] != NULL) {
+ vector_add(list, innconf->extraoverviewadvertised->strings[i]);
+ }
}
- if (field >= ARRAY_SIZE(fields)) {
- if (!full)
- warn("additional field %s not marked with :full", line);
- vector_add(list, line);
- } else {
- if (strcasecmp(line, fields[field]) != 0)
- warn("field %d is %s, should be %s", field, line,
- fields[field]);
- }
- field++;
}
- if (QIOerror(qp)) {
- if (QIOtoolong(qp)) {
- warn("line too long in %s", schema);
- } else {
- syswarn("error while reading %s", schema);
+
+ if (hidden) {
+ if (innconf->extraoverviewhidden->strings != NULL) {
+ for (i = 0; i < innconf->extraoverviewhidden->count; i++) {
+ if (innconf->extraoverviewhidden->strings[i] != NULL) {
+ vector_add(list, innconf->extraoverviewhidden->strings[i]);
+ }
+ }
}
}
- result = list;
-done:
- if (schema != NULL)
- free(schema);
- if (qp != NULL)
- QIOclose(qp);
- if (result == NULL && list != NULL)
- vector_free(list);
- return result;
+ return list;
}
@@ -267,7 +243,7 @@
/*
** Check the given overview data and make sure it's well-formed. Extension
-** headers are not checked against overview.fmt (having a different set of
+** headers are not checked against LIST OVERVIEW.FMT (having a different set of
** extension headers doesn't make the data invalid), but the presence of the
** standard fields is checked. Also checked is whether the article number in
** the data matches the passed article number. Returns true if the data is
Modified: storage/tradindexed/tdx-util.c
===================================================================
--- storage/tradindexed/tdx-util.c 2009-04-11 17:49:27 UTC (rev 8408)
+++ storage/tradindexed/tdx-util.c 2009-04-11 22:13:41 UTC (rev 8409)
@@ -315,7 +315,7 @@
sysdie("cannot open history %s", histpath);
free(histpath);
- extra = overview_extra_fields();
+ extra = overview_extra_fields(true);
files = article_list(path);
info.count = 0;
Modified: support/mkmanifest
===================================================================
--- support/mkmanifest 2009-04-11 17:49:27 UTC (rev 8408)
+++ support/mkmanifest 2009-04-11 22:13:41 UTC (rev 8409)
@@ -239,7 +239,6 @@
site/nntpsend.ctl
site/nocem.ctl
site/ovdb.conf
-site/overview.fmt
site/passwd.nntp
site/radius.conf
site/readers.conf
Modified: tests/data/etc/inn.conf
===================================================================
--- tests/data/etc/inn.conf 2009-04-11 17:49:27 UTC (rev 8408)
+++ tests/data/etc/inn.conf 2009-04-11 22:13:41 UTC (rev 8409)
@@ -1,12 +1,13 @@
# inn.conf -- Minimal inn.conf for testing the storage subsystem.
# $Id$
-domain: news.example.com
-mta: "/usr/sbin/sendmail -oi -oem %s"
-hismethod: hisv6
-enableoverview: false
-wireformat: true
+domain: news.example.com
+mta: "/usr/sbin/sendmail -oi -oem %s"
+hismethod: hisv6
+enableoverview: false
+extraoverviewhidden: [ Path ]
+wireformat: true
-pathnews: .
-patharchive: archive
-patharticles: spool
+pathnews: .
+patharchive: archive
+patharticles: spool
Deleted: tests/data/etc/overview.fmt
===================================================================
--- tests/data/etc/overview.fmt 2009-04-11 17:49:27 UTC (rev 8408)
+++ tests/data/etc/overview.fmt 2009-04-11 22:13:41 UTC (rev 8409)
@@ -1,12 +0,0 @@
-# overview.fmt -- overview.fmt for testing makehistory.
-# $Id$
-
-Subject:
-From:
-Date:
-Message-ID:
-References:
-Bytes:
-Lines:
-Xref:full
-Path:full
Modified: tests/data/upgrade/inn.conf.ok
===================================================================
--- tests/data/upgrade/inn.conf.ok 2009-04-11 17:49:27 UTC (rev 8408)
+++ tests/data/upgrade/inn.conf.ok 2009-04-11 22:13:41 UTC (rev 8409)
@@ -6,10 +6,13 @@
# testing comment
-# Added by innupgrade
+# Added by innupgrade.
hismethod: hisv6
-# Moved from sasl.conf by innupgrade
+# Moved from overview.fmt by innupgrade.
+extraoverviewadvertised: [ Path Keywords Injection-Info ]
+
+# Moved from sasl.conf by innupgrade.
tlscafile: /usr/local/news/lib/ca.cert
tlscapath: /usr/local/news/lib/cas
Added: tests/data/upgrade/overview.fmt
===================================================================
--- tests/data/upgrade/overview.fmt (rev 0)
+++ tests/data/upgrade/overview.fmt 2009-04-11 22:13:41 UTC (rev 8409)
@@ -0,0 +1,18 @@
+## overview.fmt for testing its migration to inn.conf.
+## $Id$
+
+Subject:
+
+From:
+Date:
+Message-ID:
+References:
+Bytes:
+Lines:
+Xref:full
+Path:full
+
+Keywords:full
+# A
+ # Comment.
+Injection-Info:full # Another comment.
Property changes on: trunk/tests/data/upgrade/overview.fmt
___________________________________________________________________
Name: svn:keywords
+ Author Date Id Revision
Name: svn:eol-style
+ native
Modified: tests/innd/artparse-t.c
===================================================================
--- tests/innd/artparse-t.c 2009-04-11 17:49:27 UTC (rev 8408)
+++ tests/innd/artparse-t.c 2009-04-11 22:13:41 UTC (rev 8409)
@@ -7,6 +7,7 @@
#include "inn/buffer.h"
#include "inn/innconf.h"
#include "inn/messages.h"
+#include "inn/vector.h"
#include "inn/wire.h"
#include "inn/libinn.h"
#include "libtest.h"
@@ -63,6 +64,8 @@
innconf->logipaddr = false;
innconf->maxartsize = 8 * 1024;
innconf->pathetc = xstrdup("../data/etc");
+ innconf->extraoverviewadvertised = vector_new();
+ innconf->extraoverviewhidden = vector_new();
}
/* Create a fake channel with just enough data filled in to be able to use it
@@ -85,8 +88,8 @@
static void
initialize(void)
{
- if (access("../data/etc/overview.fmt", F_OK) < 0)
- if (access("data/etc/overview.fmt", F_OK) == 0)
+ if (access("../data/etc/storage.conf", F_OK) < 0)
+ if (access("data/etc/storage.conf", F_OK) == 0)
if (chdir("innd") != 0)
sysdie("Cannot cd to innd");
fake_innconf();
Modified: tests/lib/innconf-t.c
===================================================================
--- tests/lib/innconf-t.c 2009-04-11 17:49:27 UTC (rev 8408)
+++ tests/lib/innconf-t.c 2009-04-11 22:13:41 UTC (rev 8409)
@@ -8,8 +8,9 @@
#include "inn/messages.h"
#include "libtest.h"
+/* We will have strings, integers, bools and lists. */
static const char grep[] =
-"egrep 'mta|organization|ovmethod|hismethod|path|pgpverify'\
+"egrep 'mta|organization|ovmethod|hismethod|path|port|overview|pgpverify'\
../../samples/inn.conf > config/tmp";
int
@@ -55,7 +56,7 @@
fclose(config);
ok(7, !innconf_check("config/tmp"));
unlink("config/tmp");
- ok_string(8, "config/tmp:27: unknown parameter foo\n", errors);
+ ok_string(8, "config/tmp:36: unknown parameter foo\n", errors);
errors_uncapture();
free(errors);
errors = NULL;
More information about the inn-committers
mailing list