INN commit: trunk (doc/pod/news.pod doc/pod/sm.pod frontends/sm.c)
INN Commit
rra at isc.org
Sat Nov 21 07:01:49 UTC 2020
Date: Friday, November 20, 2020 @ 23:01:48
Author: iulius
Revision: 10404
sm: support reading and storing wire-format articles
Any number of articles can now be given in wire format to sm
on its standard input when both "-s" and "-R" are used. Only native
format was previously possible.
This might be useful for certain types of internal migrations.
Thanks to Bo Lindbergh for the patch.
Modified:
trunk/doc/pod/news.pod
trunk/doc/pod/sm.pod
trunk/frontends/sm.c
------------------+
doc/pod/news.pod | 6 ++
doc/pod/sm.pod | 23 +++++---
frontends/sm.c | 141 ++++++++++++++++++++++++++++++++++++++++++++---------
3 files changed, 139 insertions(+), 31 deletions(-)
Modified: doc/pod/news.pod
===================================================================
--- doc/pod/news.pod 2020-11-15 21:11:33 UTC (rev 10403)
+++ doc/pod/news.pod 2020-11-21 07:01:48 UTC (rev 10404)
@@ -26,6 +26,12 @@
=item *
+B<sm> can now read and store any number of articles given in wire format
+on its standard input when both B<-s> and B<-R> are used. Only native
+format was previously possible. Thanks to Bo Lindbergh for the patch.
+
+=item *
+
Added new B<-a> flag to B<rnews> to disallow, if needed, the use
of additional unpackers from the F<rnews.libexec> sub-directory of
I<pathbin> (as set in F<inn.conf>); only C<rnews> and C<cunbatch>
Modified: doc/pod/sm.pod
===================================================================
--- doc/pod/sm.pod 2020-11-15 21:11:33 UTC (rev 10403)
+++ doc/pod/sm.pod 2020-11-21 07:01:48 UTC (rev 10404)
@@ -26,11 +26,12 @@
standard output the corresponding article for each token given.
If B<-s> is given, B<sm> instead stores the article given on standard
-input (in native format, not wire format) using the standard rules of the
-storage subsystem. If the article is stored successfully, the token of
-the article is printed to standard output. Please note that this does not
-make any attempt to write a history entry or any overview data, and is
-therefore only useful under very specific circumstances.
+input (in native format, unless B<-R> is given, in which case wire
+format is expected) using the standard rules of the storage subsystem.
+If the article is stored successfully, the token of the article is
+printed to standard output. Please note that this does not make any
+attempt to write a history entry or any overview data, and is therefore
+only useful under very specific circumstances.
=head1 OPTIONS
@@ -73,6 +74,8 @@
will still be escaped for sending over NNTP, and the article will end in
a CRLF.CRLF sequence.
+When used with B<-s>, read articles in wire format.
+
=item B<-S>
Write the article to standard output in the format used by B<rnews> spool
@@ -83,10 +86,12 @@
=item B<-s>
-Store the article given on standard input using the normal storage rules
-for articles as configured in storage.conf(5). Print the new token for
-the message to standard output if it is stored successfully. If this
-option is given, no other options except possibly B<-q> should be given.
+Store the article given on standard input using the normal storage
+rules for articles as configured in storage.conf(5). Print the new
+token for the message to standard output if it is stored successfully.
+If this option is given, no other options except B<-R> and possibly B<-q>
+should be given. When B<-R> is given, any number of articles in wire
+format are read on standard input and stored.
=back
Modified: frontends/sm.c
===================================================================
--- frontends/sm.c 2020-11-15 21:11:33 UTC (rev 10403)
+++ frontends/sm.c 2020-11-21 07:01:48 UTC (rev 10404)
@@ -15,6 +15,8 @@
#include "inn/libinn.h"
#include "inn/storage.h"
+#define WIRE_CHUNK_SIZE 0x10000
+
static const char usage[] = "\
Usage: sm [-cdHiqrRSs] [token ...]\n\
\n\
@@ -29,8 +31,9 @@
-i Translate tokens into newsgroup names and article numbers\n\
-q Suppress all error messages except usage\n\
-R Display the raw article rather than undoing wire format\n\
+ or consume wire format articles when storing\n\
-S Output articles in rnews batch file format\n\
- -s Store the article provided on stdin.\n";
+ -s Store the article provided on stdin\n";
/* The options that can be set on the command line, used to determine what to
do with each token. */
@@ -39,14 +42,13 @@
bool clearinfo; /* Show clear information about a token. */
bool delete; /* Delete articles instead of showing them. */
bool header; /* Display article headers only. */
- bool raw; /* Show the raw wire-format articles. */
+ bool raw; /* Display or consume wire-format articles. */
bool rnews; /* Output articles as rnews batch files. */
};
/*
-** Given a file descriptor, read a post from that file descriptor and then
-** store it. This basically duplicates what INN does, except that INN has to
+** This basically duplicates what INN does, except that INN has to
** do more complex things to add the Path header.
**
** Note that we make no attempt to add history or overview information, at
@@ -53,19 +55,13 @@
** least right now.
*/
static bool
-store_article(int fd)
+store_article_common(char *text, size_t size)
{
- struct buffer *article;
- size_t size;
- char *text, *start, *end;
+ char *start, *end;
ARTHANDLE handle = ARTHANDLE_INITIALIZER;
TOKEN token;
/* Build the basic article handle. */
- article = buffer_new();
- if (!buffer_read_file(article, fd))
- sysdie("cannot read article");
- text = wire_from_native(article->data, article->left, &size);
handle.type = TOKEN_EMPTY;
handle.data = text;
handle.iov = xmalloc(sizeof(struct iovec));
@@ -75,7 +71,6 @@
handle.len = size;
handle.arrived = 0;
handle.expires = 0;
- buffer_free(article);
/* Find the expiration time, if any. */
start = wire_findheader(text, size, "Expires", true);
@@ -118,8 +113,8 @@
/* Store the article. */
token = SMstore(handle);
- free(text);
free(handle.iov);
+
if (token.type == TOKEN_EMPTY) {
warn("failed to store article: %s", SMerrorstr);
return false;
@@ -131,6 +126,104 @@
/*
+** Given a file descriptor, read a post from that file descriptor and then
+** store it.
+*/
+static bool
+store_article(int fd)
+{
+ struct buffer *article;
+ size_t size;
+ char *text;
+ bool result;
+
+ article = buffer_new();
+ if (!buffer_read_file(article, fd)) {
+ sysdie("cannot read article");
+ }
+ text = wire_from_native(article->data, article->left, &size);
+ buffer_free(article);
+ result = store_article_common(text, size);
+ free(text);
+ return result;
+}
+
+
+/*
+** Given a file descriptor, read any number of posts in wire format
+** from that file descriptor and store them.
+*/
+static bool
+store_wire_articles(int fd)
+{
+ bool result = true;
+ bool skipping = false;
+ struct buffer *input;
+ size_t offset;
+ ssize_t got;
+
+ input = buffer_new();
+ buffer_resize(input, WIRE_CHUNK_SIZE);
+ offset = 0;
+ for (;;) {
+ got = buffer_read(input, fd);
+ if (got < 0) {
+ result = false;
+ syswarn("article read failed");
+ break;
+ }
+ if (got == 0)
+ break;
+
+ while (buffer_find_string(input, "\r\n.\r\n", offset, &offset)) {
+ size_t size;
+
+ size = offset + 5;
+ if (skipping) {
+ skipping = false;
+ } else {
+ char *text;
+
+ text = input->data + input->used;
+ if (!store_article_common(text, size))
+ result = false;
+ }
+ input->used += size;
+ input->left -= size;
+ offset = 0;
+ }
+
+ if (!skipping && input->used <= 0 && input->left >= input->size) {
+ if (input->size >= innconf->maxartsize) {
+ skipping = true;
+ result = false;
+ warn("article too large, skipping");
+ } else {
+ buffer_resize(input, input->size + WIRE_CHUNK_SIZE);
+ }
+ }
+ if (skipping && input->left > 4) {
+ input->used += input->left - 4;
+ input->left = 4;
+ }
+ if (input->left >= 4) {
+ offset = input->left - 4;
+ } else {
+ offset = 0;
+ }
+ if (input->used > 0)
+ buffer_compact(input);
+ }
+
+ if (input->left > 0)
+ warn("trailing garbage at end of input");
+
+ buffer_free(input);
+ return result;
+}
+
+
+/*
** Process a single token, performing the operations specified in the given
** options struct. Calls warn and die to display error messages; -q is
** implemented by removing all the warn and die error handlers.
@@ -251,12 +344,12 @@
die("-R cannot be used with -S");
if (options.header && options.rnews)
die("-H cannot be used with -S");
- if (store && (options.artinfo || options.delete || options.header))
- die("-s cannot be used with -i, -r, -d, -H, -R, or -S");
- if (store && (options.raw || options.rnews))
- die("-s cannot be used with -i, -r, -d, -H, -R, or -S");
- if (options.clearinfo && (options.artinfo || options.delete || options.header
- || options.raw || options.rnews || store))
+ if (store && (options.artinfo || options.delete
+ || options.header || options.rnews))
+ die("-s cannot be used with -i, -r, -d, -H, or -S");
+ if (options.clearinfo && (options.artinfo || options.delete
+ || options.header || options.raw
+ || options.rnews || store))
die("-c cannot be used with -i, -r, -d, -H, -R, -S, or -s");
/* Initialize the storage manager. If we're doing article deletions, we
@@ -270,9 +363,13 @@
if (!SMinit())
die("cannot initialize storage manager: %s", SMerrorstr);
- /* If we're storing an article, do that and then exit. */
+ /* If we're storing articles, do that and then exit. */
if (store) {
- status = store_article(fileno(stdin));
+ if (options.raw) {
+ status = store_wire_articles(fileno(stdin));
+ } else {
+ status = store_article(fileno(stdin));
+ }
exit(status ? 0 : 1);
}
More information about the inn-committers
mailing list