INN commit: branches/2.6 (3 files)

INN Commit rra at isc.org
Sat Nov 21 07:02:43 UTC 2020


    Date: Friday, November 20, 2020 @ 23:02:43
  Author: iulius
Revision: 10405

sm:  support reading and storing wire-format articles

Any number of articles can now be given in wire format to sm
on its standard input when both "-s" and "-R" are used.  Only native
format was previously possible.

This might be useful for certain types of internal migrations.

Thanks to Bo Lindbergh for the patch.

Modified:
  branches/2.6/doc/pod/news.pod
  branches/2.6/doc/pod/sm.pod
  branches/2.6/frontends/sm.c

------------------+
 doc/pod/news.pod |    6 ++
 doc/pod/sm.pod   |   23 +++++---
 frontends/sm.c   |  141 ++++++++++++++++++++++++++++++++++++++++++++---------
 3 files changed, 139 insertions(+), 31 deletions(-)

Modified: doc/pod/news.pod
===================================================================
--- doc/pod/news.pod	2020-11-21 07:01:48 UTC (rev 10404)
+++ doc/pod/news.pod	2020-11-21 07:02:43 UTC (rev 10405)
@@ -26,6 +26,12 @@
 
 =item *
 
+B<sm> can now read and store any number of articles given in wire format
+on its standard input when both B<-s> and B<-R> are used.  Only native
+format was previously possible.  Thanks to Bo Lindbergh for the patch.
+
+=item *
+
 Added new B<-a> flag to B<rnews> to disallow, if needed, the use
 of additional unpackers from the F<rnews.libexec> sub-directory of
 I<pathbin> (as set in F<inn.conf>); only C<rnews> and C<cunbatch>

Modified: doc/pod/sm.pod
===================================================================
--- doc/pod/sm.pod	2020-11-21 07:01:48 UTC (rev 10404)
+++ doc/pod/sm.pod	2020-11-21 07:02:43 UTC (rev 10405)
@@ -26,11 +26,12 @@
 standard output the corresponding article for each token given.
 
 If B<-s> is given, B<sm> instead stores the article given on standard
-input (in native format, not wire format) using the standard rules of the
-storage subsystem.  If the article is stored successfully, the token of
-the article is printed to standard output.  Please note that this does not
-make any attempt to write a history entry or any overview data, and is
-therefore only useful under very specific circumstances.
+input (in native format, unless B<-R> is given, in which case wire
+format is expected) using the standard rules of the storage subsystem.
+If the article is stored successfully, the token of the article is
+printed to standard output.  Please note that this does not make any
+attempt to write a history entry or any overview data, and is therefore
+only useful under very specific circumstances.
 
 =head1 OPTIONS
 
@@ -73,6 +74,8 @@
 will still be escaped for sending over NNTP, and the article will end in
 a CRLF.CRLF sequence.
 
+When used with B<-s>, read articles in wire format.
+
 =item B<-S>
 
 Write the article to standard output in the format used by B<rnews> spool
@@ -83,10 +86,12 @@
 
 =item B<-s>
 
-Store the article given on standard input using the normal storage rules
-for articles as configured in storage.conf(5).  Print the new token for
-the message to standard output if it is stored successfully.  If this
-option is given, no other options except possibly B<-q> should be given.
+Store the article given on standard input using the normal storage
+rules for articles as configured in storage.conf(5).  Print the new
+token for the message to standard output if it is stored successfully.
+If this option is given, no other options except B<-R> and possibly B<-q>
+should be given.  When B<-R> is given, any number of articles in wire
+format are read on standard input and stored.
 
 =back
 

Modified: frontends/sm.c
===================================================================
--- frontends/sm.c	2020-11-21 07:01:48 UTC (rev 10404)
+++ frontends/sm.c	2020-11-21 07:02:43 UTC (rev 10405)
@@ -15,6 +15,8 @@
 #include "inn/libinn.h"
 #include "inn/storage.h"
 
+#define WIRE_CHUNK_SIZE 0x10000
+
 static const char usage[] = "\
 Usage: sm [-cdHiqrRSs] [token ...]\n\
 \n\
@@ -29,8 +31,9 @@
     -i          Translate tokens into newsgroup names and article numbers\n\
     -q          Suppress all error messages except usage\n\
     -R          Display the raw article rather than undoing wire format\n\
+                or consume wire format articles when storing\n\
     -S          Output articles in rnews batch file format\n\
-    -s          Store the article provided on stdin.\n";
+    -s          Store the article provided on stdin\n";
 
 /* The options that can be set on the command line, used to determine what to
    do with each token. */
@@ -39,14 +42,13 @@
     bool clearinfo;             /* Show clear information about a token. */
     bool delete;                /* Delete articles instead of showing them. */
     bool header;                /* Display article headers only. */
-    bool raw;                   /* Show the raw wire-format articles. */
+    bool raw;                   /* Display or consume wire-format articles. */
     bool rnews;                 /* Output articles as rnews batch files. */
 };
 
 
 /*
-**  Given a file descriptor, read a post from that file descriptor and then
-**  store it.  This basically duplicates what INN does, except that INN has to
+**  This basically duplicates what INN does, except that INN has to
 **  do more complex things to add the Path header.
 **
 **  Note that we make no attempt to add history or overview information, at
@@ -53,19 +55,13 @@
 **  least right now.
 */
 static bool
-store_article(int fd)
+store_article_common(char *text, size_t size)
 {
-    struct buffer *article;
-    size_t size;
-    char *text, *start, *end;
+    char *start, *end;
     ARTHANDLE handle = ARTHANDLE_INITIALIZER;
     TOKEN token;
 
     /* Build the basic article handle. */
-    article = buffer_new();
-    if (!buffer_read_file(article, fd))
-        sysdie("cannot read article");
-    text = wire_from_native(article->data, article->left, &size);
     handle.type = TOKEN_EMPTY;
     handle.data = text;
     handle.iov = xmalloc(sizeof(struct iovec));
@@ -75,7 +71,6 @@
     handle.len = size;
     handle.arrived = 0;
     handle.expires = 0;
-    buffer_free(article);
 
     /* Find the expiration time, if any. */
     start = wire_findheader(text, size, "Expires", true);
@@ -118,8 +113,8 @@
 
     /* Store the article. */
     token = SMstore(handle);
-    free(text);
     free(handle.iov);
+
     if (token.type == TOKEN_EMPTY) {
         warn("failed to store article: %s", SMerrorstr);
         return false;
@@ -131,6 +126,104 @@
 
 
 /*
+**  Given a file descriptor, read a post from that file descriptor and then
+**  store it.
+*/
+static bool
+store_article(int fd)
+{
+    struct buffer *article;
+    size_t size;
+    char *text;
+    bool result;
+
+    article = buffer_new();
+    if (!buffer_read_file(article, fd)) {
+        sysdie("cannot read article");
+    }
+    text = wire_from_native(article->data, article->left, &size);
+    buffer_free(article);
+    result = store_article_common(text, size);
+    free(text);
+    return result;
+}
+
+
+/*
+**  Given a file descriptor, read any number of posts in wire format
+**  from that file descriptor and store them.
+*/
+static bool
+store_wire_articles(int fd)
+{
+    bool result = true;
+    bool skipping = false;
+    struct buffer *input;
+    size_t offset;
+    ssize_t got;
+
+    input = buffer_new();
+    buffer_resize(input, WIRE_CHUNK_SIZE);
+    offset = 0;
+    for (;;) {
+        got = buffer_read(input, fd);
+        if (got < 0) {
+            result = false;
+            syswarn("article read failed");
+            break;
+        }
+        if (got == 0)
+            break;
+
+        while (buffer_find_string(input, "\r\n.\r\n", offset, &offset)) {
+            size_t size;
+
+            size = offset + 5;
+            if (skipping) {
+                skipping = false;
+            } else {
+                char *text;
+ 
+                text = input->data + input->used;
+                if (!store_article_common(text, size))
+                    result = false;
+            }
+            input->used += size;
+            input->left -= size;
+            offset = 0;
+        }
+
+        if (!skipping && input->used <= 0 && input->left >= input->size) {
+            if (input->size >= innconf->maxartsize) {
+                skipping = true;
+                result = false;
+                warn("article too large, skipping");
+            } else {
+                buffer_resize(input, input->size + WIRE_CHUNK_SIZE);
+            }
+        }
+        if (skipping && input->left > 4) {
+            input->used += input->left - 4;
+            input->left = 4;
+        }
+        if (input->left >= 4) {
+            offset = input->left - 4;
+        } else {
+            offset = 0;
+        }
+        if (input->used > 0)
+            buffer_compact(input);
+    }
+
+    if (input->left > 0)
+        warn("trailing garbage at end of input");
+
+    buffer_free(input);
+    return result;
+}
+
+
+/*
 **  Process a single token, performing the operations specified in the given
 **  options struct.  Calls warn and die to display error messages; -q is
 **  implemented by removing all the warn and die error handlers.
@@ -251,12 +344,12 @@
         die("-R cannot be used with -S");
     if (options.header && options.rnews)
         die("-H cannot be used with -S");
-    if (store && (options.artinfo || options.delete || options.header))
-        die("-s cannot be used with -i, -r, -d, -H, -R, or -S");
-    if (store && (options.raw || options.rnews))
-        die("-s cannot be used with -i, -r, -d, -H, -R, or -S");
-    if (options.clearinfo && (options.artinfo || options.delete || options.header
-                              || options.raw || options.rnews || store))
+    if (store && (options.artinfo || options.delete
+                  || options.header || options.rnews))
+        die("-s cannot be used with -i, -r, -d, -H, or -S");
+    if (options.clearinfo && (options.artinfo || options.delete
+                              || options.header || options.raw
+                              || options.rnews || store))
         die("-c cannot be used with -i, -r, -d, -H, -R, -S, or -s");
 
     /* Initialize the storage manager.  If we're doing article deletions, we
@@ -270,9 +363,13 @@
     if (!SMinit())
         die("cannot initialize storage manager: %s", SMerrorstr);
 
-    /* If we're storing an article, do that and then exit. */
+    /* If we're storing articles, do that and then exit. */
     if (store) {
-        status = store_article(fileno(stdin));
+        if (options.raw) {
+            status = store_wire_articles(fileno(stdin));
+        } else {
+            status = store_article(fileno(stdin));
+        }
         exit(status ? 0 : 1);
     }
 



More information about the inn-committers mailing list