PATCH: Diablo-style hashfeeds

"Miquel van Smoorenburg" list-inn-workers at news.cistron.nl
Wed Jun 1 21:50:53 UTC 2005


I'm using INN to (help) feed a bunch of Diablo boxes. In the
diablo world, it is common to split one full feed over several
boxes using something called a "hashfeed". This patch adds
that functionality to innd.

( Russ, did you look at the innfeed-refcount-arthandle patch
  I sent some time ago yet ? )

inn-2.4.2-hashfeed.patch

diff -ruN inn-2.4.2/doc/pod/newsfeeds.pod inn-2.4.2-new/doc/pod/newsfeeds.pod
--- inn-2.4.2/doc/pod/newsfeeds.pod	2004-12-22 05:21:19.000000000 +0100
+++ inn-2.4.2-new/doc/pod/newsfeeds.pod	2005-05-03 23:54:12.000000000 +0200
@@ -382,6 +382,29 @@
 new process will run with.  This flag can be used to raise the priority to
 normal if you're using the I<nicekids> parameter in F<inn.conf>.
 
+=item B<Q> I<hashfeed>
+
+The hashfeed value for this site. The message-id of the article is
+hashed using a quick (and dirty) hash method (31-bits sum of the value
+of all characters). The hashfeed value must be given as value/mod
+or as start-end/mod. If the message-id hash modulus "mod" plus one
+equals "number" or is between "start" and "end", the article will be
+fed to the site.
+
+Example:
+
+    Q1/2     Feeds about 50% of the messages to this site
+    Q2/2     Feeds the other 50% of the messages
+
+    Q1-3/10  Feeds about 30% of the messages
+    Q4-5/10  Feeds about 30% of the messages
+    Q6/10    Feeds about 50% of the messages
+
+If this flag is specified multiple times the contents will be
+logically ORed together (just one match needed).
+
+The algorithm is compatible with the one used by Diablo.
+
 =item B<S> I<size>
 
 If the amount of data queued for the site gets to be larger than I<size>
diff -ruN inn-2.4.2/innd/art.c inn-2.4.2-new/innd/art.c
--- inn-2.4.2/innd/art.c	2004-12-22 05:21:19.000000000 +0100
+++ inn-2.4.2-new/innd/art.c	2005-05-03 23:36:05.000000000 +0200
@@ -1547,6 +1547,29 @@
 }
 
 /*
+**  Return true if an element of the QHASHLIST matches the
+**  quickhash of the message-id.
+*/
+static bool
+QHashMatch(QHASHLIST *qh, char *MessageID)
+{
+  unsigned char *p = (unsigned char *)MessageID;
+  int h = 0;
+
+  while (*p)
+    h += *p++;
+
+  while (qh) {
+    if ((h % qh->mod + 1) >= qh->begin &&
+        (h % qh->mod + 1) <= qh->end)
+	  return true;
+    qh = qh->next;
+  }
+
+  return false;
+}
+
+/*
 **  Propagate an article to the sites have "expressed an interest."
 */
 static void
@@ -1619,6 +1642,10 @@
        * cross-posting. */
       continue;
 
+    if (sp->QHashList && !QHashMatch(sp->QHashList, HDR(HDR__MESSAGE_ID)))
+      /* quickhash doesn't match */
+      continue;
+
     if (list && *list != NULL && sp->Distributions &&
       !DISTwantany(sp->Distributions, list))
       /* Not in the site's desired list of distributions. */
diff -ruN inn-2.4.2/innd/innd.h inn-2.4.2-new/innd/innd.h
--- inn-2.4.2/innd/innd.h	2004-12-22 05:21:19.000000000 +0100
+++ inn-2.4.2-new/innd/innd.h	2005-05-03 23:36:29.000000000 +0200
@@ -407,6 +407,17 @@
 
 
 /*
+**  Diablo-style hashed feeds or hashfeeds.
+*/
+typedef struct _QHASHLIST {
+  int			begin;
+  int			end;
+  int			mod;
+  struct _QHASHLIST	*next;
+} QHASHLIST;
+
+
+/*
 **  A site may reject something in its subscription list if it has
 **  too many hops, or a bad distribution.
 */
@@ -458,6 +469,7 @@
   struct buffer	  Buffer;
   bool		  Buffered;
   char	      **  Originator;
+  QHASHLIST    *  QHashList;
   int		  Next;
   int		  Prev;
 } SITE;
diff -ruN inn-2.4.2/innd/newsfeeds.c inn-2.4.2-new/innd/newsfeeds.c
--- inn-2.4.2/innd/newsfeeds.c	2004-12-22 05:21:19.000000000 +0100
+++ inn-2.4.2-new/innd/newsfeeds.c	2005-05-04 00:01:57.000000000 +0200
@@ -448,6 +448,7 @@
     int			isp;
     SITE		*nsp;
     struct buffer	b;
+    QHASHLIST		*qh;
 
     b = sp->Buffer;
     *sp = SITEnull;
@@ -467,6 +468,7 @@
     sp->NeedOverviewCreation = false;
     sp->FeedwithoutOriginator = false;
     sp->DropFiltered = false;
+    sp->QHashList = NULL;
 
     /* Nip off the first field, the site name. */
     if ((f2 = strchr(Entry, NF_FIELD_SEP)) == NULL)
@@ -603,6 +605,18 @@
             if (*++p && CTYPE(isdigit, *p))
                 sp->Nice = atoi(p);
             break;
+	case 'Q':
+	    qh = xmalloc(sizeof(QHASHLIST));
+	    p++;
+	    if (sscanf(p, "%d/%d", &qh->begin, &qh->mod) == 2) {
+		qh->end = qh->begin;
+	    } else if (sscanf(p, "%d-%d/%d", &qh->begin, &qh->end, &qh->mod) != 3) {
+		free(qh);
+		return "hash not in x/z or x-y/z format";
+	    }
+	    qh->next = sp->QHashList;
+	    sp->QHashList = qh;
+	    break;
 	case 'S':
 	    if (*++p && CTYPE(isdigit, *p))
 		sp->StartSpooling = atol(p);
diff -ruN inn-2.4.2/innd/site.c inn-2.4.2-new/innd/site.c
--- inn-2.4.2/innd/site.c	2004-12-22 05:21:19.000000000 +0100
+++ inn-2.4.2-new/innd/site.c	2005-06-01 23:41:42.000000000 +0200
@@ -997,6 +997,7 @@
 SITEfree(SITE *sp)
 {
     SITE                *s;
+    QHASHLIST		*qh, *qn;
     int                 new;
     int                 i;
     
@@ -1051,6 +1052,13 @@
 	sp->FNLnames.data = NULL;
 	sp->FNLnames.size = 0;
     }
+    if (sp->QHashList) {
+	for (qh = sp->QHashList; qh; qh = qn) {
+	    qn = qh->next;
+	    free(qh);
+	}
+	sp->QHashList = NULL;
+    }
 
     /* If this site was a master, find a new one. */
     if (sp->IsMaster) {

-- 
The From: and Reply-To: addresses are internal news2mail gateway addresses.
Reply to the list or to "Miquel van Smoorenburg" <miquels at cistron.nl>


More information about the inn-workers mailing list