Patch to enable optional overchan process to write overview
Richard Todd
rmtodd at skywalker.ecn.ou.edu
Sat Jul 17 02:05:00 UTC 1999
This patch resurrects overchan as a separate process, and lets you either
have innd do all the overview writes, or have the overview writes done by
a separate 'overchan' process. A new inn.conf parameter "useoverchan" is added;
"true" means that innd does not write overview records (and thus an overchan
process should be running to do this); "false" means innd writes overview
records as before. Some commented-out sample code in innwatch.ctl.in is given
to throttle innd if overchan gets too far behind and unthrottle it as it catches
up.
Index: backends/Makefile
===================================================================
RCS file: /home/rmtodd/TempCVS/Temp-CVS-Repository/inn-current/backends/Makefile,v
retrieving revision 1.1.1.2
diff -u -r1.1.1.2 Makefile
--- Makefile 1999/06/04 01:43:02 1.1.1.2
+++ Makefile 1999/07/15 06:32:33
@@ -6,12 +6,12 @@
SOURCES = archive.c batcher.c buffchan.c cvtbatch.c filechan.c \
innxmit.c innxbatch.c nntpget.c shlock.c \
- shrinkfile.c crosspost.c actsync.c inndf.c
+ shrinkfile.c crosspost.c actsync.c inndf.c overchan.c
ALL = archive batcher buffchan cvtbatch filechan \
innxmit innxbatch nntpget shlock \
shrinkfile crosspost actsync actsyncd actmerge \
- sendxbatches inndf
+ sendxbatches inndf overchan
all: $(ALL)
@@ -82,6 +82,9 @@
shrinkfile: $(P) shrinkfile.o $(LIBNEWS)
$(LIBTOOL) $(CC) $(LDFLAGS) -o $@ shrinkfile.o $(LIBNEWS) $(LIBS)
+
+overchan: $(P) overchan.o $(LIBNEWS)
+ $(LIBTOOL) $(CC) $(LDFLAGS) -o $@ overchan.o $(LIBNEWS) $(LIBS)
inndf: $(P) inndf.o
$(LIBTOOL) $(CC) $(LDFLAGS) -o $@ inndf.o
Index: backends/overchan.c
===================================================================
RCS file: /home/rmtodd/TempCVS/Temp-CVS-Repository/inn-current/backends/overchan.c,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 overchan.c
--- overchan.c 1999/02/12 23:28:49 1.1.1.1
+++ overchan.c 1999/07/15 08:21:31
@@ -8,195 +8,51 @@
#include "configdata.h"
#include "clibrary.h"
#include <errno.h>
-#include <sys/stat.h>
-#include <sys/uio.h>
-#include <fcntl.h>
#include "libinn.h"
#include "macros.h"
#include "paths.h"
#include "qio.h"
#include <syslog.h>
+#include "ov.h"
-/*
-** Try to make one directory. Return FALSE on error.
-*/
-STATIC BOOL MakeDir(char *Name)
-{
- struct stat Sb;
-
- if (mkdir(Name, GROUPDIR_MODE) >= 0)
- return TRUE;
-
- /* See if it failed because it already exists. */
- return stat(Name, &Sb) >= 0 && S_ISDIR(Sb.st_mode);
-}
-
-
-/*
-** Make overview directory if not in spool directory. Return 0 if ok,
-** else -1.
-*/
-STATIC BOOL MakeOverDir(char *Name)
-{
- char *p;
- BOOL made;
-
- /* Optimize common case -- parent almost always exists. */
- if (MakeDir(Name))
- return TRUE;
-
- /* Try to make each of comp and comp/foo in turn. */
- for (p = Name; *p; p++)
- if (*p == '/') {
- *p = '\0';
- made = MakeDir(Name);
- *p = '/';
- if (!made)
- return FALSE;
- }
-
- return MakeDir(Name);
-}
-
+unsigned int NumArts;
+unsigned int StartTime;
+unsigned int TotOvTime;
/*
-** Get the lock for the group, then open the data file and append the
-** new data. Return FALSE on error.
-*/
-STATIC BOOL WriteData(char *Dir, char *Art, char *Rest, BOOL Complain)
-{
- static char TAB[] = "\t";
- static char NL[] = "\n";
- struct iovec iov[4];
- int fd;
- char file[SPOOLNAMEBUFF];
- int i;
- struct stat Sb;
-
- /* Name the data file. */
- (void)sprintf(file, "%s/%s", Dir, innconf->overviewname);
- /* Open and lock the file. */
- for ( ; ; ) {
- if ((fd = open(file, O_WRONLY | O_CREAT | O_APPEND, ARTFILE_MODE)) < 0) {
- if (Complain && errno != ENOENT)
- (void)fprintf(stderr, "overchan cant open %s, %s\n",
- file, strerror(errno));
- return FALSE;
- }
- if (LockFile(fd, FALSE) < 0)
- /* Wait for it. */
- (void)LockFile(fd, TRUE);
- else {
- /* Got the lock; make sure the file is still there. */
- if (fstat(fd, &Sb) < 0) {
- (void)fprintf(stderr, "overchan cant fstat %s, %s\n",
- file, strerror(errno));
- (void)close(fd);
- return FALSE;
- }
- if (Sb.st_nlink > 0)
- break;
- }
- /* Close file -- expireover might have removed it -- and try again. */
- (void)close(fd);
- }
- /* Build the I/O vector. */
- i = 0;
- iov[i].iov_base = Art;
- iov[i++].iov_len = strlen(Art);
- iov[i].iov_base = TAB;
- iov[i++].iov_len = 1;
- iov[i].iov_base = Rest;
- iov[i++].iov_len = strlen(Rest);
- iov[i].iov_base = NL;
- iov[i++].iov_len = 1;
-
- if (xwritev(fd, iov, i) < 0) {
- (void)fprintf(stderr, "overchan cant write %s %s\n",
- file, strerror(errno));
- close(fd);
- return FALSE;
- }
- if (close(fd) < 0) {
- (void)fprintf(stderr, "overchan cant close %s %s\n",
- file, strerror(errno));
- return FALSE;
+ * Timer function (lifted from innd/timer.c).
+ * This function is designed to report the number of milliseconds since
+ * the first invocation. I wanted better resolution than time(), and
+ * something easier to work with than gettimeofday()'s struct timeval's.
+ */
+
+static unsigned gettime(void)
+{
+ static int init = 0;
+ static struct timeval start_tv;
+ struct timeval tv;
+
+ if (! init) {
+ gettimeofday(&start_tv, NULL);
+ init++;
}
- return TRUE;
+ gettimeofday(&tv, NULL);
+ return((tv.tv_sec - start_tv.tv_sec) * 1000 + (tv.tv_usec - start_tv.tv_usec) / 1000);
}
/*
-** Get the lock for the group, then open the data file and append the
-** new data. Return FALSE on error.
+** Process the input. Data comes from innd in the form:
+** @token@ data
*/
-STATIC BOOL WriteUnifiedData(HASH *Hash, char *Dir, char *Art)
-{
- int fd;
- char file[SPOOLNAMEBUFF];
- OVERINDEX index;
- struct stat Sb;
- char packed[OVERINDEXPACKSIZE];
-
- /* Name the data file. */
- (void)sprintf(file, "%s/%s.index", Dir, innconf->overviewname);
- /* Open and lock the file. */
- for ( ; ; ) {
- if ((fd = open(file, O_WRONLY | O_CREAT | O_APPEND, ARTFILE_MODE)) < 0) {
- if (errno != ENOENT)
- (void)fprintf(stderr, "overchan cant open %s, %s\n",
- file, strerror(errno));
- return FALSE;
- }
- if (LockFile(fd, FALSE) < 0)
- /* Wait for it. */
- (void)LockFile(fd, TRUE);
- else {
- /* Got the lock; make sure the file is still there. */
- if (fstat(fd, &Sb) < 0) {
- (void)fprintf(stderr, "overchan cant fstat %s, %s\n",
- file, strerror(errno));
- (void)close(fd);
- return FALSE;
- }
- if (Sb.st_nlink > 0)
- break;
- }
- /* Close file -- expireover might have removed it -- and try again. */
- (void)close(fd);
- }
- index.artnum = atol(Art);
- index.hash = *Hash;
- PackOverIndex(&index, packed);
- if (xwrite(fd, packed, OVERINDEXPACKSIZE) < 0) {
- (void)fprintf(stderr, "overchan cant write %s %s\n",
- file, strerror(errno));
- close(fd);
- return FALSE;
- }
- if (close(fd) < 0) {
- (void)fprintf(stderr, "overchan cant close %s %s\n",
- file, strerror(errno));
- return FALSE;
- }
- return TRUE;
-}
-
-/*
-** Process the input. Data can come from innd:
-** news/group/name/<number> [space news/group/<number>]... \t data
-** or from mkov:
-** news/group/name \t number \t data
-*/
+#define TEXT_TOKEN_LEN (2*sizeof(TOKEN)+2)
STATIC void ProcessIncoming(QIOSTATE *qp)
{
- char *Xref = NULL;
- char *OrigXref;
char *Data;
- char *Dir;
- char *Art;
- HASH Hash;
- char *p;
+ char *texttoken;
+ char *p;
+ TOKEN token;
+ unsigned int starttime, endtime;
for ( ; ; ) {
/* Read the first line of data. */
@@ -207,135 +63,48 @@
}
break;
}
-
- /* Check if we're handling a token and if so split it out from
- * the rest of the data */
- if (innconf->storageapi) {
- if (Data[0] == '[') {
- p = strchr(Data, ' ');
- *p = '\0';
- if (!((p - Data == sizeof(HASH) * 2 + 2) && *(p-1) == ']')) {
- fprintf(stderr, "overchan malformed token, %s\n", Data);
- continue;
- }
- Hash = TextToHash(&Data[1]);
- for (p++; *p == ' '; p++);
- Xref = p;
- } else {
- fprintf(stderr, "overchan malformed token, %s\n", Data);
- continue;
- }
- } else {
- /* Find the groups and article numbers. */
- if ((Xref = strstr(Data, "Xref:")) == NULL) {
- fprintf(stderr, "overchan missing xref header\n");
- continue;
- }
- if ((Xref = strchr(Xref, ' ')) == NULL) {
- fprintf(stderr, "overchan malformed xref header\n");
- continue;
- }
- for (Xref++; *Xref == ' '; Xref++);
- if ((Xref = strchr(Xref, ' ')) == NULL) {
- fprintf(stderr, "overchan malformed xref header\n");
- continue;
- }
- for (Xref++; *Xref == ' '; Xref++);
- }
- Xref = COPY(Xref);
- OrigXref = Xref; /* save pointer so we can do free() later */
-
- for (p = Xref; *p; p++) {
- if (*p == '.')
- *p = '/';
- if (*p == '\t') {
- *p = '\0';
- break;
- }
- }
-
-
- /* Process all fields in the first part. */
- for (; *Xref; Xref = p) {
-
- /* Split up this field, then split it up. */
- for (p = Dir = Xref; *p; p++)
- if (ISWHITE(*p)) {
- *p++ = '\0';
- break;
- }
- if ((Art = strrchr(Dir, ':')) == NULL || Art[1] == '\0') {
- (void)fprintf(stderr, "overchan bad entry %s\n", Dir);
- continue;
- }
- *Art++ = '\0';
-
- /* Write data. */
- if (innconf->storageapi) {
- if (!WriteUnifiedData(&Hash, Dir, Art) &&
- (!MakeOverDir(Dir) || !WriteUnifiedData(&Hash, Dir, Art)))
- (void)fprintf(stderr, "overchan cant update %s %s\n",
- Dir, strerror(errno));
- } else {
- if (!WriteData(Dir, Art, Data, FALSE) &&
- (!MakeOverDir(Dir) || !WriteData(Dir, Art, Data, TRUE)))
- (void)fprintf(stderr, "overchan cant update %s %s\n",
- Dir, strerror(errno));
- }
+ if (Data[0] != '@' || strlen(Data) < TEXT_TOKEN_LEN+2
+ || Data[TEXT_TOKEN_LEN-1] != '@' || Data[TEXT_TOKEN_LEN] != ' ') {
+ fprintf(stderr, "overchan malformed token, %s\n", Data);
+ continue;
+ }
+ token = TextToToken(Data);
+ Data += TEXT_TOKEN_LEN+1; /* skip over token and space */
+ NumArts++;
+ starttime = gettime();
+ if (!OVadd(token, Data, strlen(Data))) {
+ fprintf(stderr, "overchan: Can't write overview \"%s\", %s\n", Data, strerror(errno));
}
- DISPOSE(OrigXref);
+ endtime = gettime();
+ TotOvTime += endtime - starttime;
}
-
- if (QIOerror(qp))
-#if HAVE_SOCKETPAIR
- if (errno != ECONNABORTED)
-#endif
- (void)fprintf(stderr, "overchan cant read %s\n", strerror(errno));
QIOclose(qp);
}
-STATIC NORETURN Usage(void)
-{
- (void)fprintf(stderr, "usage: overchan [-c] [-D dir] [files...]\n");
- exit(1);
-}
-
-
int main(int ac, char *av[])
{
int i;
QIOSTATE *qp;
- char *Dir;
+ unsigned int now;
/* First thing, set up logging and our identity. */
openlog("overchan", L_OPENLOG_FLAGS | LOG_PID, LOG_INN_PROG);
/* Set defaults. */
if (ReadInnConf() < 0) exit(1);
- Dir = innconf->pathoverview;
(void)umask(NEWSUMASK);
- /* Parse JCL. */
- while ((i = getopt(ac, av, "D:")) != EOF)
- switch (i) {
- default:
- Usage();
- /* NOTREACHED */
- case 'D':
- Dir = optarg;
- break;
- }
- ac -= optind;
- av += optind;
+ ac -= 1;
+ av += 1;
- if (chdir(Dir) < 0) {
- (void)fprintf(stderr, "overchan cant chdir %s %s\n",
- Dir, strerror(errno));
+ if (!OVopen(OV_WRITE)) {
+ syslog(L_FATAL, "overchan cant open overview");
exit(1);
}
+ StartTime = gettime();
if (ac == 0)
ProcessIncoming(QIOfdopen(STDIN));
else {
@@ -348,7 +117,9 @@
else
ProcessIncoming(qp);
}
-
+ OVclose();
+ now = gettime();
+ syslog(L_NOTICE, "overchan timings %u arts %u of %u ms", NumArts, TotOvTime, now-StartTime);
exit(0);
/* NOTREACHED */
}
Index: include/innconf.h
===================================================================
RCS file: /home/rmtodd/TempCVS/Temp-CVS-Repository/inn-current/include/innconf.h,v
retrieving revision 1.1.1.6
diff -u -r1.1.1.6 innconf.h
--- innconf.h 1999/07/04 05:48:03 1.1.1.6
+++ innconf.h 1999/07/16 03:45:13
@@ -332,7 +332,10 @@
{ _CONF_WIREFORMAT, "", 2, 1 },
#define _CONF_OVMETHOD "ovmethod"
#define CONF_VAR_OVMETHOD 103
- { _CONF_OVMETHOD, "", 1, 1 }
+ { _CONF_OVMETHOD, "", 1, 1 },
+#define _CONF_USEOVERCHAN "useoverchan"
+#define CONF_VAR_USEOVERCHAN 104
+ { _CONF_USEOVERCHAN, "", 1, 1 }
};
-#define MAX_CONF_VAR 104
+#define MAX_CONF_VAR 105
Index: include/libinn.h
===================================================================
RCS file: /home/rmtodd/TempCVS/Temp-CVS-Repository/inn-current/include/libinn.h,v
retrieving revision 1.7
diff -u -r1.7 libinn.h
--- libinn.h 1999/07/04 05:56:47 1.7
+++ libinn.h 1999/07/16 03:44:34
@@ -183,6 +183,7 @@
int enableoverview;
int wireformat; /* enable/disable wire format for tradspool */
char *ovmethod;
+ int useoverchan; /* should innd write overview, or should overchan */
};
extern struct conf_vars *innconf;
extern char *innconffile;
Index: innd/art.c
===================================================================
RCS file: /home/rmtodd/TempCVS/Temp-CVS-Repository/inn-current/innd/art.c,v
retrieving revision 1.11
diff -u -r1.11 art.c
--- art.c 1999/07/04 05:56:49 1.11
+++ art.c 1999/07/16 23:56:09
@@ -2468,7 +2468,7 @@
TMRstop(TMR_ARTWRITE);
ARTmakeoverview(&Data, FALSE);
MadeOverview = TRUE;
- if (innconf->enableoverview) {
+ if (innconf->enableoverview && !innconf->useoverchan) {
TMRstart(TMR_OVERV);
if (!OVadd(token, Data.Overview->Data, Data.Overview->Left)) {
syslog(L_ERROR, "%s cant store ov2 for %s", LogName, TokenToText(token));
Index: innd/innd.c
===================================================================
RCS file: /home/rmtodd/TempCVS/Temp-CVS-Repository/inn-current/innd/innd.c,v
retrieving revision 1.9
diff -u -r1.9 innd.c
--- innd.c 1999/07/15 00:33:36 1.9
+++ innd.c 1999/07/15 07:42:50
@@ -727,7 +727,7 @@
if (innconf->enableoverview) {
if (!OVopen(OV_WRITE)) {
- syslog(L_FATAL, "%s cant open ov3");
+ syslog(L_FATAL, "%s cant open ov3", LogName);
exit(1);
}
}
Index: lib/getconfig.c
===================================================================
RCS file: /home/rmtodd/TempCVS/Temp-CVS-Repository/inn-current/lib/getconfig.c,v
retrieving revision 1.1.1.8
diff -u -r1.1.1.8 getconfig.c
--- getconfig.c 1999/07/04 05:46:54 1.1.1.8
+++ getconfig.c 1999/07/16 23:55:58
@@ -273,6 +273,7 @@
innconf->enableoverview = TRUE;
innconf->wireformat = FALSE;
innconf->ovmethod = NULL;
+ innconf->useoverchan = FALSE;
}
void ClearInnConf()
@@ -997,6 +998,11 @@
TEST_CONFIG(CONF_VAR_OVMETHOD, bit);
if (!bit) innconf->ovmethod = COPY(p);
SET_CONFIG(CONF_VAR_OVMETHOD);
+ } else
+ if (EQ(ConfigBuff,_CONF_USEOVERCHAN)) {
+ TEST_CONFIG(CONF_VAR_USEOVERCHAN, bit);
+ if (!bit && boolval != -1 ) innconf->useoverchan = boolval;
+ SET_CONFIG(CONF_VAR_USEOVERCHAN);
}
}
(void)Fclose(F);
Index: samples/inn.conf.in
===================================================================
RCS file: /home/rmtodd/TempCVS/Temp-CVS-Repository/inn-current/samples/inn.conf.in,v
retrieving revision 1.1.1.7
diff -u -r1.1.1.7 inn.conf.in
--- inn.conf.in 1999/07/04 05:49:46 1.1.1.7
+++ inn.conf.in 1999/07/16 23:55:39
@@ -45,6 +45,7 @@
hiscachesize: 0
overcachesize: 15
enableoverview: true
+useoverchan: false
ovmethod: tradindexed
readertrack: false
strippostcc: false
Index: samples/innwatch.ctl.in
===================================================================
RCS file: /home/rmtodd/TempCVS/Temp-CVS-Repository/inn-current/samples/innwatch.ctl.in,v
retrieving revision 1.1.1.3
diff -u -r1.1.1.3 innwatch.ctl.in
--- innwatch.ctl.in 1999/06/25 01:09:22 1.1.1.3
+++ innwatch.ctl.in 1999/07/17 01:55:01
@@ -25,6 +25,12 @@
!load!load hiload! uptime | tr -d ,. | awk '{ print $(NF - 2) }' ! lt ! ${INNWATCHLOLOAD} ! go ! loadav
!hiload!+ load! uptime | tr -d ,. | awk '{ print $(NF - 2) }' ! gt ! ${INNWATCHHILOAD} ! throttle ! loadav
!load!+! uptime | tr -d ,. | awk '{ print $(NF - 2) }' ! gt ! ${INNWATCHPAUSELOAD} ! pause ! loadav
+##
+## Uncomment these to keep overchan backlog in check. Assumes your overchan
+## feed is named 'overview!'.
+#::overblog:ctlinnd feedinfo overview!|awk 'NR==1{print $7}':lt:100000:go:overviewbacklog
+#:overblog:+:ctlinnd feedinfo overview!|awk 'NR==1{print $7}':gt:400000:throttle:overviewbacklog
+##
## If load is OK, check space (and inodes) on various filesystems
!!! ${INNDF} . ! lt ! ${INNWATCHSPOOLSPACE} ! throttle ! No space (spool)
More information about the inn-patches
mailing list