< 10 hour makehistory.

[Katsuhiro Kondou]

In article <200008080643.e786hcC18395 at vulpine.ao.net>,
	Dan Merillat <harik at chaos.ao.net> wrote;

} One of my thoughts in private email was to 'fork' makehistory so spool reads
} and overview writes were in seperate processes,   That would keep diskwait
} on either side from stopping the other.  (Keep them drives busy!)

Just written.  Could you try?
-- 
Katsuhiro Kondou

--- expire/makehistory.c.orig	Sat Jul 15 15:41:54 2000
+++ expire/makehistory.c	Tue Aug  8 16:31:41 2000
@@ -49,6 +49,7 @@
 char *ActivePath = NULL;
 char *HistoryPath = NULL;
 FILE *HistFile;
+FILE *Overchan;
 BOOL DoOverview;
 BOOL Fork;
 BOOL Cutofflow = FALSE;
@@ -349,7 +350,26 @@
 {
     char temp[SMBUF];
     char *p, *q, *r;
+    int i;
 
+    if (sorttype == OVNOSORT) {
+	if (Fork) {
+	    (void)fprintf(Overchan, "%s %d %d ", TokenToText(*token), arrived, expires);
+	    if (fwrite((POINTER)overdata, (SIZE_T)1, (SIZE_T)overlen, Overchan) != overlen) {
+		fprintf(stderr, "makehistory: writing overview failed\n");
+		exit(1);
+	    }
+	    fputc('\n', Overchan);
+	} else if (!OVadd(*token, overdata, overlen, arrived, expires)) {
+	    if (OVctl(OVSPACE, (void *)&i) && i == OV_NOSPACE) {
+		fprintf(stderr, "makehistory: no space left for overview\n");
+		OVclose();
+		exit(1);
+	    }
+	    fprintf(stderr, "makehistory: Can't write overview data for article \"%s\"\n", TokenToText(*token));
+	}
+	return;
+    }
     if (OverTmpPath == NULL) {
 	/* need new temp file, so create it. */
 	(void)sprintf(temp, "%s/histXXXXXX", TmpDir);
@@ -797,6 +817,7 @@
     char *p;
     char *OldHistoryPath;
     dbzoptions opt;
+    char *buff;
     
     /* First thing, set up logging and our identity. */
     openlog("makehistory", L_OPENLOG_FLAGS | LOG_PID, LOG_INN_PROG);     
@@ -879,7 +900,7 @@
     /* Read in the overview schema */
     ARTreadschema(DoOverview);
     
-    if (DoOverview && !Fork) {
+    if (DoOverview) {
 	/* init the overview setup. */
 	if (!OVopen(OV_WRITE)) {
 	    fprintf(stderr, "makehistory: OVopen failed\n");
@@ -889,11 +910,23 @@
 	    fprintf(stderr, "makehistory: OVctl(OVSORT) failed\n");
 	    exit(1);
 	}
-	if (!OVctl(OVCUTOFFLOW, (void *)&Cutofflow)) {
-	    fprintf(stderr, "makehistory: OVctl(OVCUTOFFLOW) failed\n");
-	    exit(1);
+	if (!Fork) {
+	    if (!OVctl(OVCUTOFFLOW, (void *)&Cutofflow)) {
+	        fprintf(stderr, "makehistory: OVctl(OVCUTOFFLOW) failed\n");
+	        exit(1);
+	    }
+	    OverAddAllNewsgroups();
+	} else {
+	    if (sorttype == OVNOSORT) {
+		buff = concat(innconf->pathbin, "overchan", NULL);
+		if ((Overchan = popen(buff, "w")) == NULL) {
+		    fprintf(stderr, "makehistory: forking overchan failed\n");
+		    exit(1);
+		}
+		DISPOSE(buff);
+	    }
+	    OVclose();
 	}
-	OverAddAllNewsgroups();
     }
 
     /* Init the Storage Manager */
@@ -945,10 +978,18 @@
     }
 
     if (DoOverview) {
-	int status;
-	FlushOverTmpFile();
-	if(Fork)
-	    wait(&status);
+	if (sorttype == OVNOSORT && Fork) {
+	    if (fflush(Overchan) == EOF || ferror(Overchan) || pclose(Overchan) == EOF) {
+		(void)fprintf(stderr, "Can't flush overview data , %s\n", strerror(errno));
+		exit(1);
+	    }
+	}
+	if (sorttype != OVNOSORT) {
+	    int status;
+	    FlushOverTmpFile();
+	    if(Fork)
+		wait(&status);
+	}
     }
     if(!Fork)
 	OVclose();
--- include/ov.h.orig	Mon May 22 06:11:57 2000
+++ include/ov.h	Mon Aug  7 07:21:48 2000
@@ -12,7 +12,7 @@
 
 typedef enum {OVSPACE, OVSORT, OVCUTOFFLOW, OVGROUPBASEDEXPIRE, OVSTATICSEARCH} OVCTLTYPE;
 #define OV_NOSPACE 100
-typedef enum {OVNEWSGROUP, OVARRIVED} OVSORTTYPE;
+typedef enum {OVNEWSGROUP, OVARRIVED, OVNOSORT} OVSORTTYPE;
 
 typedef struct _OVGE {
     BOOL	delayrm;	  /* append tokens to filename if true */
--- storage/buffindexed/buffindexed.c.orig	Mon May 22 06:11:58 2000
+++ storage/buffindexed/buffindexed.c	Mon Aug  7 07:22:24 2000
@@ -1899,7 +1899,7 @@
     return TRUE;
   case OVSORT:
     sorttype = (OVSORTTYPE *)val;
-    *sorttype = OVARRIVED;
+    *sorttype = OVNOSORT;
     return TRUE;
   case OVCUTOFFLOW:
     Cutofflow = *(BOOL *)val;