patch for nnrpd-slaves mounting spool ro via NFS (tradindexd)
S.P.Zeidler
spz at serpens.de
Wed Mar 28 21:24:04 UTC 2001
Hi,
I have:
- one Sun Sol8 with disks attached locally, running innd 2.3.1 with
CNFSi and tradindexd, the feed
- several (currently two) Sun Sol8 NFS-mounting /var/news/db and /news
containing, among others, /news/spool/overview and /news/spool/cycbuffs,
and running nnrpd from inetd, the slaves
In order to get this to work I not only had to wave the customary black
chicken over the config files, but also to somewhat mangle the appropriate
parts of libstorage to not run into problems with a) libstorage wanting to
open everything readwrite and b) the readers never seeing new articles without
umount/mount of the spool due to NFS caching.
The patch requires a modified libstorage on both the feed and the slaves,
-DSYNCGINDEXMMAP gives you the feed libstorage and -DNFSREADER gives you
the slaves'. I suspect the patch to do some not strictly necessary things
by throwing out mmap for the readers completely, and it may pretty well be
that I don't need the msync on the feed either and the utime would have done
all I needed as well. I wanted to get rid of the symptom so I whacked it
thoroughly. :-]
Also the slaves' patch doesn't do a few things it probably should
(on the order of: tradindexed_add(..){#ifdef NFSREADER syslog(L_ERROR, "you
don't -do- this, silly!"); return FALSE; #endif ...}).
Hey, I'm a lazy slob and it actually works quite satisfactorily. .->
I see no performance degradation from the changes I made, even though the
feed probably has infinitesimal amounts of increased work; the mmap
removal for the slaves really doesn't make much of a difference because
frankly the nnrpds don't live long enough to make mmap worthwhile.
Maybe some like-lazy souls will find the patch useful even though it's not
entirely aesthetically pleasing. :-) Oh, and while I'm at it, my inn2.3.1
speaks ipv6 (although not through its Altheons), which required some
smallish changes to allow the patch kit by Satosi KOBAYASI
(<kobayasi at north.ad.jp>) from 2001/02/13 to work on Solaris (as the
v6-patches' configure barfs out at Sol8 not knowing what to assume from it).
I've seen someone else works on v6-Patches, so I might even skip around
getting the changes back into circulation .->
---------------------- patch snip -------------------------
*** ov3.c.orig Sat Mar 24 01:17:22 2001
--- ov3.c Tue Mar 27 14:09:31 2001
***************
*** 15,20 ****
--- 15,24 ----
# include <fcntl.h>
#endif
+ #ifdef SYNCGINDEXMMAP
+ # include <limits.h>
+ #endif
+
#ifdef HAVE_RLIMIT
# ifdef HAVE_SYS_TIME_H
# include <sys/time.h>
***************
*** 122,127 ****
--- 126,134 ----
STATIC GROUPENTRY *GROUPentries = NULL;
STATIC int GROUPcount = 0;
STATIC GROUPLOC GROUPemptyloc = { -1 };
+ #ifdef SYNCGINDEXMMAP
+ STATIC char GROUPfn[PATH_MAX+1];
+ #endif
STATIC CACHEENTRY *CACHEdata[CACHETABLESIZE];
STATIC int OV3mode;
***************
*** 179,187 ****
--- 186,203 ----
groupfn = NEW(char, strlen(dirname) + strlen("/group.index") + 1);
strcpy(groupfn, dirname);
strcat(groupfn, "/group.index");
+ #ifdef SYNCGINDEXMMAP
+ strncpy(GROUPfn, groupfn, PATH_MAX);
+ #endif
+ #ifndef NFSREADER
GROUPfd = open(groupfn, O_RDWR | O_CREAT, ARTFILE_MODE);
if (GROUPfd < 0) {
syslog(L_FATAL, "tradindexed: could not create %s: %m", groupfn);
+ #else
+ GROUPfd = open(groupfn, O_RDONLY, ARTFILE_MODE);
+ if (GROUPfd < 0) {
+ syslog(L_FATAL, "tradindexed: could not open %s: %m", groupfn);
+ #endif
DISPOSE(groupfn);
return FALSE;
}
***************
*** 193,198 ****
--- 209,215 ----
return FALSE;
}
if (sb.st_size > sizeof(GROUPHEADER)) {
+ #ifndef NFSREADER
if (mode & OV_READ)
flag |= PROT_READ;
if (mode & OV_WRITE) {
***************
*** 202,215 ****
--- 219,242 ----
*/
flag |= PROT_WRITE|PROT_READ;
}
+ #endif
GROUPcount = (sb.st_size - sizeof(GROUPHEADER)) / sizeof(GROUPENTRY);
+ #ifndef NFSREADER
if ((GROUPheader = (GROUPHEADER *)mmap(0, GROUPfilesize(GROUPcount), flag,
MAP_SHARED, GROUPfd, 0)) == (GROUPHEADER *) -1) {
syslog(L_FATAL, "tradindexed: could not mmap %s in tradindexed_open: %m", groupfn);
+ #else
+ GROUPheader = (GROUPHEADER *) malloc(GROUPfilesize(GROUPcount));
+
+ if (GROUPfilesize(GROUPcount) != read(GROUPfd, (void *)GROUPheader, GROUPfilesize(GROUPcount))) {
+ syslog(L_FATAL, "tradindexed: could not read %s in tradindexed_open: %m", groupfn);
+ free(GROUPheader);
+ #endif
DISPOSE(groupfn);
close(GROUPfd);
return FALSE;
}
+
GROUPentries = (GROUPENTRY *)((char *)GROUPheader + sizeof(GROUPHEADER));
} else {
GROUPcount = 0;
***************
*** 316,333 ****
return TRUE;
if (GROUPheader) {
if (munmap((void *)GROUPheader, GROUPfilesize(GROUPcount)) < 0) {
syslog(L_FATAL, "tradindexed: could not munmap group.index in GROUPremapifneeded: %m");
return FALSE;
}
}
GROUPcount = (sb.st_size - sizeof(GROUPHEADER)) / sizeof(GROUPENTRY);
GROUPheader = (GROUPHEADER *)mmap(0, GROUPfilesize(GROUPcount),
PROT_READ | PROT_WRITE, MAP_SHARED, GROUPfd, 0);
if (GROUPheader == (GROUPHEADER *) -1) {
syslog(L_FATAL, "tradindexed: could not mmap group.index in GROUPremapifneeded: %m");
! return FALSE;
}
GROUPentries = (GROUPENTRY *)((char *)GROUPheader + sizeof(GROUPHEADER));
return TRUE;
--- 343,372 ----
return TRUE;
if (GROUPheader) {
+ #ifndef NFSREADER
if (munmap((void *)GROUPheader, GROUPfilesize(GROUPcount)) < 0) {
syslog(L_FATAL, "tradindexed: could not munmap group.index in GROUPremapifneeded: %m");
return FALSE;
}
+ #else
+ free(GROUPheader);
+ #endif
}
GROUPcount = (sb.st_size - sizeof(GROUPHEADER)) / sizeof(GROUPENTRY);
+ #ifndef NFSREADER
GROUPheader = (GROUPHEADER *)mmap(0, GROUPfilesize(GROUPcount),
PROT_READ | PROT_WRITE, MAP_SHARED, GROUPfd, 0);
if (GROUPheader == (GROUPHEADER *) -1) {
syslog(L_FATAL, "tradindexed: could not mmap group.index in GROUPremapifneeded: %m");
! #else
! GROUPheader = (GROUPHEADER *) malloc(GROUPfilesize(GROUPcount));
!
! if (GROUPfilesize(GROUPcount) != read(GROUPfd, (void *)GROUPheader, GROUPfilesize(GROUPcount))) {
! syslog(L_FATAL, "tradindexed: could not read group.index in GROUPremapifneeded: %m");
! free(GROUPheader);
! #endif
! return FALSE;
}
GROUPentries = (GROUPENTRY *)((char *)GROUPheader + sizeof(GROUPHEADER));
return TRUE;
***************
*** 339,348 ****
--- 378,391 ----
int flag = 0;
if (GROUPheader) {
+ #ifndef NFSREADER
if (munmap((void *)GROUPheader, GROUPfilesize(GROUPcount)) < 0) {
syslog(L_FATAL, "tradindexed: could not munmap group.index in GROUPexpand: %m");
return FALSE;
}
+ #else
+ free(GROUPheader);
+ #endif
}
GROUPcount += 1024;
if (ftruncate(GROUPfd, GROUPfilesize(GROUPcount)) < 0) {
***************
*** 349,354 ****
--- 392,398 ----
syslog(L_FATAL, "tradindexed: could not extend group.index: %m");
return FALSE;
}
+ #ifndef NFSREADER
if (mode & OV_READ)
flag |= PROT_READ;
if (mode & OV_WRITE) {
***************
*** 362,369 ****
flag, MAP_SHARED, GROUPfd, 0);
if (GROUPheader == (GROUPHEADER *) -1) {
syslog(L_FATAL, "tradindexed: could not mmap group.index in GROUPexpand: %m");
! return FALSE;
! }
GROUPentries = (GROUPENTRY *)((char *)GROUPheader + sizeof(GROUPHEADER));
if (GROUPheader->magic != GROUPHEADERMAGIC) {
GROUPheader->magic = GROUPHEADERMAGIC;
--- 406,420 ----
flag, MAP_SHARED, GROUPfd, 0);
if (GROUPheader == (GROUPHEADER *) -1) {
syslog(L_FATAL, "tradindexed: could not mmap group.index in GROUPexpand: %m");
! #else
! GROUPheader = (GROUPHEADER *) malloc(GROUPfilesize(GROUPcount));
!
! if (GROUPfilesize(GROUPcount) != read(GROUPfd, (void *)GROUPheader, GROUPfilesize(GROUPcount))) {
! syslog(L_FATAL, "tradindexed: could not read group.index in GROUPexpand: %m");
! free(GROUPheader);
! #endif
! return FALSE;
! }
GROUPentries = (GROUPENTRY *)((char *)GROUPheader + sizeof(GROUPHEADER));
if (GROUPheader->magic != GROUPHEADERMAGIC) {
GROUPheader->magic = GROUPHEADERMAGIC;
***************
*** 473,478 ****
--- 524,530 ----
gh->indexmem = (INDEXENTRY *)-1;
return TRUE;
}
+ #ifndef NFSREADER
if (!gh->datamem) {
if ((gh->datamem = (char *)mmap(0, gh->datalen, PROT_READ, MAP_SHARED,
gh->datafd, 0)) == (char *)-1) {
***************
*** 480,485 ****
--- 532,549 ----
return FALSE;
}
}
+ #else
+ if (gh->datamem)
+ free (gh->datamem);
+
+ gh->datamem = (char *)malloc(gh->datalen);
+ if (gh->datalen != read(gh->datafd, (void *)gh->datamem, gh->datalen)) {
+ syslog(L_ERROR, "tradindexed: could not read data file for %s: %m", gh->group);
+ free(gh->datamem);
+ return FALSE;
+ }
+ #endif
+ #ifndef NFSREADER
if (!gh->indexmem) {
if ((gh->indexmem = (INDEXENTRY *)mmap(0, gh->indexlen, PROT_READ, MAP_SHARED,
gh->indexfd, 0)) == (INDEXENTRY *)-1) {
***************
*** 488,493 ****
--- 552,569 ----
return FALSE;
}
}
+ #else
+ if (gh->indexmem)
+ free(gh->indexmem);
+
+ gh->indexmem = (INDEXENTRY *)malloc(gh->indexlen);
+ if (gh->indexlen != read(gh->indexfd, (void *)gh->indexmem, gh->indexlen)) {
+ syslog(L_ERROR, "tradindexed: could not read index file for %s: %m", gh->group);
+ free(gh->indexmem);
+ free(gh->datamem);
+ return FALSE;
+ }
+ #endif
return TRUE;
}
***************
*** 523,529 ****
--- 599,609 ----
gh = NEW(GROUPHANDLE, 1);
memset(gh, '\0', sizeof(GROUPHANDLE));
+ #ifndef NFSREADER
if ((gh->datafd = open(DATpath, O_RDWR| O_APPEND | O_CREAT, 0660)) < 0) {
+ #else
+ if ((gh->datafd = open(DATpath, O_RDONLY, 0660)) < 0) {
+ #endif
p = strrchr(IDXpath, '/');
*p = '\0';
if (!MakeDirectory(IDXpath, TRUE)) {
***************
*** 531,537 ****
--- 611,621 ----
return NULL;
}
*p = '/';
+ #ifndef NFSREADER
if ((gh->datafd = open(DATpath, O_RDWR| O_APPEND | O_CREAT, 0660)) < 0) {
+ #else
+ if ((gh->datafd = open(DATpath, O_RDONLY, 0660)) < 0) {
+ #endif
DISPOSE(gh);
if (errno == ENOENT)
return NULL;
***************
*** 539,545 ****
--- 623,633 ----
return NULL;
}
}
+ #ifndef NFSREADER
if ((gh->indexfd = open(IDXpath, O_RDWR | O_CREAT, 0660)) < 0) {
+ #else
+ if ((gh->indexfd = open(IDXpath, O_RDONLY, 0660)) < 0) {
+ #endif
close(gh->datafd);
DISPOSE(gh);
syslog(L_ERROR, "tradindexed: could not open %s: %m", IDXpath);
***************
*** 566,577 ****
--- 654,679 ----
close(gh->indexfd);
close(gh->datafd);
if (gh->indexmem)
+ #ifndef NFSREADER
munmap((void *)gh->indexmem, gh->indexlen);
+ #else
+ free(gh->indexmem);
+ #endif
if (gh->datamem)
+ #ifndef NFSREADER
munmap(gh->datamem, gh->datalen);
+ #else
+ free(gh->datamem);
+ #endif
if (gh->group)
DISPOSE(gh->group);
DISPOSE(gh);
+ #ifdef SYNCGINDEXMMAP
+ /* msync async to not hit the feed too badly, may be altogether superfluous */
+ msync((void *)GROUPheader, GROUPfilesize(GROUPcount), MS_ASYNC);
+ utime(GROUPfn, NULL);
+ #endif
+
}
***************
*** 1219,1227 ****
--- 1321,1333 ----
close(GROUPfd);
if (GROUPheader) {
+ #ifndef NFSREADER
if (munmap((void *)GROUPheader, GROUPfilesize(GROUPcount)) < 0) {
syslog(L_FATAL, "tradindexed: could not munmap group.index in tradindexed_close: %m");
return;
}
+ #else
+ free(GROUPheader);
+ #endif
}
}
----------------- patch snap -------------------
enjoy,
spz
--
spz at serpens.de (S.P.Zeidler)
More information about the inn-workers
mailing list