INN commit: branches/2.5 (5 files)
INN Commit
rra at isc.org
Mon Sep 7 08:14:53 UTC 2009
Date: Monday, September 7, 2009 @ 01:14:53
Author: iulius
Revision: 8613
Increase the CNFS blocksize from 512 bytes to 4096 bytes
and the size limit for a buffer from 1 TB to 16 TB.
Currently the CNFS storage method uses a 512 byte (1 sector)-
granularity for its "filesystem". That was great in the nineties,
but nowadays that is very limiting:
- most filesystems use 4K blocks, so a write to a 512 byte-
CNFS block can result in a read-modify-write cycle, slowing
down writes enormously (effectively making them synchronous)
- With larger devices, the block-bitmap at the start balloons in size
- The size limit of a CNFS file/partition is 2^31 * 512 = 1 TB.
(the block-offset is stored in the CNFS token as a signed integer..)
So I have updated storage/cnfs/ to use 4K blocks.
This introduces a new CNFS version in the CNFS header, version 4.
The header now includes a blocksize member, which is 4K by default.
The block offset is now encoded in the CNFS token as a unsigned int.
CNFSv4 supports files/partitions up to 16 TB with a 4K blocksize.
If we want to support > 16TB with 4K blocks, that is doable by
stealing a few bits from the 'cycnum' value in the CNFS token.
The code was updated so that for CNFSv4 and up the cyclenumber
wraps on 2^24 instead of 2^32 (with one wrap per day, that's good
for 45000 years, so there is no problems there). So we have 8 bits
for that, but the rest of the code has not been written yet.
The code works fine with existing CNFSv3 files/partitions.
cnfsstat and cnfsheadconf have also been updated to understand CNFSv4.
Right now a new CNFS file/device is always initialized with
4K blocksize, but it would be trivial to make that configurable.
With larger blocksizes we might want to look at the CNFS write
padding though it is perhaps not useful to pad CNFS writes
to larger blocks than 4K. It doesn't do any harm though.
Thanks to Miquel van Smoorenburg for the CNFSv4 patch.
Modified:
branches/2.5/doc/pod/news.pod
branches/2.5/frontends/cnfsheadconf.in
branches/2.5/frontends/cnfsstat.in
branches/2.5/storage/cnfs/cnfs-private.h
branches/2.5/storage/cnfs/cnfs.c
-----------------------------+
doc/pod/news.pod | 10 ++
frontends/cnfsheadconf.in | 9 +-
frontends/cnfsstat.in | 13 +-
storage/cnfs/cnfs-private.h | 8 +
storage/cnfs/cnfs.c | 188 ++++++++++++++++++++++++++----------------
5 files changed, 150 insertions(+), 78 deletions(-)
Modified: doc/pod/news.pod
===================================================================
--- doc/pod/news.pod 2009-09-05 16:08:50 UTC (rev 8612)
+++ doc/pod/news.pod 2009-09-07 08:14:53 UTC (rev 8613)
@@ -38,6 +38,16 @@
=item *
+A new CNFS version has been introduced by Miquel van Smoorenburg in
+the CNFS header. CNFSv4 uses S<4 KB> blocks instead of S<512 bytes>,
+which more particularly makes writes faster. CNFSv4 supports
+files/partitions up to S<16 TB> with a S<4 KB> blocksize.
+
+Existing CNFS buffers are kept unchanged; only new CNFS buffers are
+initialized with that new version.
+
+=item *
+
B<grephistory -l> now returns the contents of the expires history field
as well as the hash of the message-ID. Besides, when the storage API
token does not exist, B<grephistory -v> now also returns the hash
Modified: frontends/cnfsheadconf.in
===================================================================
--- frontends/cnfsheadconf.in 2009-09-05 16:08:50 UTC (rev 8612)
+++ frontends/cnfsheadconf.in 2009-09-07 08:14:53 UTC (rev 8613)
@@ -238,9 +238,9 @@
my($CNFSNASIZ)=16;
my($CNFSPASIZ)=64;
my($CNFSLASIZ)=16;
- my($headerlength) = 2 * $CNFSMASIZ + 2 * $CNFSNASIZ + $CNFSPASIZ + (5 * $CNFSLASIZ);
+ my($headerlength) = 2 * $CNFSMASIZ + 2 * $CNFSNASIZ + $CNFSPASIZ + (6 * $CNFSLASIZ);
my($buff, @entries, $e);
- my($magic, $name, $path, $lena, $freea, $updatea, $cyclenuma, $metaname, $orderinmeta, $currentbuff);
+ my($magic, $name, $path, $lena, $freea, $updatea, $cyclenuma, $metaname, $orderinmeta, $currentbuff, $blksza);
if ($opt_w) {
if(! open(BUFF, "+< $buffpath") ) {
@@ -260,7 +260,7 @@
exit(1);
}
- ($magic, $name, $path, $lena, $freea, $updatea, $cyclenuma, $metaname, $orderinmeta, $currentbuff) = unpack("a8 a16 a64 a16 a16 a16 a16 a16 a16 a8", $buff);
+ ($magic, $name, $path, $lena, $freea, $updatea, $cyclenuma, $metaname, $orderinmeta, $currentbuff, $blksza) = unpack("a8 a16 a64 a16 a16 a16 a16 a16 a16 a8 a16", $buff);
if(!$magic) {
print STDERR "Error while unpacking header ...\n";
@@ -271,6 +271,7 @@
my($free) = bhex($freea);
my($update) = hex($updatea);
my($cyclenum) = hex($cyclenuma) - 1;
+ my($blksz) = ($magic =~ m/^CBuf4/) ? hex($blksza) : 512;
my ($nupdate_str, $nago_str) = &make_time ($update);
@@ -285,6 +286,7 @@
print(" Meta $metaname, order: ");
printf("%d", $orderinmeta);
print(", current: $currentbuff");
+ print(", blocksize: $blksz");
print "\n Newest: $nupdate_str, $nago_str ago\n";
@@ -334,6 +336,7 @@
$currentbuff = sprintf("%0.8s", $in);
}
$buff = pack("a8 a16 a64 a16 a16 a16 a16 a16 a16 a8", $magic, $name, $path, $lena, $freea, $updatea, $cyclenuma, $metaname, $orderinmeta, $currentbuff);
+ $buff .= pack("a16", $blksza) if ($magic =~ m/^CBuf4/);
seek(BUFF, 0, 0);
if(! syswrite(BUFF, $buff, $headerlength) ) {
print STDERR "Cannot write $headerlength bytes to file $buffpath...\n";
Modified: frontends/cnfsstat.in
===================================================================
--- frontends/cnfsstat.in 2009-09-05 16:08:50 UTC (rev 8612)
+++ frontends/cnfsstat.in 2009-09-07 08:14:53 UTC (rev 8613)
@@ -416,7 +416,7 @@
my($CNFSNASIZ)=16;
my($CNFSPASIZ)=64;
my($CNFSLASIZ)=16;
- my($headerlength) = $CNFSMASIZ + $CNFSNASIZ + $CNFSPASIZ + (4 * $CNFSLASIZ);
+ my($headerlength) = 2 * $CNFSMASIZ + 2 * $CNFSNASIZ + $CNFSPASIZ + (6 * $CNFSLASIZ);
my($buff, @entries, $e);
my($magic, $name, $path, $lena, $freea, $updatea, $cyclenuma);
@@ -432,8 +432,10 @@
exit(1);
}
- ($magic, $name, $path, $lena, $freea, $updatea, $cyclenuma) =
- unpack("a8 a16 a64 a16 a16 a16 a16", $buff);
+ use vars qw($metaname $orderinmeta $currentbuff);
+ ($magic, $name, $path, $lena, $freea, $updatea, $cyclenuma, $metaname,
+ $orderinmeta, $currentbuff, $blksza) =
+ unpack("a8 a16 a64 a16 a16 a16 a16 a16 a16 a8 a16", $buff);
if(!$magic) {
print STDERR "Error while unpacking header ...\n";
@@ -444,11 +446,12 @@
my($free) = bhex($freea);
my($update) = hex($updatea);
my($cyclenum) = hex($cyclenuma) - 1;
+ my($blksz) = ($magic =~ m/^CBuf4/) ? hex($blksza) : 512;
if ($opt_a) {
my $pagesize = 16384;
- my $minartoffset = int($len / (512 * 8)) + 512;
+ my $minartoffset = int($len / ($blksz * 8)) + 512;
# Align upwards:
$minartoffset = ($minartoffset + $pagesize) & ~($pagesize - 1);
@@ -481,7 +484,7 @@
}
# Just in case we chopped Message-ID in two, use the end
# at the front in next iteration.
- $buff = substr ($buff, -512);
+ $buff = substr ($buff, -$blksz);
} while ($sentinel -= $pagesize > 0);
}
Modified: storage/cnfs/cnfs-private.h
===================================================================
--- storage/cnfs/cnfs-private.h 2009-09-05 16:08:50 UTC (rev 8612)
+++ storage/cnfs/cnfs-private.h 2009-09-07 08:14:53 UTC (rev 8613)
@@ -20,10 +20,12 @@
#define CNFS_MAGICV1 "Cycbuff" /* CNFSMASIZ bytes */
#define CNFS_MAGICV2 "CBuf1" /* CNFSMASIZ bytes */
#define CNFS_MAGICV3 "CBuf3" /* CNFSMASIZ bytes */
-#define CNFS_BLOCKSIZE 512 /* Unit block size we'll work with */
+#define CNFS_MAGICV4 "CBuf4" /* CNFSMASIZ bytes */
+#define CNFS_DFL_BLOCKSIZE 4096 /* Unit block size we'll work with */
+#define CNFS_MAX_BLOCKSIZE 16384 /* Max unit block size */
/* Amount of data stored at beginning of CYCBUFF before the bitfield */
-#define CNFS_BEFOREBITF (1 * CNFS_BLOCKSIZE)
+#define CNFS_BEFOREBITF 512 /* Rounded up to CNFS_HDR_PAGESIZE */
struct metacycbuff; /* Definition comes below */
@@ -49,6 +51,7 @@
storage */
bool needflush; /* true if CYCBUFFEXTERN is needed to be
flushed */
+ int blksz; /* Blocksize */
struct _CYCBUFF *next;
bool currentbuff; /* true if this cycbuff is currently used */
char metaname[CNFSNASIZ];/* Symbolic name of meta */
@@ -70,6 +73,7 @@
char metaname[CNFSNASIZ];
char orderinmeta[CNFSLASIZ];
char currentbuff[CNFSMASIZ];
+ char blksza[CNFSLASIZ]; /* ASCII version of blksz */
} CYCBUFFEXTERN;
#define METACYCBUFF_UPDATE 25
Modified: storage/cnfs/cnfs.c
===================================================================
--- storage/cnfs/cnfs.c 2009-09-05 16:08:50 UTC (rev 8612)
+++ storage/cnfs/cnfs.c 2009-09-07 08:14:53 UTC (rev 8613)
@@ -61,9 +61,9 @@
static int refresh_interval = REFRESH_INTERVAL;
static TOKEN CNFSMakeToken(char *cycbuffname, off_t offset,
- uint32_t cycnum, STORAGECLASS class) {
+ int blksz, uint32_t cycnum, STORAGECLASS class) {
TOKEN token;
- int32_t int32;
+ uint32_t uint32;
/*
** XXX We'll assume that TOKENSIZE is 16 bytes and that we divvy it
@@ -73,10 +73,10 @@
token.type = TOKEN_CNFS;
token.class = class;
memcpy(token.token, cycbuffname, CNFSMAXCYCBUFFNAME);
- int32 = htonl(offset / CNFS_BLOCKSIZE);
- memcpy(&token.token[8], &int32, sizeof(int32));
- int32 = htonl(cycnum);
- memcpy(&token.token[12], &int32, sizeof(int32));
+ uint32 = htonl(offset / blksz);
+ memcpy(&token.token[8], &uint32, sizeof(uint32));
+ uint32 = htonl(cycnum);
+ memcpy(&token.token[12], &uint32, sizeof(uint32));
return token;
}
@@ -85,20 +85,20 @@
*/
static bool CNFSBreakToken(TOKEN token, char *cycbuffname,
- off_t *offset, uint32_t *cycnum) {
- int32_t int32;
+ uint32_t *blk, uint32_t *cycnum) {
+ uint32_t uint32;
- if (cycbuffname == NULL || offset == NULL || cycnum == NULL) {
+ if (cycbuffname == NULL || blk == NULL || cycnum == NULL) {
warn("CNFS: BreakToken: invalid argument: %s", cycbuffname);
SMseterror(SMERR_INTERNAL, "BreakToken: invalid argument");
return false;
}
memcpy(cycbuffname, token.token, CNFSMAXCYCBUFFNAME);
*(cycbuffname + CNFSMAXCYCBUFFNAME) = '\0'; /* Just to be paranoid */
- memcpy(&int32, &token.token[8], sizeof(int32));
- *offset = (off_t)ntohl(int32) * (off_t)CNFS_BLOCKSIZE;
- memcpy(&int32, &token.token[12], sizeof(int32));
- *cycnum = ntohl(int32);
+ memcpy(&uint32, &token.token[8], sizeof(uint32));
+ *blk = ntohl(uint32);
+ memcpy(&uint32, &token.token[12], sizeof(uint32));
+ *cycnum = ntohl(uint32);
return true;
}
@@ -188,9 +188,12 @@
return false;
}
memset(&rpx, 0, sizeof(CYCBUFFEXTERN));
- if (cycbuff->magicver == 3) {
+ if (cycbuff->magicver == 3 || cycbuff->magicver == 4) {
cycbuff->updated = time(NULL);
- strncpy(rpx.magic, CNFS_MAGICV3, strlen(CNFS_MAGICV3));
+ if (cycbuff->magicver == 3)
+ strncpy(rpx.magic, CNFS_MAGICV3, strlen(CNFS_MAGICV3));
+ else
+ strncpy(rpx.magic, CNFS_MAGICV4, strlen(CNFS_MAGICV4));
strncpy(rpx.name, cycbuff->name, CNFSNASIZ);
strncpy(rpx.path, cycbuff->path, CNFSPASIZ);
/* Don't use sprintf() directly ... the terminating '\0' causes grief */
@@ -205,6 +208,7 @@
} else {
strncpy(rpx.currentbuff, "FALSE", CNFSMASIZ);
}
+ strncpy(rpx.blksza, CNFSofft2hex(cycbuff->blksz, true), CNFSLASIZ);
memcpy(cycbuff->bitfield, &rpx, sizeof(CYCBUFFEXTERN));
msync(cycbuff->bitfield, cycbuff->minartoffset, MS_ASYNC);
cycbuff->needflush = false;
@@ -326,8 +330,7 @@
static bool CNFSparse_part_line(char *l) {
char *p;
struct stat sb;
- off_t len, minartoffset;
- int tonextblock;
+ off_t len;
CYCBUFF *cycbuff, *tmp;
/* Symbolic cnfs partition name */
@@ -384,16 +387,7 @@
cycbuff->next = (CYCBUFF *)NULL;
cycbuff->needflush = false;
cycbuff->bitfield = NULL;
- /*
- ** The minimum article offset will be the size of the bitfield itself,
- ** len / (blocksize * 8), plus however many additional blocks the CYCBUFF
- ** external header occupies ... then round up to the next block.
- */
- minartoffset =
- cycbuff->len / (CNFS_BLOCKSIZE * 8) + CNFS_BEFOREBITF;
- tonextblock = CNFS_HDR_PAGESIZE - (minartoffset & (CNFS_HDR_PAGESIZE - 1));
- cycbuff->minartoffset = minartoffset + tonextblock;
-
+ cycbuff->minartoffset = 0;
if (cycbufftab == (CYCBUFF *)NULL)
cycbufftab = cycbuff;
else {
@@ -542,7 +536,9 @@
char buf[64];
CYCBUFFEXTERN *rpx;
int fd;
+ int tonextblock;
off_t tmpo;
+ off_t minartoffset;
bool oneshot;
/*
@@ -571,7 +567,7 @@
}
}
errno = 0;
- cycbuff->bitfield = mmap(NULL, cycbuff->minartoffset,
+ cycbuff->bitfield = mmap(NULL, CNFS_HDR_PAGESIZE,
SMopenmode ? (PROT_READ | PROT_WRITE) : PROT_READ,
MAP_SHARED, cycbuff->fd, 0);
if (cycbuff->bitfield == MAP_FAILED || errno != 0) {
@@ -586,8 +582,14 @@
** & buggy & particularly icky & unupdated. Use at your own risk. :-)
*/
rpx = (CYCBUFFEXTERN *)cycbuff->bitfield;
+ cycbuff->magicver = 0;
if (strncmp(rpx->magic, CNFS_MAGICV3, strlen(CNFS_MAGICV3)) == 0) {
cycbuff->magicver = 3;
+ cycbuff->blksz = 512;
+ }
+ if (strncmp(rpx->magic, CNFS_MAGICV4, strlen(CNFS_MAGICV4)) == 0)
+ cycbuff->magicver = 4;
+ if (cycbuff->magicver >= 3) {
if (strncmp(rpx->name, cycbuff->name, CNFSNASIZ) != 0) {
warn("CNFS: Mismatch 3: read %s for cycbuff %s", rpx->name,
cycbuff->name);
@@ -621,20 +623,57 @@
cycbuff->currentbuff = true;
} else
cycbuff->currentbuff = false;
+ if (cycbuff->magicver > 3) {
+ strncpy(buf, rpx->blksza, CNFSLASIZ);
+ buf[CNFSLASIZ] = '\0';
+ cycbuff->blksz = CNFShex2offt(buf);
+ }
+ if (cycbuff->blksz < 512 || cycbuff->blksz > CNFS_MAX_BLOCKSIZE ||
+ 2 * (cycbuff->blksz / 2) != cycbuff->blksz) {
+ warn("CNFS: Invalid: read 0x%s blocksize for cycbuff %s",
+ CNFSofft2hex(cycbuff->blksz, false), cycbuff->path);
+ return false;
+ }
} else {
notice("CNFS: no magic cookie found for cycbuff %s, initializing",
cycbuff->name);
- cycbuff->magicver = 3;
+ cycbuff->magicver = 4;
cycbuff->free = cycbuff->minartoffset;
cycbuff->updated = 0;
cycbuff->cyclenum = 1;
cycbuff->currentbuff = true;
cycbuff->order = 0; /* to indicate this is newly added cycbuff */
cycbuff->needflush = true;
+ cycbuff->blksz = CNFS_DFL_BLOCKSIZE;
+ cycbuff->free = 0;
memset(cycbuff->metaname, '\0', CNFSLASIZ);
- if (!CNFSflushhead(cycbuff))
- return false;
}
+ /*
+ ** The minimum article offset will be the size of the bitfield itself,
+ ** len / (blocksize * 8), plus however many additional blocks the CYCBUFF
+ ** external header occupies ... then round up to the next block.
+ */
+ minartoffset = cycbuff->len / (cycbuff->blksz * 8) + CNFS_BEFOREBITF;
+ tonextblock = CNFS_HDR_PAGESIZE - (minartoffset & (CNFS_HDR_PAGESIZE - 1));
+ cycbuff->minartoffset = minartoffset + tonextblock;
+
+ munmap(cycbuff->bitfield, CNFS_HDR_PAGESIZE);
+ errno = 0;
+ cycbuff->bitfield = mmap(NULL, cycbuff->minartoffset,
+ SMopenmode ? (PROT_READ | PROT_WRITE) : PROT_READ,
+ MAP_SHARED, cycbuff->fd, 0);
+ if (cycbuff->bitfield == MAP_FAILED || errno != 0) {
+ warn("CNFS: CNFSinitdisks: mmap for %s offset %d len %ld failed: %m",
+ cycbuff->path, 0, (long) cycbuff->minartoffset);
+ cycbuff->bitfield = NULL;
+ return false;
+ }
+
+ if (cycbuff->free == 0)
+ cycbuff->free = cycbuff->minartoffset;
+ if (cycbuff->needflush && !CNFSflushhead(cycbuff))
+ return false;
+
if (oneshot)
break;
}
@@ -908,13 +947,13 @@
bufoff, bufmin, bufmax);
return 0;
}
- if (offset % CNFS_BLOCKSIZE != 0) {
+ if (offset % cycbuff->blksz != 0) {
SMseterror(SMERR_INTERNAL, NULL);
warn("CNFS: CNFSsetusedbitbyrp: offset %s not on %d-byte block"
- " boundary", CNFSofft2hex(offset, false), CNFS_BLOCKSIZE);
+ " boundary", CNFSofft2hex(offset, false), cycbuff->blksz);
return 0;
}
- blocknum = offset / CNFS_BLOCKSIZE;
+ blocknum = offset / cycbuff->blksz;
longoffset = blocknum / (longsize * 8);
bitoffset = blocknum % (longsize * 8);
where = (ULONG *)cycbuff->bitfield + (CNFS_BEFOREBITF / longsize)
@@ -1053,7 +1092,7 @@
METACYCBUFF *metacycbuff = NULL;
int i;
static char buf[1024];
- static char alignbuf[CNFS_BLOCKSIZE];
+ static char alignbuf[CNFS_MAX_BLOCKSIZE];
char *artcycbuffname;
off_t artoffset, middle;
uint32_t artcyclenum;
@@ -1093,18 +1132,18 @@
/* cycbuff->free should have already been aligned by the last write, but
realign it just to be sure. */
- tonextblock = CNFS_BLOCKSIZE - (cycbuff->free & (CNFS_BLOCKSIZE - 1));
- if (tonextblock != CNFS_BLOCKSIZE)
+ tonextblock = cycbuff->blksz - (cycbuff->free & (cycbuff->blksz - 1));
+ if (tonextblock != cycbuff->blksz)
cycbuff->free += tonextblock;
/* Article too big? */
- if (cycbuff->len - cycbuff->free < CNFS_BLOCKSIZE + 1)
+ if (cycbuff->len - cycbuff->free < cycbuff->blksz + 1)
left = 0;
else
- left = cycbuff->len - cycbuff->free - CNFS_BLOCKSIZE - 1;
+ left = cycbuff->len - cycbuff->free - cycbuff->blksz - 1;
if ((off_t) article.len > left) {
- for (middle = cycbuff->free ;middle < cycbuff->len - CNFS_BLOCKSIZE - 1;
- middle += CNFS_BLOCKSIZE) {
+ for (middle = cycbuff->free ;middle < cycbuff->len - cycbuff->blksz - 1;
+ middle += cycbuff->blksz) {
CNFSUsedBlock(cycbuff, middle, true, false);
}
if (innconf->nfswriter) {
@@ -1112,8 +1151,13 @@
}
cycbuff->free = cycbuff->minartoffset;
cycbuff->cyclenum++;
- if (cycbuff->cyclenum == 0)
- cycbuff->cyclenum += 2; /* cnfs_next() needs this */
+ if (cycbuff->magicver <= 3) {
+ if (cycbuff->cyclenum == 0)
+ cycbuff->cyclenum += 2; /* cnfs_next() needs this */
+ } else {
+ if ((cycbuff->cyclenum & 0xFFFFFF) == 0) /* 24 bits max */
+ cycbuff->cyclenum = 2; /* cnfs_next() needs this */
+ }
cycbuff->needflush = true;
if (metacycbuff->metamode == INTERLEAVE) {
CNFSflushhead(cycbuff); /* Flush, just for giggles */
@@ -1176,10 +1220,11 @@
iov[i].iov_len = article.iov[i-1].iov_len;
totlen += iov[i].iov_len;
}
- if ((totlen & (CNFS_BLOCKSIZE - 1)) != 0) {
- /* Want to xwritev an exact multiple of CNFS_BLOCKSIZE */
+ if ((totlen & (cycbuff->blksz - 1)) != 0) {
+ /* Want to xwritev an exact multiple of cycbuff->blksz */
iov[i].iov_base = alignbuf;
- iov[i].iov_len = CNFS_BLOCKSIZE - (totlen & (CNFS_BLOCKSIZE - 1));
+ iov[i].iov_len = cycbuff->blksz -
+ (totlen & (cycbuff->blksz - 1));
totlen += iov[i].iov_len;
i++;
}
@@ -1208,21 +1253,23 @@
}
}
CNFSUsedBlock(cycbuff, artoffset, true, true);
- for (middle = artoffset + CNFS_BLOCKSIZE; middle < cycbuff->free;
- middle += CNFS_BLOCKSIZE) {
+ for (middle = artoffset + cycbuff->blksz; middle < cycbuff->free;
+ middle += cycbuff->blksz) {
CNFSUsedBlock(cycbuff, middle, true, false);
}
if (innconf->nfswriter) {
cnfs_mapcntl(NULL, 0, MS_ASYNC);
}
if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
- return CNFSMakeToken(artcycbuffname, artoffset, artcyclenum, class);
+ return CNFSMakeToken(artcycbuffname, artoffset,
+ cycbuff->blksz, artcyclenum, class);
}
ARTHANDLE *cnfs_retrieve(const TOKEN token, const RETRTYPE amount) {
char cycbuffname[9];
off_t offset;
uint32_t cycnum;
+ uint32_t block;
CYCBUFF *cycbuff;
ARTHANDLE *art;
CNFSARTHEADER cah;
@@ -1238,7 +1285,7 @@
SMseterror(SMERR_INTERNAL, NULL);
return NULL;
}
- if (! CNFSBreakToken(token, cycbuffname, &offset, &cycnum)) {
+ if (! CNFSBreakToken(token, cycbuffname, &block, &cycnum)) {
/* SMseterror() should have already been called */
return NULL;
}
@@ -1247,7 +1294,7 @@
if (!nomessage) {
warn("CNFS: cnfs_retrieve: token %s: bogus cycbuff name:"
" %s:0x%s:%d", TokenToText(token), cycbuffname,
- CNFSofft2hex(offset, false), cycnum);
+ CNFSofft2hex(block, false), cycnum);
nomessage = true;
}
return NULL;
@@ -1257,6 +1304,7 @@
warn("CNFS: cycbuff '%s' initialization fail", cycbuff->name);
return NULL;
}
+ offset = (off_t)block * cycbuff->blksz;
if (! CNFSArtMayBeHere(cycbuff, offset, cycnum)) {
SMseterror(SMERR_NOENT, NULL);
if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
@@ -1310,7 +1358,7 @@
plusoffset = sizeof(oldCNFSARTHEADER)-sizeof(CNFSARTHEADER);
}
#endif /* OLD_CNFS */
- if (offset > cycbuff->len - CNFS_BLOCKSIZE - (off_t) ntohl(cah.size) - 1) {
+ if (offset > cycbuff->len - cycbuff->blksz - (off_t) ntohl(cah.size) - 1) {
if (!SMpreopen) {
SMseterror(SMERR_UNDEFINED, "CNFSARTHEADER size overflow");
warn("CNFS: could not match article size token %s %s:0x%s:%d: %ld",
@@ -1321,7 +1369,7 @@
return NULL;
}
CNFSReadFreeAndCycle(cycbuff);
- if (offset > cycbuff->len - CNFS_BLOCKSIZE - (off_t) ntohl(cah.size) - 1) {
+ if (offset > cycbuff->len - cycbuff->blksz - (off_t) ntohl(cah.size) - 1) {
SMseterror(SMERR_UNDEFINED, "CNFSARTHEADER size overflow");
warn("CNFS: could not match article size token %s %s:0x%s:%d: %ld",
TokenToText(token), cycbuffname, CNFSofft2hex(offset, false),
@@ -1455,13 +1503,14 @@
char cycbuffname[9];
off_t offset;
uint32_t cycnum;
+ uint32_t block;
CYCBUFF *cycbuff;
if (token.type != TOKEN_CNFS) {
SMseterror(SMERR_INTERNAL, NULL);
return false;
}
- if (! CNFSBreakToken(token, cycbuffname, &offset, &cycnum)) {
+ if (! CNFSBreakToken(token, cycbuffname, &block, &cycnum)) {
SMseterror(SMERR_INTERNAL, NULL);
/* SMseterror() should have already been called */
return false;
@@ -1475,6 +1524,7 @@
warn("CNFS: cycbuff '%s' initialization fail", cycbuff->name);
return false;
}
+ offset = (off_t)block * cycbuff->blksz;
if (! (cycnum == cycbuff->cyclenum ||
(cycnum == cycbuff->cyclenum - 1 && offset > cycbuff->free) ||
(cycnum + 1 == 0 && cycbuff->cyclenum == 2 && offset > cycbuff->free))) {
@@ -1561,19 +1611,19 @@
}
}
if (!priv.rollover) {
- for (middle = priv.offset ;middle < cycbuff->len - CNFS_BLOCKSIZE - 1;
- middle += CNFS_BLOCKSIZE) {
+ for (middle = priv.offset ;middle < cycbuff->len - cycbuff->blksz - 1;
+ middle += cycbuff->blksz) {
if (CNFSUsedBlock(cycbuff, middle, false, false) != 0)
break;
}
- if (middle >= cycbuff->len - CNFS_BLOCKSIZE - 1) {
+ if (middle >= cycbuff->len - cycbuff->blksz - 1) {
priv.rollover = true;
middle = cycbuff->minartoffset;
}
break;
} else {
for (middle = priv.offset ;middle < cycbuff->free;
- middle += CNFS_BLOCKSIZE) {
+ middle += cycbuff->blksz) {
if (CNFSUsedBlock(cycbuff, middle, false, false) != 0)
break;
}
@@ -1614,8 +1664,8 @@
*private = priv;
private->cycbuff = cycbuff;
private->offset = middle;
- if (cycbuff->len - cycbuff->free < (off_t) ntohl(cah.size) + CNFS_BLOCKSIZE + 1) {
- private->offset += CNFS_BLOCKSIZE;
+ if (cycbuff->len - cycbuff->free < (off_t) ntohl(cah.size) + cycbuff->blksz + 1) {
+ private->offset += cycbuff->blksz;
art->data = NULL;
art->len = 0;
art->token = NULL;
@@ -1623,11 +1673,11 @@
return art;
}
/* check the bitmap to ensure cah.size is not broken */
- blockfudge = (sizeof(cah) + plusoffset + ntohl(cah.size)) % CNFS_BLOCKSIZE;
- limit = private->offset + sizeof(cah) + plusoffset + ntohl(cah.size) - blockfudge + CNFS_BLOCKSIZE;
+ blockfudge = (sizeof(cah) + plusoffset + ntohl(cah.size)) % cycbuff->blksz;
+ limit = private->offset + sizeof(cah) + plusoffset + ntohl(cah.size) - blockfudge + cycbuff->blksz;
if (offset < cycbuff->free) {
- for (middle = offset + CNFS_BLOCKSIZE; (middle < cycbuff->free) && (middle < limit);
- middle += CNFS_BLOCKSIZE) {
+ for (middle = offset + cycbuff->blksz; (middle < cycbuff->free) && (middle < limit);
+ middle += cycbuff->blksz) {
if (CNFSUsedBlock(cycbuff, middle, false, false) != 0)
/* Bitmap set. This article assumes to be broken */
break;
@@ -1641,8 +1691,8 @@
return art;
}
} else {
- for (middle = offset + CNFS_BLOCKSIZE; (middle < cycbuff->len) && (middle < limit);
- middle += CNFS_BLOCKSIZE) {
+ for (middle = offset + cycbuff->blksz; (middle < cycbuff->len) && (middle < limit);
+ middle += cycbuff->blksz) {
if (CNFSUsedBlock(cycbuff, middle, false, false) != 0)
/* Bitmap set. This article assumes to be broken */
break;
@@ -1667,10 +1717,12 @@
}
private->offset += (off_t) ntohl(cah.size) + sizeof(cah) + plusoffset;
- tonextblock = CNFS_BLOCKSIZE - (private->offset & (CNFS_BLOCKSIZE - 1));
+ tonextblock = cycbuff->blksz - (private->offset & (cycbuff->blksz - 1));
private->offset += (off_t) tonextblock;
art->arrived = ntohl(cah.arrived);
- token = CNFSMakeToken(cycbuff->name, offset, (offset > cycbuff->free) ? cycbuff->cyclenum - 1 : cycbuff->cyclenum, cah.class);
+ token = CNFSMakeToken(cycbuff->name, offset, cycbuff->blksz,
+ (offset > cycbuff->free) ? cycbuff->cyclenum - 1 : cycbuff->cyclenum,
+ cah.class);
art->token = &token;
offset += sizeof(cah) + plusoffset;
if (innconf->articlemmap) {
More information about the inn-committers
mailing list