INN commit: trunk (3 files)
INN Commit
rra at isc.org
Sat Sep 5 15:12:54 UTC 2009
Date: Saturday, September 5, 2009 @ 08:12:54
Author: iulius
Revision: 8609
Support for sequence numbers higher than 65535 (2^16-1) in
the timecaf storage method. We can now integrate up to (2^32-1)
articles in a mere CAF. A CAF contains all the articles that
arrive during 256 seconds.
Thanks to Kamil Jonca for the bug report and his patch.
Also explain how to decode timecaf tokens and paths.
And use fixed 16-bit and 32-bit numbers for tokens; timecaf
did not work on systems where short ints were not 16-bit integers.
Modified:
trunk/CONTRIBUTORS
trunk/doc/pod/news.pod
trunk/storage/timecaf/timecaf.c
---------------------------+
CONTRIBUTORS | 2 -
doc/pod/news.pod | 16 +++++++++++
storage/timecaf/timecaf.c | 64 ++++++++++++++++++++++++++------------------
3 files changed, 55 insertions(+), 27 deletions(-)
Modified: CONTRIBUTORS
===================================================================
--- CONTRIBUTORS 2009-09-05 13:53:05 UTC (rev 8608)
+++ CONTRIBUTORS 2009-09-05 15:12:54 UTC (rev 8609)
@@ -270,4 +270,4 @@
J. Thomas Halliley, Matija Nalis, Geraint A. Edwards, Alexander Bartolich,
David Hlacik, Andreas Mattheiss, James Ralston, Wim Lewis, Johan van Selst,
Wolfgang M. Weyand, Berend Reitsma, William Kronert, Petr Novopashenniy,
-Steve Crook, John F. Morse, Tim Woodall, Jonathan Kamens
+Steve Crook, John F. Morse, Tim Woodall, Jonathan Kamens, Kamil Jonca
Modified: doc/pod/news.pod
===================================================================
--- doc/pod/news.pod 2009-09-05 13:53:05 UTC (rev 8608)
+++ doc/pod/news.pod 2009-09-05 15:12:54 UTC (rev 8609)
@@ -38,6 +38,22 @@
=item *
+Fixed a bug in the timecaf storage method: only the first 65535 articles
+could be retrievable in a CAF, though everything was properly stored. (A
+Crunched Article File contains all the articles that arrive to the news
+server during 256 seconds.)
+
+The storage token now uses 4 bytes to store the article sequence number
+for timecaf, instead of only 2 bytes. Thanks to Kamil Jonca for the
+bug report and also the patch.
+
+=item *
+
+Fixed a bug in both timecaf and timehash which prevented them from working
+on systems where short ints were not 16-bit integers.
+
+=item *
+
A new CNFS version has been introduced by Miquel van Smoorenburg in
the CNFS header. CNFSv4 uses S<4 KB> blocks instead of S<512 bytes>,
which more particularly makes writes faster. CNFSv4 supports
Modified: storage/timecaf/timecaf.c
===================================================================
--- storage/timecaf/timecaf.c 2009-09-05 13:53:05 UTC (rev 8608)
+++ storage/timecaf/timecaf.c 2009-09-05 15:12:54 UTC (rev 8609)
@@ -98,11 +98,16 @@
static int TOCCacheHits, TOCCacheMisses;
-static TOKEN MakeToken(time_t now, int seqnum, STORAGECLASS class, TOKEN *oldtoken) {
+static TOKEN MakeToken(time_t now, ARTNUM seqnum, STORAGECLASS class, TOKEN *oldtoken) {
TOKEN token;
- unsigned int i;
- unsigned short s;
+ uint32_t i;
+ uint16_t s;
+ /* The token is @04nn00aabbccyyyyxxxx0000000000000000@
+ * where "02" is the timehash method number,
+ * "nn" the hexadecimal value of the storage class,
+ * "aabbccdd" the arrival time in hexadecimal (dd is unused),
+ * "xxxxyyyy" the hexadecimal sequence number seqnum. */
if (oldtoken == (TOKEN *)NULL)
memset(&token, '\0', sizeof(token));
else
@@ -111,49 +116,56 @@
token.class = class;
i = htonl(now);
memcpy(token.token, &i, sizeof(i));
- if (sizeof(i) > 4)
- memmove(token.token, &token.token[sizeof(i) - 4], 4);
- s = htons(seqnum);
- memcpy(&token.token[4], &s + (sizeof(s) - 2), 2);
+ s = htons(seqnum & 0xffff);
+ memcpy(&token.token[sizeof(i)], &s, sizeof(s));
+ s = htons((seqnum >> 16) & 0xffff);
+ memcpy(&token.token[sizeof(i)+sizeof(s)], &s, sizeof(s));
return token;
}
-static void BreakToken(TOKEN token, int *now, int *seqnum) {
- unsigned int i;
- unsigned short s = 0;
+static void BreakToken(TOKEN token, time_t *now, ARTNUM *seqnum) {
+ uint32_t i;
+ uint16_t s1 = 0;
+ uint16_t s2 = 0;
memcpy(&i, token.token, sizeof(i));
- memcpy(&s, &token.token[4], sizeof(s));
+ memcpy(&s1, &token.token[sizeof(i)], sizeof(s1));
+ memcpy(&s2, &token.token[sizeof(i)+sizeof(s1)], sizeof(s2));
*now = ntohl(i);
- *seqnum = (int)ntohs(s);
+ *seqnum = (ARTNUM)((ntohs(s2) << 16) + ntohs(s1));
}
/*
** Note: the time here is really "time>>8", i.e. a timestamp that's been
** shifted right by 8 bits.
*/
-static char *MakePath(int now, const STORAGECLASS class) {
+static char *MakePath(time_t now, const STORAGECLASS class) {
char *path;
size_t length;
- /* innconf->patharticles + '/timecaf-zz/xx/xxxx.CF' */
+ /* innconf->patharticles + '/timecaf-nn/bb/aacc.CF'
+ * where "nn" is the hexadecimal value of the storage class,
+ * "aabbccdd" the arrival time in hexadecimal (dd is unused). */
length = strlen(innconf->patharticles) + 32;
path = xmalloc(length);
snprintf(path, length, "%s/timecaf-%02x/%02x/%02x%02x.CF",
innconf->patharticles, class,
- (now >> 8) & 0xff, (now >> 16) & 0xff, now & 0xff);
+ (unsigned int)((now >> 8) & 0xff),
+ (unsigned int)((now >> 16) & 0xff),
+ (unsigned int)(now & 0xff));
return path;
}
static TOKEN *PathNumToToken(char *path, ARTNUM artnum) {
int n;
- unsigned int t1, t2, class;
- unsigned int timestamp;
+ unsigned int t1, t2;
+ STORAGECLASS class;
+ time_t timestamp;
static TOKEN token;
- n = sscanf(path, "timecaf-%02x/%02x/%04x.CF", &class, &t1, &t2);
+ n = sscanf(path, "timecaf-%02x/%02x/%04x.CF", (unsigned int *)&class, &t1, &t2);
if (n != 3)
return (TOKEN *)NULL;
timestamp = ((t1 << 8) & 0xff00) | ((t2 << 8) & 0xff0000) | ((t2 << 0) & 0xff);
@@ -188,7 +200,7 @@
*/
static CAFTOCCACHEENT *
-CheckTOCCache(int timestamp, int tokenclass)
+CheckTOCCache(time_t timestamp, STORAGECLASS tokenclass)
{
CAFTOCL2CACHE *l2;
CAFTOCL1CACHE *l1;
@@ -217,7 +229,7 @@
** cache.
*/
static CAFTOCCACHEENT *
-AddTOCCache(int timestamp, CAFTOCENT *toc, CAFHEADER head, int tokenclass)
+AddTOCCache(time_t timestamp, CAFTOCENT *toc, CAFHEADER head, STORAGECLASS tokenclass)
{
CAFTOCL2CACHE *l2;
CAFTOCL1CACHE *l1;
@@ -259,7 +271,7 @@
*/
static ARTHANDLE *
-StatArticle(int timestamp, ARTNUM artnum, int tokenclass)
+StatArticle(time_t timestamp, ARTNUM artnum, STORAGECLASS tokenclass)
{
CAFTOCCACHEENT *cent;
CAFTOCENT *toc;
@@ -322,7 +334,7 @@
char *path;
char *p;
time_t now;
- int timestamp;
+ time_t timestamp;
TOKEN token;
int fd;
ssize_t result;
@@ -542,8 +554,8 @@
}
ARTHANDLE *timecaf_retrieve(const TOKEN token, const RETRTYPE amount) {
- int timestamp;
- int artnum;
+ time_t timestamp;
+ ARTNUM artnum;
char *path;
ARTHANDLE *art;
static TOKEN ret_token;
@@ -639,8 +651,8 @@
}
bool timecaf_cancel(TOKEN token) {
- int now;
- int seqnum;
+ time_t now;
+ ARTNUM seqnum;
char *path;
BreakToken(token, &now, &seqnum);
More information about the inn-committers
mailing list