makehistory improvements/bugfix: "Bytes" header -- 2.3 20000511

[News Subsystem]

This patch is for INN 2.3-20000511's makehistory

The attached patch adds the "-e" option to makehistory, which requests
"entire" articles from the storage API using RETR_ALL instead of
RETR_HEAD, so that the Bytes header of the overview may be generated
during -O processing.

This is particularly useful for converting to uniover or old overview to
the new ovdb format (or other new overview formats) without losing the
value(s) for the Bytes field.

On a Dual P3-550mhz machine with 2GB of 100mhz RAM and a Mylex DAC1100
64mb Write-Behind RAID-0 5x18gb 1 logical unit running FreeBSD 4.0, it
took under 15 minutes to fully process about 5gb worth of articles and
generate my new overview from my 53 CNFS 1gb files. :)  I only shudder at
how long this would take on slower setups.

--
Roy Hooper (rhooper at eisa.com)
Director, Network Architecture
eisa.com


*** inn-BETA-20000511/expire/makehistory.c      Thu May 11 06:04:05 2000
--- inn-BETA-20000511.2/expire/makehistory.c    Thu May 11 18:41:10 2000
***************
*** 58,63 ****
--- 58,64 ----
  char *OverTmpPath = NULL;
  BOOL NoHistory;
  OVSORTTYPE sorttype;
+ int RetrMode;

  TIMEINFO Now;

***************
*** 519,524 ****
--- 520,526 ----
      char                      *hash;
      HASH                      key;
      char                      overdata[BIG_BUFFER];
+     char                      bytes[BIG_BUFFER];
      struct artngnum           ann;

      /* Set up place to store headers. */
***************
*** 554,559 ****
--- 556,567 ----
                continue;
            }
              fp->HeaderLength = p2 - fp->Header;
+       } else if (RetrMode == RETR_ALL && strcmp(fp->Headername, "Bytes") == 0)
+       {
+               sprintf(bytes, "%d", art->len);
+               fp->HasHeader = TRUE;
+               fp->Header = bytes;
+               fp->HeaderLength = strlen(bytes);
        }
      }
      if (Missfieldsize > 0) {
***************
*** 695,700 ****
--- 703,709 ----
  {
      fprintf(stderr, "Usage: makehistory [-b] [-f file] [-O] [-I] [-l overtmpsegsize [-a] [-u] [-x] [-T tmpdir]\n");
      fprintf(stderr, "\t-b -- delete bad articles from spool\n");
+     fprintf(stderr, "\t-e -- read entire articles to compute proper Bytes headers\n");
      fprintf(stderr, "\t-f -- write history entries to file (default $pathdb/history)\n");
      fprintf(stderr, "\t-a -- open output history file in append mode\n");
      fprintf(stderr, "\t-O -- create overview entries for articles\n");
***************
*** 789,796 ****
      Fork = FALSE;
      AppendMode = FALSE;
      NoHistory = FALSE;

!     while ((i = getopt(argc, argv, "abf:Il:OT:xF")) != EOF) {
        switch(i) {
        case 'T':
            TmpDir = optarg;
--- 798,806 ----
      Fork = FALSE;
      AppendMode = FALSE;
      NoHistory = FALSE;
+     RetrMode = RETR_HEAD;

!     while ((i = getopt(argc, argv, "aebf:Il:OT:xF")) != EOF) {
        switch(i) {
        case 'T':
            TmpDir = optarg;
***************
*** 819,824 ****
--- 829,838 ----
        case 'F':
            Fork = TRUE;
            break;
+       case 'e':
+           RetrMode = RETR_ALL;
+           break;
+
        default:
            Usage();
            break;
***************
*** 894,900 ****
       * Scan the entire spool, nuke any bad arts if needed, and process each
       * article.
       */
!     while ((art = SMnext(art, RETR_HEAD)) != NULL) {
        if (art->len == 0) {
            if (NukeBadArts && art->data == NULL && art->token != NULL)
                (void)SMcancel(*art->token);
--- 908,915 ----
       * Scan the entire spool, nuke any bad arts if needed, and process each
       * article.
       */
!
!     while ((art = SMnext(art, RetrMode)) != NULL) {
        if (art->len == 0) {
            if (NukeBadArts && art->data == NULL && art->token != NULL)
                (void)SMcancel(*art->token);