INN commit: trunk (15 files)

INN Commit rra at isc.org
Fri Jan 28 21:28:32 UTC 2011


    Date: Friday, January 28, 2011 @ 13:28:32
  Author: iulius
Revision: 9174

Update our test suite to use the official C TAP Harness release
(version 1.6).

Well, for the time being, use the legacy syntax for the test suite.
A variable named LIBTEST_NEW_FORMAT can be set at the beginning
of a test so as to use the new syntax.

Added:
  trunk/tests/libtest.sh
Modified:
  trunk/MANIFEST
  trunk/tests/lib/date-t.c
  trunk/tests/lib/hashtab-t.c
  trunk/tests/lib/inet_aton-t.c
  trunk/tests/lib/inet_ntop-t.c
  trunk/tests/lib/messages-t.c
  trunk/tests/lib/snprintf-t.c
  trunk/tests/lib/tst-t.c
  trunk/tests/lib/uwildmat-t.c
  trunk/tests/lib/xwrite-t.c
  trunk/tests/libtest.c
  trunk/tests/libtest.h
  trunk/tests/overview/api-t.c
  trunk/tests/runtests.c

-------------------------+
 MANIFEST                |    3 
 tests/lib/date-t.c      |   10 
 tests/lib/hashtab-t.c   |   30 -
 tests/lib/inet_aton-t.c |   11 
 tests/lib/inet_ntop-t.c |    2 
 tests/lib/messages-t.c  |    8 
 tests/lib/snprintf-t.c  |   10 
 tests/lib/tst-t.c       |   17 
 tests/lib/uwildmat-t.c  |   18 
 tests/lib/xwrite-t.c    |    4 
 tests/libtest.c         |  604 +++++++++++++++++++++++++++----
 tests/libtest.h         |  122 +++++-
 tests/libtest.sh        |  223 +++++++++++
 tests/overview/api-t.c  |    6 
 tests/runtests.c        |  889 ++++++++++++++++++++++++++++++++--------------
 15 files changed, 1554 insertions(+), 403 deletions(-)

Modified: MANIFEST
===================================================================
--- MANIFEST	2011-01-25 18:46:13 UTC (rev 9173)
+++ MANIFEST	2011-01-28 21:28:32 UTC (rev 9174)
@@ -882,8 +882,9 @@
 tests/lib/xmalloc.c                   Helper program for xmalloc tests
 tests/lib/xmalloc.t                   Tests for lib/xmalloc.c
 tests/lib/xwrite-t.c                  Tests for lib/xwrite.c
-tests/libtest.c                       Helper library for writing tests
+tests/libtest.c                       Helper C library for writing tests
 tests/libtest.h                       Interface to libtest
+tests/libtest.sh                      Helper shell library for writing tests
 tests/nnrpd                           test suite for nnrpd (Directory)
 tests/nnrpd/auth-ext-t.c              Tests for auth_external in nnrpd
 tests/nnrpd/auth-test                 Helper program for external auth tests

Modified: tests/lib/date-t.c
===================================================================
--- tests/lib/date-t.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/lib/date-t.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -80,10 +80,12 @@
 ok_time(int n, time_t wanted, time_t seen)
 {
     if (wanted == seen)
-        printf("ok %d\n", n);
-    else
-        printf("not ok %d\n  wanted %lu seen %lu\n", n,
-               (unsigned long) wanted, (unsigned long) seen);
+        ok(n, true);
+    else {
+        ok(n, false);
+        diag("wanted %lu seen %lu\n",
+             (unsigned long) wanted, (unsigned long) seen);
+    }
 }
 
 static void

Modified: tests/lib/hashtab-t.c
===================================================================
--- tests/lib/hashtab-t.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/lib/hashtab-t.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -56,7 +56,8 @@
 {
     struct hash *hash;
     FILE *words;
-    int reported, i;
+    bool reported;
+    int i;
     char buffer[1024];
     char *word;
     char *test, *testing, *strange, *change, *foo, *bar;
@@ -108,13 +109,12 @@
     ok(26, hash_count(hash) == 3);
 
     hash_traverse(hash, string_traverse, &wordrefs[0]);
-    reported = 0;
+    reported = false;
     for (i = 0; wordrefs[i].word != NULL; i++)
         if (wordrefs[i].count != 1 && !reported) {
-            printf("not ");
-            reported = 1;
+            reported = true;
         }
-    puts("ok 27");
+    ok(27, !reported);
     ok(28, wordrefs[3].count == 0);
 
     hash_free(hash);
@@ -151,39 +151,35 @@
 
     hash = hash_create(4, hash_string, string_key, string_equal,
                        string_delete);
-    reported = 0;
+    reported = false;
     if (hash == NULL)
-        printf("not ");
+        reported = true;
     else {
         while (fgets(buffer, sizeof(buffer), words)) {
             buffer[strlen(buffer) - 1] = '\0';
             word = xstrdup(buffer);
             if (!hash_insert(hash, word, word)) {
-                if (!reported)
-                    printf("not ");
-                reported = 1;
+                reported = true;
             }
         }
     }
-    puts("ok 37");
+    ok(37, !reported);
 
     if (fseek(words, 0, SEEK_SET) < 0)
         sysdie("Unable to rewind words file");
-    reported = 0;
+    reported = false;
     if (hash == NULL)
-        printf("not ");
+        reported = true;
     else {
         while (fgets(buffer, sizeof(buffer), words)) {
             buffer[strlen(buffer) - 1] = '\0';
             word = hash_lookup(hash, buffer);
             if (!word || strcmp(word, buffer) != 0) {
-                if (!reported)
-                    printf("not ");
-                reported = 1;
+                reported = true;
             }
         }
     }
-    puts("ok 38");
+    ok(38, !reported);
 
     hash_free(hash);
 

Modified: tests/lib/inet_aton-t.c
===================================================================
--- tests/lib/inet_aton-t.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/lib/inet_aton-t.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -18,11 +18,12 @@
     success = test_inet_aton(string, &in);
     okay = (success && in.s_addr == htonl(addr));
     
-    printf("%sok %d\n", okay ? "" : "not ", n);
-    if (!okay && !success) printf("  success: %d\n", success);
+    ok(n, okay);
+    if (!okay && !success)
+        diag("  success: %d\n", success);
     if (!okay && in.s_addr != htonl(addr))
-        printf("  want: %lx\n   saw: %lx\n", (unsigned long) htonl(addr),
-               (unsigned long) in.s_addr);
+        diag("  want: %lx\n   saw: %lx\n", (unsigned long) htonl(addr),
+             (unsigned long) in.s_addr);
 }
 
 static void
@@ -34,7 +35,7 @@
     in.s_addr = htonl(0x01020304UL);
     success = test_inet_aton(string, &in);
     success = (success == 0 && in.s_addr == htonl(0x01020304UL));
-    printf("%sok %d\n", success ? "" : "not ", n);
+    ok(n, success);
 }
 
 int

Modified: tests/lib/inet_ntop-t.c
===================================================================
--- tests/lib/inet_ntop-t.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/lib/inet_ntop-t.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -23,7 +23,7 @@
     in.s_addr = htonl(addr);
     if (test_inet_ntop(AF_INET, &in, result, sizeof(result)) == NULL) {
         ok(n++, false);
-        printf("# cannot convert %lu: %s", addr, strerror(errno));
+        diag("cannot convert %lu: %s", addr, strerror(errno));
     } else
         ok(n++, true);
     ok_string(n++, expected, result);

Modified: tests/lib/messages-t.c
===================================================================
--- tests/lib/messages-t.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/lib/messages-t.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -183,15 +183,15 @@
 
     real_status = run_test(function, buf, sizeof(buf));
     if (!WIFEXITED(real_status) || status != WEXITSTATUS(real_status)) {
-        printf("  unexpected exit status %d\n", real_status);
+        diag("  unexpected exit status %d\n", real_status);
         succeeded = 0;
     }
     if (strcmp(output, buf)) {
-        printf("  unexpected output: %s", buf);
-        printf("    expected output: %s", output);
+        diag("  unexpected output: %s", buf);
+        diag("    expected output: %s", output);
         succeeded = 0;
     }
-    printf("%sok %d\n", succeeded ? "" : "not ", n);
+    ok(n, succeeded);
 }
 
 /* Given the test number, intended status, intended message sans the

Modified: tests/lib/snprintf-t.c
===================================================================
--- tests/lib/snprintf-t.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/lib/snprintf-t.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -82,14 +82,14 @@
     result = test_vsnprintf(buf, truncate ? 32 : sizeof(buf), format, args);
     va_end(args);
     if (!strcmp(buf, expected) && result == count) {
-        printf("ok %d\n", n);
+        ok(n, true);
     } else {
-        printf("not ok %d\n", n);
-        printf("  format: %s\n", format);
+        ok(n, false);
+        diag("  format: %s\n", format);
         if (strcmp(buf, expected))
-            printf("   saw: %s\n  want: %s\n", buf, expected);
+            diag("   saw: %s\n  want: %s\n", buf, expected);
         if (result != count)
-            printf("  %d != %d\n", result, count);
+            diag("  %d != %d\n", result, count);
     }
 }
 

Modified: tests/lib/tst-t.c
===================================================================
--- tests/lib/tst-t.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/lib/tst-t.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -84,7 +84,7 @@
     tst = tst_init(1000);
     reported = false;
     if (tst == NULL)
-        printf("not ");
+        reported = true;
     else {
         while (fgets((char *) buffer, sizeof(buffer), words)) {
             buffer[ustrlen(buffer) - 1] = '\0';
@@ -93,20 +93,19 @@
             word = (unsigned char *) xstrdup((char *) buffer);
             if (tst_insert(tst, buffer, word, 0, NULL) != TST_OK) {
                 if (!reported) {
-                    printf("# Failed insert of word %s\n", word);
-                    printf("not ");
+                    diag("Failed insert of word %s\n", word);
                 }
                 reported = true;
             }
         }
     }
-    puts("ok 37");
+    ok(37, !reported);
 
     if (fseek(words, 0, SEEK_SET) < 0)
         sysdie("Unable to rewind words file");
     reported = false;
     if (tst == NULL)
-        printf("not ");
+        reported = true;
     else {
         while (fgets((char *) buffer, sizeof(buffer), words)) {
             buffer[ustrlen(buffer) - 1] = '\0';
@@ -115,16 +114,14 @@
             word = tst_search(tst, buffer);
             if (word == NULL || strcmp((char *) word, (char *) buffer) != 0) {
                 if (!reported) {
-                    printf("# Failed search of word %s\n", word);
-                    printf("not ");
+                    diag("Failed search of word %s\n", word);
                 }
                 reported = true;
             }
             word = tst_delete(tst, buffer);
             if (word == NULL || strcmp((char *) word, (char *) buffer) != 0) {
                 if (!reported) {
-                    printf("# Failed delete of word %s\n", word);
-                    printf("not ");
+                    diag("Failed delete of word %s\n", word);
                 }
                 reported = true;
             }
@@ -132,7 +129,7 @@
         }
     }
     tst_cleanup(tst);
-    puts("ok 38");
+    ok(38, !reported);
 
     return 0;
 }

Modified: tests/lib/uwildmat-t.c
===================================================================
--- tests/lib/uwildmat-t.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/lib/uwildmat-t.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -16,9 +16,9 @@
     bool matched;
 
     matched = uwildmat(text, pattern);
-    printf("%sok %d\n", matched == matches ? "" : "not ", n);
+    ok(n, matched == matches);
     if (matched != matches)
-        printf("  %s\n  %s\n  expected %d\n", text, pattern, matches);
+        diag("  %s\n  %s\n  expected %d\n", text, pattern, matches);
 }
 
 static void
@@ -27,10 +27,10 @@
     enum uwildmat matched;
 
     matched = uwildmat_poison(text, pattern);
-    printf("%sok %d\n", matched == matches ? "" : "not ", n);
+    ok(n, matched == matches);
     if (matched != matches)
-        printf("  %s\n  %s\n  expected %d got %d\n", text, pattern,
-               (int) matches, (int) matched);
+        diag("  %s\n  %s\n  expected %d got %d\n", text, pattern,
+             (int) matches, (int) matched);
 }
 
 static void
@@ -39,9 +39,9 @@
     bool matched;
 
     matched = uwildmat_simple(text, pattern);
-    printf("%sok %d\n", matched == matches ? "" : "not ", n);
+    ok(n, matched == matches);
     if (matched != matches)
-        printf("  %s\n  %s\n  expected %d\n", text, pattern, matches);
+        diag("  %s\n  %s\n  expected %d\n", text, pattern, matches);
 }
 
 static void
@@ -50,9 +50,9 @@
     bool matched;
 
     matched = is_valid_utf8(text);
-    printf("%sok %d\n", matched == matches ? "" : "not ", n);
+    ok(n, matched == matches);
     if (matched != matches)
-        printf("  %s\n  expected %d\n", text, matches);
+        diag("  %s\n  expected %d\n", text, matches);
 }
 
 int

Modified: tests/lib/xwrite-t.c
===================================================================
--- tests/lib/xwrite-t.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/lib/xwrite-t.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -23,9 +23,9 @@
     int success;
 
     success = (status == total && memcmp(data, write_buffer, 256) == 0);
-    printf("%sok %d\n", success ? "" : "not ", n);
+    ok(n, success);
     if (!success && status != total)
-        printf("  status %d, total %d\n", status, total);
+        diag("  status %d, total %d\n", status, total);
 }
 
 int

Modified: tests/libtest.c
===================================================================
--- tests/libtest.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/libtest.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -1,139 +1,598 @@
-/*  $Id$
-**
-**  Some utility routines for writing tests.
-**
-**  Herein are a variety of utility routines for writing tests.  All
-**  routines of the form ok*() take a test number and some number of
-**  appropriate arguments, check to be sure the results match the expected
-**  output using the arguments, and print out something appropriate for that
-**  test number.  Other utility routines help in constructing more complex
-**  tests.
-*/
+/* $Id$
+ *
+ * Some utility routines for writing tests.
+ *
+ * Here are a variety of utility routines for writing tests compatible with
+ * the TAP protocol.  All routines of the form ok() or is*() take a test
+ * number and some number of appropriate arguments, check to be sure the
+ * results match the expected output using the arguments, and print out
+ * something appropriate for that test number.  Other utility routines help in
+ * constructing more complex tests, skipping tests, or setting up the TAP
+ * output format.
+ *
+ * This file is part of C TAP Harness.  The current version plus supporting
+ * documentation is at <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
+ *
+ * Copyright 2009, 2010 Russ Allbery <rra at stanford.edu>
+ * Copyright 2006, 2007, 2008
+ *     Board of Trustees, Leland Stanford Jr. University
+ * Copyright (c) 2004, 2005, 2006
+ *     by Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (c) 1991, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001,
+ *     2002, 2003 by The Internet Software Consortium and Rich Salz
+ *
+ * This code is derived from software contributed to the Internet Software
+ * Consortium by Rich Salz.
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
 
 #include "config.h"
 #include "clibrary.h"
 
 #include "inn/messages.h"
 #include "inn/libinn.h"
+#include <errno.h>
+#include <math.h>
 #include "libtest.h"
 
-/* A global buffer into which message_log_buffer stores error messages. */
-char *errors = NULL;
+/*
+ * The test count.  Always contains the number that will be used for the next
+ * test status.
+ */
+unsigned long testnum = 1;
 
+/*
+ * Status information stored so that we can give a test summary at the end of
+ * the test case.  We store the planned final test and the count of failures.
+ * We can get the highest test count from testnum.
+ *
+ * We also store the PID of the process that called plan() and only summarize
+ * results when that process exits, so as to not misreport results in forked
+ * processes.
+ *
+ * If _lazy is true, we're doing lazy planning and will print out the plan
+ * based on the last test number at the end of testing.
+ */
+static unsigned long _planned = 0;
+static unsigned long _failed  = 0;
+static pid_t _process = 0;
+static int _lazy = 0;
 
+
 /*
-**  Initialize things.  Turns on line buffering on stdout and then prints out
-**  the number of tests in the test suite.
-*/
+ * Our exit handler.  Called on completion of the test to report a summary of
+ * results provided we're still in the original process.
+ */
+static void
+finish(void)
+{
+    unsigned long highest = testnum - 1;
+
+    if (_planned == 0 && !_lazy)
+        return;
+    fflush(stderr);
+    if (_process != 0 && getpid() == _process) {
+        if (_lazy) {
+            printf("1..%lu\n", highest);
+            _planned = highest;
+        }
+        if (_planned > highest)
+            printf("# Looks like you planned %lu test%s but only ran %lu\n",
+                   _planned, (_planned > 1 ? "s" : ""), highest);
+        else if (_planned < highest)
+            printf("# Looks like you planned %lu test%s but ran %lu extra\n",
+                   _planned, (_planned > 1 ? "s" : ""), highest - _planned);
+        else if (_failed > 0)
+            printf("# Looks like you failed %lu test%s of %lu\n", _failed,
+                   (_failed > 1 ? "s" : ""), _planned);
+        else if (_planned > 1)
+            printf("# All %lu tests successful or skipped\n", _planned);
+        else
+            printf("# %lu test successful or skipped\n", _planned);
+    }
+}
+
+
+/*
+ * Initialize things.  Turns on line buffering on stdout and then prints out
+ * the number of tests in the test suite.
+ */
+#ifndef LIBTEST_NEW_FORMAT
 void
 test_init(int count)
 {
+    plan(count);
+}
+#endif
+void
+plan(unsigned long count)
+{
     if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0)
-        syswarn("cannot set stdout to line buffered");
-    printf("%d\n", count);
+        fprintf(stderr, "# cannot set stdout to line buffered: %s\n",
+                strerror(errno));
+    fflush(stderr);
+    printf("1..%lu\n", count);
+    testnum = 1;
+    _planned = count;
+    _process = getpid();
+    atexit(finish);
 }
 
 
 /*
-**  Takes a boolean success value and assumes the test passes if that value
-**  is true and fails if that value is false.
-*/
+ * Initialize things for lazy planning, where we'll automatically print out a
+ * plan at the end of the program.  Turns on line buffering on stdout as well.
+ */
 void
-ok(int n, int success)
+plan_lazy(void)
 {
-    printf("%sok %d\n", success ? "" : "not ", n);
+    if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0)
+        fprintf(stderr, "# cannot set stdout to line buffered: %s\n",
+                strerror(errno));
+    testnum = 1;
+    _process = getpid();
+    _lazy = 1;
+    atexit(finish);
 }
 
 
 /*
-**  Takes an expected integer and a seen integer and assumes the test passes
-**  if those two numbers match.
-*/
+ * Skip the entire test suite and exits.  Should be called instead of plan(),
+ * not after it, since it prints out a special plan line.
+ */
 void
-ok_int(int n, int wanted, int seen)
+skip_all(const char *format, ...)
 {
+    fflush(stderr);
+    printf("1..0 # skip");
+    if (format != NULL) {
+        va_list args;
+
+        putchar(' ');
+        va_start(args, format);
+        vprintf(format, args);
+        va_end(args);
+    }
+    putchar('\n');
+    exit(0);
+}
+
+
+/*
+ * Print the test description.
+ */
+static void
+print_desc(const char *format, va_list args)
+{
+    printf(" - ");
+    vprintf(format, args);
+}
+
+
+/*
+ * Takes a boolean success value and assumes the test passes if that value
+ * is true and fails if that value is false.
+ */
+#if defined LIBTEST_NEW_FORMAT
+void
+ok(int success, const char *format, ...)
+#else
+void
+ok(int n UNUSED, int success) {
+    new_ok(success, NULL);
+}
+
+void
+new_ok(int success, const char *format, ...)
+#endif
+{
+    fflush(stderr);
+    printf("%sok %lu", success ? "" : "not ", testnum++);
+    if (!success)
+        _failed++;
+    if (format != NULL) {
+        va_list args;
+
+        va_start(args, format);
+        print_desc(format, args);
+        va_end(args);
+    }
+    putchar('\n');
+}
+
+
+/*
+ * Same as ok(), but takes the format arguments as a va_list.
+ */
+void
+okv(int success, const char *format, va_list args)
+{
+    fflush(stderr);
+    printf("%sok %lu", success ? "" : "not ", testnum++);
+    if (!success)
+        _failed++;
+    if (format != NULL)
+        print_desc(format, args);
+    putchar('\n');
+}
+
+
+/*
+ * Skip a test.
+ */
+#if defined LIBTEST_NEW_FORMAT
+void
+skip(const char *reason, ...)
+#else
+void
+skip(int n UNUSED, const char *reason) {
+    new_skip(reason);
+}
+
+void
+new_skip(const char *reason, ...)
+#endif
+{
+    fflush(stderr);
+    printf("ok %lu # skip", testnum++);
+    if (reason != NULL) {
+        va_list args;
+
+        va_start(args, reason);
+        putchar(' ');
+        vprintf(reason, args);
+        va_end(args);
+    }
+    putchar('\n');
+}
+
+
+/*
+ * Report the same status on the next count tests.
+ */
+#if defined LIBTEST_NEW_FORMAT
+void
+ok_block(unsigned long count, int status, const char *format, ...)
+#else
+void
+ok_block(int n UNUSED, int count, int success) {
+    new_ok_block(count, success, NULL);
+}
+
+void
+new_ok_block(unsigned long count, int status, const char *format, ...)
+#endif
+{
+    unsigned long i;
+
+    fflush(stderr);
+    for (i = 0; i < count; i++) {
+        printf("%sok %lu", status ? "" : "not ", testnum++);
+        if (!status)
+            _failed++;
+        if (format != NULL) {
+            va_list args;
+
+            va_start(args, format);
+            print_desc(format, args);
+            va_end(args);
+        }
+        putchar('\n');
+    }
+}
+
+
+/*
+ * Skip the next count tests.
+ */
+#if defined LIBTEST_NEW_FORMAT
+void
+skip_block(unsigned long count, const char *reason, ...)
+#else
+void
+skip_block(int n UNUSED, int count, const char *reason) {
+    new_skip_block(count, reason);
+}
+  
+void
+new_skip_block(unsigned long count, const char *reason, ...)
+#endif
+{
+    unsigned long i;
+
+    fflush(stderr);
+    for (i = 0; i < count; i++) {
+        printf("ok %lu # skip", testnum++);
+        if (reason != NULL) {
+            va_list args;
+
+            va_start(args, reason);
+            putchar(' ');
+            vprintf(reason, args);
+            va_end(args);
+        }
+        putchar('\n');
+    }
+}
+
+
+/*
+ * Takes an expected integer and a seen integer and assumes the test passes
+ * if those two numbers match.
+ */
+#ifndef LIBTEST_NEW_FORMAT
+void
+ok_int(int n UNUSED, int wanted, int seen)
+{
+    is_int(wanted, seen, NULL);
+}
+#endif
+void
+is_int(long wanted, long seen, const char *format, ...)
+{
+    fflush(stderr);
     if (wanted == seen)
-        printf("ok %d\n", n);
-    else
-        printf("not ok %d\n  wanted: %d\n    seen: %d\n", n, wanted, seen);
+        printf("ok %lu", testnum++);
+    else {
+        printf("# wanted: %ld\n#   seen: %ld\n", wanted, seen);
+        printf("not ok %lu", testnum++);
+        _failed++;
+    }
+    if (format != NULL) {
+        va_list args;
+
+        va_start(args, format);
+        print_desc(format, args);
+        va_end(args);
+    }
+    putchar('\n');
 }
 
 
 /*
-**  Takes a string and what the string should be, and assumes the test
-**  passes if those strings match (using strcmp).
-*/
+ * Takes a string and what the string should be, and assumes the test passes
+ * if those strings match (using strcmp).
+ */
+#ifndef LIBTEST_NEW_FORMAT
 void
-ok_string(int n, const char *wanted, const char *seen)
+ok_string(int n UNUSED, const char *wanted, const char *seen)
 {
+    is_string(wanted, seen, NULL);
+}   
+#endif
+void
+is_string(const char *wanted, const char *seen, const char *format, ...)
+{
     if (wanted == NULL)
         wanted = "(null)";
     if (seen == NULL)
         seen = "(null)";
-    if (strcmp(wanted, seen) != 0)
-        printf("not ok %d\n  wanted: %s\n    seen: %s\n", n, wanted, seen);
-    else
-        printf("ok %d\n", n);
+    fflush(stderr);
+    if (strcmp(wanted, seen) == 0)
+        printf("ok %lu", testnum++);
+    else {
+        printf("# wanted: %s\n#   seen: %s\n", wanted, seen);
+        printf("not ok %lu", testnum++);
+        _failed++;
+    }
+    if (format != NULL) {
+        va_list args;
+
+        va_start(args, format);
+        print_desc(format, args);
+        va_end(args);
+    }
+    putchar('\n');
 }
 
 
 /*
-**  Takes an expected double and a seen double and assumes the test passes
-**  if those two numbers match.
-*/
+ * Takes an expected double and a seen double and assumes the test passes if
+ * those two numbers are within delta of each other.
+ */
+#ifndef LIBTEST_NEW_FORMAT
 void
-ok_double(int n, double wanted, double seen)
+ok_double(int n UNUSED, double wanted, double seen)
 {
-    if (wanted == seen)
-        printf("ok %d\n", n);
-    else
-        printf("not ok %d\n  wanted: %g\n    seen: %g\n", n, wanted, seen);
+    is_double(wanted, seen, 0.01, NULL);
+}   
+#endif
+void
+is_double(double wanted, double seen, double epsilon, const char *format, ...)
+{
+    fflush(stderr);
+    if ((isnan(wanted) && isnan(seen))
+        || (isinf(wanted) && isinf(seen) && wanted == seen)
+        || fabs(wanted - seen) <= epsilon)
+        printf("ok %lu", testnum++);
+    else {
+        printf("# wanted: %g\n#   seen: %g\n", wanted, seen);
+        printf("not ok %lu", testnum++);
+        _failed++;
+    }
+    if (format != NULL) {
+        va_list args;
+
+        va_start(args, format);
+        print_desc(format, args);
+        va_end(args);
+    }
+    putchar('\n');
 }
 
 
 /*
-**  Skip a test.
-*/
+ * Takes an expected unsigned long and a seen unsigned long and assumes the
+ * test passes if the two numbers match.  Otherwise, reports them in hex.
+ */
 void
-skip(int n, const char *reason)
+is_hex(unsigned long wanted, unsigned long seen, const char *format, ...)
 {
-    printf("ok %d # skip", n);
-    if (reason != NULL)
-        printf(" - %s", reason);
+    fflush(stderr);
+    if (wanted == seen)
+        printf("ok %lu", testnum++);
+    else {
+        printf("# wanted: %lx\n#   seen: %lx\n", (unsigned long) wanted,
+               (unsigned long) seen);
+        printf("not ok %lu", testnum++);
+        _failed++;
+    }
+    if (format != NULL) {
+        va_list args;
+
+        va_start(args, format);
+        print_desc(format, args);
+        va_end(args);
+    }
     putchar('\n');
 }
 
 
 /*
-**  Report the same status on the next count tests.
-*/
+ * Bail out with an error.
+ */
 void
-ok_block(int n, int count, int status)
+bail(const char *format, ...)
 {
-    int i;
+    va_list args;
 
-    for (i = 0; i < count; i++)
-        ok(n++, status);
+    fflush(stderr);
+    fflush(stdout);
+    printf("Bail out! ");
+    va_start(args, format);
+    vprintf(format, args);
+    va_end(args);
+    printf("\n");
+    exit(1);
 }
 
 
 /*
-**  Skip the next count tests.
-*/
+ * Bail out with an error, appending strerror(errno).
+ */
 void
-skip_block(int n, int count, const char *reason)
+sysbail(const char *format, ...)
 {
+    va_list args;
+    int oerrno = errno;
+
+    fflush(stderr);
+    fflush(stdout);
+    printf("Bail out! ");
+    va_start(args, format);
+    vprintf(format, args);
+    va_end(args);
+    printf(": %s\n", strerror(oerrno));
+    exit(1);
+}
+
+
+/*
+ * Report a diagnostic to stderr.
+ */
+void
+diag(const char *format, ...)
+{
+    va_list args;
+
+    fflush(stderr);
+    fflush(stdout);
+    printf("# ");
+    va_start(args, format);
+    vprintf(format, args);
+    va_end(args);
+    printf("\n");
+}
+
+
+/*
+ * Report a diagnostic to stderr, appending strerror(errno).
+ */
+void
+sysdiag(const char *format, ...)
+{
+    va_list args;
+    int oerrno = errno;
+
+    fflush(stderr);
+    fflush(stdout);
+    printf("# ");
+    va_start(args, format);
+    vprintf(format, args);
+    va_end(args);
+    printf(": %s\n", strerror(oerrno));
+}
+
+
+/*
+ * Locate a test file.  Given the partial path to a file, look under BUILD and
+ * then SOURCE for the file and return the full path to the file.  Returns
+ * NULL if the file doesn't exist.  A non-NULL return should be freed with
+ * test_file_path_free().
+ *
+ * This function uses sprintf because it attempts to be independent of all
+ * other portability layers.  The use immediately after a memory allocation
+ * should be safe without using snprintf or strlcpy/strlcat.
+ */
+char *
+test_file_path(const char *file)
+{
+    char *base;
+    char *path = NULL;
+    size_t length;
+    const char *envs[] = { "BUILD", "SOURCE", NULL };
     int i;
 
-    for (i = 0; i < count; i++)
-        skip(n++, reason);
+    for (i = 0; envs[i] != NULL; i++) {
+        base = getenv(envs[i]);
+        if (base == NULL)
+            continue;
+        length = strlen(base) + 1 + strlen(file) + 1;
+        path = malloc(length);
+        if (path == NULL)
+            sysbail("cannot allocate memory");
+        sprintf(path, "%s/%s", base, file);
+        if (access(path, R_OK) == 0)
+            break;
+        free(path);
+        path = NULL;
+    }
+    return path;
 }
 
 
 /*
-**  An error handler that appends all errors to the errors global.  Used by
-**  error_capture.
-*/
+ * Free a path returned from test_file_path().  This function exists primarily
+ * for Windows, where memory must be freed from the same library domain that
+ * it was allocated from.
+ */
+void
+test_file_path_free(char *path)
+{
+    if (path != NULL)
+        free(path);
+}
+
+
+#ifndef LIBTEST_NEW_FORMAT
+/* A global buffer into which message_log_buffer stores error messages. */
+char *errors = NULL;
+
+/*
+ *  An error handler that appends all errors to the errors global.  Used by
+ *  error_capture.
+ */
 static void
 message_log_buffer(int len, const char *fmt, va_list args, int error UNUSED)
 {
@@ -155,10 +614,10 @@
 
 
 /*
-**  Turn on the capturing of errors.  Errors will be stored in the global
-**  errors variable where they can be checked by the test suite.  Capturing is
-**  turned off with errors_uncapture.
-*/
+ *  Turn on the capturing of errors.  Errors will be stored in the global
+ *  errors variable where they can be checked by the test suite.  Capturing is
+ *  turned off with errors_uncapture.
+ */
 void
 errors_capture(void)
 {
@@ -172,11 +631,12 @@
 
 
 /*
-**  Turn off the capturing of errors again.
-*/
+ *  Turn off the capturing of errors again.
+ */
 void
 errors_uncapture(void)
 {
     message_handlers_warn(1, message_log_stderr);
     message_handlers_notice(1, message_log_stdout);
 }
+#endif

Modified: tests/libtest.h
===================================================================
--- tests/libtest.h	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/libtest.h	2011-01-28 21:28:32 UTC (rev 9174)
@@ -1,7 +1,33 @@
-/*  $Id$
-**
-**  Some utility routines for writing tests.
-*/
+/* $Id$
+ *
+ * Basic utility routines for the TAP protocol.
+ *
+ * This file is part of C TAP Harness.  The current version plus supporting
+ * documentation is at <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
+ *
+ * Copyright 2009, 2010 Russ Allbery <rra at stanford.edu>
+ * Copyright 2006, 2007, 2008
+ *     Board of Trustees, Leland Stanford Jr. University
+ * Copyright (c) 2004, 2005, 2006
+ *     by Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (c) 1991, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001,
+ *     2002, 2003 by The Internet Software Consortium and Rich Salz
+ *
+ * This code is derived from software contributed to the Internet Software
+ * Consortium by Rich Salz.
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
 
 #ifndef LIBTEST_H
 #define LIBTEST_H 1
@@ -9,30 +35,98 @@
 #include "config.h"
 #include <inn/defines.h>
 
-/* A global buffer into which errors_capture stores errors. */
-extern char *errors;
-
 BEGIN_DECLS
 
+/*
+ * The test count.  Always contains the number that will be used for the next
+ * test status.
+ */
+extern unsigned long testnum;
+
+/* Print out the number of tests and set standard output to line buffered. */
+void plan(unsigned long count);
+
+/*
+ * Prepare for lazy planning, in which the plan will be  printed automatically
+ * at the end of the test program.
+ */
+void plan_lazy(void);
+
+/* Skip the entire test suite.  Call instead of plan. */
+void skip_all(const char *format, ...)
+    __attribute__((__noreturn__, __format__(printf, 1, 2)));
+
+/*
+ * Basic reporting functions.  The okv() function is the same as ok() but
+ * takes the test description as a va_list to make it easier to reuse the
+ * reporting infrastructure when writing new tests.
+ */
+#ifndef LIBTEST_NEW_FORMAT
 void ok(int n, int success);
+void new_ok(int success, const char *format, ...)
+        __attribute__((__format__(printf, 2, 3)));
 void ok_int(int n, int wanted, int seen);
 void ok_double(int n, double wanted, double seen);
 void ok_string(int n, const char *wanted, const char *seen);
 void skip(int n, const char *reason);
-
-/* Report the same status on, or skip, the next count tests. */
+void new_skip(const char *reason, ...)
+        __attribute__((__format__(printf, 1, 2)));
 void ok_block(int n, int count, int success);
+void new_ok_block(unsigned long count, int success, const char *format, ...)
+        __attribute__((__format__(printf, 3, 4)));
 void skip_block(int n, int count, const char *reason);
+void new_skip_block(unsigned long count, const char *reason, ...)
+        __attribute__((__format__(printf, 2, 3)));
 
-/* Print out the number of tests and set standard output to line buffered. */
 void test_init(int count);
-
-/* Turn on capturing of errors with errors_capture.  Errors reported by warn
-   will be stored in the global errors variable.  Turn this off again with
-   errors_uncapture.  Caller is responsible for freeing errors when done. */
 void errors_capture(void);
 void errors_uncapture(void);
+#else
+void ok(int success, const char *format, ...)
+    __attribute__((__format__(printf, 2, 3)));
+void skip(const char *reason, ...)
+    __attribute__((__format__(printf, 1, 2)));
 
+/* Report the same status on, or skip, the next count tests. */
+void ok_block(unsigned long count, int success, const char *format, ...)
+    __attribute__((__format__(printf, 3, 4)));
+void skip_block(unsigned long count, const char *reason, ...)
+    __attribute__((__format__(printf, 2, 3)));
+#endif
+
+void okv(int success, const char *format, va_list args);
+
+/* Check an expected value against a seen value. */
+void is_int(long wanted, long seen, const char *format, ...)
+    __attribute__((__format__(printf, 3, 4)));
+void is_double(double wanted, double seen, double epsilon,
+               const char *format, ...)
+    __attribute__((__format__(printf, 4, 5)));
+void is_string(const char *wanted, const char *seen, const char *format, ...)
+    __attribute__((__format__(printf, 3, 4)));
+void is_hex(unsigned long wanted, unsigned long seen, const char *format, ...)
+    __attribute__((__format__(printf, 3, 4)));
+
+/* Bail out with an error.  sysbail appends strerror(errno). */
+void bail(const char *format, ...)
+    __attribute__((__noreturn__, __nonnull__, __format__(printf, 1, 2)));
+void sysbail(const char *format, ...)
+    __attribute__((__noreturn__, __nonnull__, __format__(printf, 1, 2)));
+
+/* Report a diagnostic to stderr prefixed with #. */
+void diag(const char *format, ...)
+    __attribute__((__nonnull__, __format__(printf, 1, 2)));
+void sysdiag(const char *format, ...)
+    __attribute__((__nonnull__, __format__(printf, 1, 2)));
+
+/*
+ * Find a test file under BUILD or SOURCE, returning the full path.  The
+ * returned path should be freed with test_file_path_free().
+ */
+char *test_file_path(const char *file)
+    __attribute__((__malloc__, __nonnull__));
+void test_file_path_free(char *path);
+
 END_DECLS
 
 #endif /* LIBTEST_H */

Added: tests/libtest.sh
===================================================================
--- tests/libtest.sh	                        (rev 0)
+++ tests/libtest.sh	2011-01-28 21:28:32 UTC (rev 9174)
@@ -0,0 +1,223 @@
+# $Id$
+#
+# Shell function library for test cases.
+#
+# This file provides a TAP-compatible shell function library useful for
+# writing test cases.  It is part of C TAP Harness, which can be found at
+# <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
+#
+# Written by Russ Allbery <rra at stanford.edu>
+# Copyright 2009, 2010 Russ Allbery <rra at stanford.edu>
+# Copyright 2006, 2007, 2008 Board of Trustees, Leland Stanford Jr. University
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# Print out the number of test cases we expect to run.
+plan () {
+    count=1
+    planned="$1"
+    failed=0
+    echo "1..$1"
+    trap finish 0
+}
+
+# Prepare for lazy planning.
+plan_lazy () {
+    count=1
+    planned=0
+    failed=0
+    trap finish 0
+}
+
+# Report the test status on exit.
+finish () {
+    local highest looks
+    highest=`expr "$count" - 1`
+    if [ "$planned" = 0 ] ; then
+        echo "1..$highest"
+        planned="$highest"
+    fi
+    looks='# Looks like you'
+    if [ "$planned" -gt 0 ] ; then
+        if [ "$planned" -gt "$highest" ] ; then
+            if [ "$planned" -gt 1 ] ; then
+                echo "$looks planned $planned tests but only ran $highest"
+            else
+                echo "$looks planned $planned test but only ran $highest"
+            fi
+        elif [ "$planned" -lt "$highest" ] ; then
+            local extra
+            extra=`expr "$highest" - "$planned"`
+            if [ "$planned" -gt 1 ] ; then
+                echo "$looks planned $planned tests but ran $extra extra"
+            else
+                echo "$looks planned $planned test but ran $extra extra"
+            fi
+        elif [ "$failed" -gt 0 ] ; then
+            if [ "$failed" -gt 1 ] ; then
+                echo "$looks failed $failed tests of $planned"
+            else
+                echo "$looks failed $failed test of $planned"
+            fi
+        elif [ "$planned" -gt 1 ] ; then
+            echo "# All $planned tests successful or skipped"
+        else
+            echo "# $planned test successful or skipped"
+        fi
+    fi
+}
+
+# Skip the entire test suite.  Should be run instead of plan.
+skip_all () {
+    local desc
+    desc="$1"
+    if [ -n "$desc" ] ; then
+        echo "1..0 # skip $desc"
+    else
+        echo "1..0 # skip"
+    fi
+    exit 0
+}
+
+# ok takes a test description and a command to run and prints success if that
+# command is successful, false otherwise.  The count starts at 1 and is
+# updated each time ok is printed.
+ok () {
+    local desc
+    desc="$1"
+    if [ -n "$desc" ] ; then
+        desc=" - $desc"
+    fi
+    shift
+    if "$@" ; then
+        echo ok $count$desc
+    else
+        echo not ok $count$desc
+        failed=`expr $failed + 1`
+    fi
+    count=`expr $count + 1`
+}
+
+# Skip the next test.  Takes the reason why the test is skipped.
+skip () {
+    echo "ok $count # skip $*"
+    count=`expr $count + 1`
+}
+
+# Report the same status on a whole set of tests.  Takes the count of tests,
+# the description, and then the command to run to determine the status.
+ok_block () {
+    local end i desc
+    i=$count
+    end=`expr $count + $1`
+    shift
+    desc="$1"
+    shift
+    while [ "$i" -lt "$end" ] ; do
+        ok "$desc" "$@"
+        i=`expr $i + 1`
+    done
+}
+
+# Skip a whole set of tests.  Takes the count and then the reason for skipping
+# the test.
+skip_block () {
+    local i end
+    i=$count
+    end=`expr $count + $1`
+    shift
+    while [ "$i" -lt "$end" ] ; do
+        skip "$@"
+        i=`expr $i + 1`
+    done
+}
+
+# Portable variant of printf '%s\n' "$*".  In the majority of cases, this
+# function is slower than printf, because the latter is often implemented
+# as a builtin command.  The value of the variable IFS is ignored.
+puts () {
+    cat << EOH
+$@
+EOH
+}
+
+# Run a program expected to succeed, and print ok if it does and produces the
+# correct output.  Takes the description, expected exit status, the expected
+# output, the command to run, and then any arguments for that command.
+# Standard output and standard error are combined when analyzing the output of
+# the command.
+#
+# If the command may contain system-specific error messages in its output,
+# add strip_colon_error before the command to post-process its output.
+ok_program () {
+    local desc w_status w_output output status
+    desc="$1"
+    shift
+    w_status="$1"
+    shift
+    w_output="$1"
+    shift
+    output=`"$@" 2>&1`
+    status=$?
+    if [ $status = $w_status ] && [ x"$output" = x"$w_output" ] ; then
+        ok "$desc" true
+    else
+        echo "#  saw: ($status) $output"
+        echo "#  not: ($w_status) $w_output"
+        ok "$desc" false
+    fi
+}
+
+# Strip a colon and everything after it off the output of a command, as long
+# as that colon comes after at least one whitespace character.  (This is done
+# to avoid stripping the name of the program from the start of an error
+# message.)  This is used to remove system-specific error messages (coming
+# from strerror, for example).
+strip_colon_error() {
+    local output status
+    output=`"$@" 2>&1`
+    status=$?
+    output=`puts "$output" | sed 's/^\([^ ]* [^:]*\):.*/\1/'`
+    puts "$output"
+    return $status
+}
+
+# Bail out with an error message.
+bail () {
+    echo 'Bail out!' "$@"
+    exit 1
+}
+
+# Output a diagnostic on standard error, preceded by the required # mark.
+diag () {
+    echo '#' "$@"
+}
+
+# Search for the given file first in $BUILD and then in $SOURCE and echo the
+# path where the file was found, or the empty string if the file wasn't
+# found.
+test_file_path () {
+    if [ -n "$BUILD" ] && [ -f "$BUILD/$1" ] ; then
+        puts "$BUILD/$1"
+    elif [ -n "$SOURCE" ] && [ -f "$SOURCE/$1" ] ; then
+        puts "$SOURCE/$1"
+    else
+        echo ''
+    fi
+}


Property changes on: trunk/tests/libtest.sh
___________________________________________________________________
Added: svn:keywords
   + Author Date Id Revision
Added: svn:eol-style
   + native

Modified: tests/overview/api-t.c
===================================================================
--- tests/overview/api-t.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/overview/api-t.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -695,15 +695,15 @@
     fake_innconf();
     innconf->ovmethod = xstrdup("tradindexed");
     innconf->tradindexedmmap = true;
-    printf("# tradindexed with mmap\n");
+    diag("tradindexed with mmap");
     n = overview_tests(1);
 
-    printf("# tradindexed without mmap\n");
+    diag("tradindexed without mmap");
     n = overview_mmap_tests(n);
 
     free(innconf->ovmethod);
     innconf->ovmethod = xstrdup("buffindexed");
-    printf("# buffindexed\n");
+    diag("buffindexed");
     n = overview_tests(n);
 
     return 0;

Modified: tests/runtests.c
===================================================================
--- tests/runtests.c	2011-01-25 18:46:13 UTC (rev 9173)
+++ tests/runtests.c	2011-01-28 21:28:32 UTC (rev 9174)
@@ -1,54 +1,81 @@
 /* $Id$
+ *
+ * Run a set of tests, reporting results.
+ *
+ * Please note that this file is maintained separately from INN by the above
+ * author (which is why the coding style is slightly different).  Any fixes
+ * added to the INN tree should also be reported to the above author if
+ * necessary.
+ *
+ * Usage:
+ *
+ *      runtests <test-list>
+ *
+ * Expects a list of executables located in the given file, one line per
+ * executable.  For each one, runs it as part of a test suite, reporting
+ * results.  Test output should start with a line containing the number of
+ * tests (numbered from 1 to this number), optionally preceded by "1..",
+ * although that line may be given anywhere in the output.  Each additional
+ * line should be in the following format:
+ *
+ *      ok <number>
+ *      not ok <number>
+ *      ok <number> # skip
+ *      not ok <number> # todo
+ *
+ * where <number> is the number of the test.  An optional comment is permitted
+ * after the number if preceded by whitespace.  ok indicates success, not ok
+ * indicates failure.  "# skip" and "# todo" are a special cases of a comment,
+ * and must start with exactly that formatting.  They indicate the test was
+ * skipped for some reason (maybe because it doesn't apply to this platform)
+ * or is testing something known to currently fail.  The text following either
+ * "# skip" or "# todo" and whitespace is the reason.
+ *
+ * As a special case, the first line of the output may be in the form:
+ *
+ *      1..0 # skip some reason
+ *
+ * which indicates that this entire test case should be skipped and gives a
+ * reason.
+ *
+ * Any other lines are ignored, although for compliance with the TAP protocol
+ * all lines other than the ones in the above format should be sent to
+ * standard error rather than standard output and start with #.
+ *
+ * This is a subset of TAP as documented in Test::Harness::TAP or
+ * TAP::Parser::Grammar, which comes with Perl.
+ *
+ * Any bug reports, bug fixes, and improvements are very much welcome and
+ * should be sent to the e-mail address below.  This program is part of C TAP
+ * Harness <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
+ *
+ * Copyright 2000, 2001, 2004, 2006, 2007, 2008, 2009, 2010
+ *     Russ Allbery <rra at stanford.edu>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+*/
 
-   Run a set of tests, reporting results.
+/* Required for fdopen(), getopt(), and putenv(). */
+#ifndef _XOPEN_SOURCE
+# define _XOPEN_SOURCE 500
+#endif
 
-   Copyright 2000, 2001, 2004 Russ Allbery <rra at stanford.edu>
-
-   Please note that this file is maintained separately from INN by the above
-   author (which is why the coding style is slightly different).  Any fixes
-   added to the INN tree should also be reported to the above author if
-   necessary.
-
-   Permission is hereby granted, free of charge, to any person obtaining a
-   copy of this software and associated documentation files (the
-   "Software"), to deal in the Software without restriction, including
-   without limitation the rights to use, copy, modify, merge, publish,
-   distribute, sublicense, and/or sell copies of the Software, and to
-   permit persons to whom the Software is furnished to do so, subject to
-   the following conditions:
-
-   The above copyright notice and this permission notice shall be included
-   in all copies or substantial portions of the Software.
-
-   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-   OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-   SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-   Usage:
-
-        runtests <test-list>
-
-   Expects a list of executables located in the given file, one line per
-   executable.  For each one, runs it as part of a test suite, reporting
-   results.  Test output should start with a line containing the number of
-   tests (numbered from 1 to this number), and then each line should be in
-   the following format:
-
-        ok <number>
-        not ok <number>
-        ok <number> # skip
-
-   where <number> is the number of the test.  ok indicates success, not ok
-   indicates failure, and "# skip" indicates the test was skipped for some
-   reason (maybe because it doesn't apply to this platform).
-
-   Any bug reports, bug fixes, and improvements are very much welcome and
-   should be sent to the e-mail address above. */
-
 #include "config.h"
 #include "clibrary.h"
 #include "portable/wait.h"
@@ -62,6 +89,21 @@
 /* sys/time.h must be included before sys/resource.h on some platforms. */
 #include <sys/resource.h>
 
+/* WCOREDUMP delt with by "portable/wait.h". */
+
+/*
+ * The source and build versions of the tests directory.  This is used to set
+ * the SOURCE and BUILD environment variables and find test programs, if set.
+ * Normally, this should be set as part of the build process to the test
+ * subdirectories of $(abs_top_srcdir) and $(abs_top_builddir) respectively.
+ */
+#ifndef SOURCE
+# define SOURCE NULL
+#endif
+#ifndef BUILD
+# define BUILD NULL
+#endif
+
 /* Test status codes. */
 enum test_status {
     TEST_FAIL,
@@ -70,6 +112,14 @@
     TEST_INVALID
 };
 
+/* Indicates the state of our plan. */
+enum plan_status {
+    PLAN_INIT,                  /* Nothing seen yet. */
+    PLAN_FIRST,                 /* Plan seen before any tests. */
+    PLAN_PENDING,               /* Test seen and no plan yet. */
+    PLAN_FINAL                  /* Plan seen after some tests. */
+};
+
 /* Error exit statuses for test processes. */
 #define CHILDERR_DUP    100     /* Couldn't redirect stderr or stdout. */
 #define CHILDERR_EXEC   101     /* Couldn't exec child process. */
@@ -77,17 +127,22 @@
 
 /* Structure to hold data for a set of tests. */
 struct testset {
-    const char *file;           /* The file name of the test. */
-    int count;                  /* Expected count of tests. */
-    int current;                /* The last seen test number. */
-    int length;                 /* The length of the last status message. */
-    int passed;                 /* Count of passing tests. */
-    int failed;                 /* Count of failing lists. */
-    int skipped;                /* Count of skipped tests (passed). */
+    char *file;                 /* The file name of the test. */
+    char *path;                 /* The path to the test program. */
+    enum plan_status plan;      /* The status of our plan. */
+    unsigned long count;        /* Expected count of tests. */
+    unsigned long current;      /* The last seen test number. */
+    unsigned int length;        /* The length of the last status message. */
+    unsigned long passed;       /* Count of passing tests. */
+    unsigned long failed;       /* Count of failing lists. */
+    unsigned long skipped;      /* Count of skipped tests (passed). */
+    unsigned long allocated;    /* The size of the results table. */
     enum test_status *results;  /* Table of results by test number. */
-    int aborted;                /* Whether the set as aborted. */
+    unsigned int aborted;       /* Whether the set as aborted. */
     int reported;               /* Whether the results were reported. */
     int status;                 /* The exit status of the test. */
+    unsigned int all_skipped;   /* Whether all tests were skipped. */
+    char *reason;               /* Why all tests were skipped. */
 };
 
 /* Structure to hold a linked list of test sets. */
@@ -96,12 +151,13 @@
     struct testlist *next;
 };
 
-/* Header used for test output.  %s is replaced by the file name of the list
-   of tests. */
+/*
+ * Header used for test output.  %s is replaced by the file name of the list
+ * of tests.
+ */
 static const char banner[] = "\n\
 Running all tests listed in %s.  If any tests fail, run the failing\n\
-test program by hand to see more details.  The test program will have the\n\
-same name as the test set but with \".t\" appended.\n\n";
+test program with runtests -o to see more details.\n\n";
 
 /* Header for reports of failed tests. */
 static const char header[] = "\n\
@@ -109,27 +165,14 @@
 -------------------------- -------------- ---- ----  ------------------------";
 
 /* Include the file name and line number in malloc failures. */
-#define xmalloc(size)   x_malloc((size), __FILE__, __LINE__)
-#define xstrdup(p)      x_strdup((p), __FILE__, __LINE__)
+#define xmalloc(size)     x_malloc((size), __FILE__, __LINE__)
+#define xrealloc(p, size) x_realloc((p), (size), __FILE__, __LINE__)
+#define xstrdup(p)        x_strdup((p), __FILE__, __LINE__)
 
-/* Internal prototypes. */
-static void sysdie(const char *format, ...);
-static void *x_malloc(size_t, const char *file, int line);
-static char *x_strdup(const char *, const char *file, int line);
-static int test_analyze(struct testset *);
-static int test_batch(const char *testlist);
-static void test_checkline(const char *line, struct testset *);
-static void test_fail_summary(const struct testlist *);
-static int test_init(const char *line, struct testset *);
-static int test_print_range(int first, int last, int chars, int limit);
-static void test_summarize(struct testset *, int status);
-static pid_t test_start(const char *path, int *fd);
-static double tv_diff(const struct timeval *, const struct timeval *);
-static double tv_seconds(const struct timeval *);
-static double tv_sum(const struct timeval *, const struct timeval *);
 
-
-/* Report a fatal error, including the results of strerror, and exit. */
+/*
+ * Report a fatal error, including the results of strerror, and exit.
+ */
 static void
 sysdie(const char *format, ...)
 {
@@ -147,21 +190,39 @@
 }
 
 
-/* Allocate memory, reporting a fatal error and exiting on failure. */
+/*
+ * Allocate memory, reporting a fatal error and exiting on failure.
+ */
 static void *
 x_malloc(size_t size, const char *file, int line)
 {
     void *p;
 
     p = malloc(size);
-    if (!p)
+    if (p == NULL)
         sysdie("failed to malloc %lu bytes at %s line %d",
                (unsigned long) size, file, line);
     return p;
 }
 
 
-/* Copy a string, reporting a fatal error and exiting on failure. */
+/*
+ * Reallocate memory, reporting a fatal error and exiting on failure.
+ */
+static void *
+x_realloc(void *p, size_t size, const char *file, int line)
+{
+    p = realloc(p, size);
+    if (p == NULL)
+        sysdie("failed to realloc %lu bytes at %s line %d",
+               (unsigned long) size, file, line);
+    return p;
+}
+
+
+/*
+ * Copy a string, reporting a fatal error and exiting on failure.
+ */
 static char *
 x_strdup(const char *s, const char *file, int line)
 {
@@ -170,7 +231,7 @@
 
     len = strlen(s) + 1;
     p = malloc(len);
-    if (!p)
+    if (p == NULL)
         sysdie("failed to strdup %lu bytes at %s line %d",
                (unsigned long) len, file, line);
     memcpy(p, s, len);
@@ -178,22 +239,30 @@
 }
 
 
-/* Given a struct timeval, return the number of seconds it represents as a
-   double.  Use difftime() to convert a time_t to a double. */
+/*
+ * Given a struct timeval, return the number of seconds it represents as a
+ * double.  Use difftime() to convert a time_t to a double.
+ */
 static double
 tv_seconds(const struct timeval *tv)
 {
     return difftime(tv->tv_sec, 0) + tv->tv_usec * 1e-6;
 }
 
-/* Given two struct timevals, return the difference in seconds. */
+
+/*
+ * Given two struct timevals, return the difference in seconds.
+ */
 static double
 tv_diff(const struct timeval *tv1, const struct timeval *tv0)
 {
     return tv_seconds(tv1) - tv_seconds(tv0);
 }
 
-/* Given two struct timevals, return the sum in seconds as a double. */
+
+/*
+ * Given two struct timevals, return the sum in seconds as a double.
+ */
 static double
 tv_sum(const struct timeval *tv1, const struct timeval *tv2)
 {
@@ -201,42 +270,24 @@
 }
 
 
-/* Read the first line of test output, which should contain the range of
-   test numbers, and initialize the testset structure.  Assume it was zeroed
-   before being passed in.  Return true if initialization succeeds, false
-   otherwise. */
-static int
-test_init(const char *line, struct testset *ts)
+/*
+ * Given a pointer to a string, skip any leading whitespace and return a
+ * pointer to the first non-whitespace character.
+ */
+static const char *
+skip_whitespace(const char *p)
 {
-    int i;
-
-    /* Prefer a simple number of tests, but if the count is given as a range
-       such as 1..10, accept that too for compatibility with Perl's
-       Test::Harness. */
-    while (isspace((unsigned char)(*line)))
-        line++;
-    if (strncmp(line, "1..", 3) == 0)
-        line += 3;
-
-    /* Get the count, check it for validity, and initialize the struct. */
-    i = atoi(line);
-    if (i <= 0) {
-        puts("invalid test count");
-        ts->aborted = 1;
-        ts->reported = 1;
-        return 0;
-    }
-    ts->count = i;
-    ts->results = xmalloc(ts->count * sizeof(enum test_status));
-    for (i = 0; i < ts->count; i++)
-        ts->results[i] = TEST_INVALID;
-    return 1;
+    while (isspace((unsigned char)(*p)))
+        p++;
+    return p;
 }
 
 
-/* Start a program, connecting its stdout to a pipe on our end and its
-   stderr to /dev/null, and storing the file descriptor to read from in the
-   two argument.  Returns the PID of the new process.  Errors are fatal. */
+/*
+ * Start a program, connecting its stdout to a pipe on our end and its stderr
+ * to /dev/null, and storing the file descriptor to read from in the two
+ * argument.  Returns the PID of the new process.  Errors are fatal.
+ */
 static pid_t
 test_start(const char *path, int *fd)
 {
@@ -276,11 +327,13 @@
 }
 
 
-/* Back up over the output saying what test we were executing. */
+/*
+ * Back up over the output saying what test we were executing.
+ */
 static void
 test_backspace(struct testset *ts)
 {
-    int i;
+    unsigned int i;
 
     if (!isatty(STDOUT_FILENO))
         return;
@@ -294,57 +347,208 @@
 }
 
 
-/* Given a single line of output from a test, parse it and return the
-   success status of that test.  Anything printed to stdout not matching the
-   form /^(not )?ok \d+/ is ignored.  Sets ts->current to the test number
-   that just reported status. */
+/*
+ * Read the plan line of test output, which should contain the range of test
+ * numbers.  We may initialize the testset structure here if we haven't yet
+ * seen a test.  Return true if initialization succeeded and the test should
+ * continue, false otherwise.
+ */
+static int
+test_plan(const char *line, struct testset *ts)
+{
+    unsigned long i;
+    long n;
+
+    /*
+     * Accept a plan without the leading 1.. for compatibility with older
+     * versions of runtests.  This will only be allowed if we've not yet seen
+     * a test result.
+     */
+    line = skip_whitespace(line);
+    if (strncmp(line, "1..", 3) == 0)
+        line += 3;
+
+    /*
+     * Get the count, check it for validity, and initialize the struct.  If we
+     * have something of the form "1..0 # skip foo", the whole file was
+     * skipped; record that.  If we do skip the whole file, zero out all of
+     * our statistics, since they're no longer relevant.  strtol is called
+     * with a second argument to advance the line pointer past the count to
+     * make it simpler to detect the # skip case.
+     */
+    n = strtol(line, (char **) &line, 10);
+    if (n == 0) {
+        line = skip_whitespace(line);
+        if (*line == '#') {
+            line = skip_whitespace(line + 1);
+            if (strncasecmp(line, "skip", 4) == 0) {
+                line = skip_whitespace(line + 4);
+                if (*line != '\0') {
+                    ts->reason = xstrdup(line);
+                    ts->reason[strlen(ts->reason) - 1] = '\0';
+                }
+                ts->all_skipped = 1;
+                ts->aborted = 1;
+                ts->count = 0;
+                ts->passed = 0;
+                ts->skipped = 0;
+                ts->failed = 0;
+                return 0;
+            }
+        }
+    }
+    if (n <= 0) {
+        puts("ABORTED (invalid test count)");
+        ts->aborted = 1;
+        ts->reported = 1;
+        return 0;
+    }
+    if (ts->plan == PLAN_INIT && ts->allocated == 0) {
+        ts->count = n;
+        ts->allocated = n;
+        ts->plan = PLAN_FIRST;
+        ts->results = xmalloc(ts->count * sizeof(enum test_status));
+        for (i = 0; i < ts->count; i++)
+            ts->results[i] = TEST_INVALID;
+    } else if (ts->plan == PLAN_PENDING) {
+        if ((unsigned long) n < ts->count) {
+            printf("ABORTED (invalid test number %lu)\n", ts->count);
+            ts->aborted = 1;
+            ts->reported = 1;
+            return 0;
+        }
+        ts->count = n;
+        if ((unsigned long) n > ts->allocated) {
+            ts->results = xrealloc(ts->results, n * sizeof(enum test_status));
+            for (i = ts->allocated; i < ts->count; i++)
+                ts->results[i] = TEST_INVALID;
+            ts->allocated = n;
+        }
+        ts->plan = PLAN_FINAL;
+    }
+    return 1;
+}
+
+
+/*
+ * Given a single line of output from a test, parse it and return the success
+ * status of that test.  Anything printed to stdout not matching the form
+ * /^(not )?ok \d+/ is ignored.  Sets ts->current to the test number that just
+ * reported status.
+ */
 static void
 test_checkline(const char *line, struct testset *ts)
 {
     enum test_status status = TEST_PASS;
-    int current;
+    const char *bail;
+    char *end;
+    long number;
+    unsigned long i, current;
+    int outlen;
 
-    /* If the given line isn't newline-terminated, it was too big for an
-       fgets(), which means ignore it. */
+    /* Before anything, check for a test abort. */
+    bail = strstr(line, "Bail out!");
+    if (bail != NULL) {
+        bail = skip_whitespace(bail + strlen("Bail out!"));
+        if (*bail != '\0') {
+            size_t length;
+
+            length = strlen(bail);
+            if (bail[length - 1] == '\n')
+                length--;
+            test_backspace(ts);
+            printf("ABORTED (%.*s)\n", (int) length, bail);
+            ts->reported = 1;
+        }
+        ts->aborted = 1;
+        return;
+    }
+
+    /*
+     * If the given line isn't newline-terminated, it was too big for an
+     * fgets(), which means ignore it.
+     */
     if (line[strlen(line) - 1] != '\n')
         return;
 
+    /* If the line begins with a hash mark, ignore it. */
+    if (line[0] == '#')
+        return;
+
+    /* If we haven't yet seen a plan, look for one. */
+    if (ts->plan == PLAN_INIT && isdigit((unsigned char)(*line))) {
+        if (!test_plan(line, ts))
+            return;
+    } else if (strncmp(line, "1..", 3) == 0) {
+        if (ts->plan == PLAN_PENDING) {
+            if (!test_plan(line, ts))
+                return;
+        } else {
+            puts("ABORTED (multiple plans)");
+            ts->aborted = 1;
+            ts->reported = 1;
+            return;
+        }
+    }
+
     /* Parse the line, ignoring something we can't parse. */
     if (strncmp(line, "not ", 4) == 0) {
         status = TEST_FAIL;
         line += 4;
     }
-    if (strncmp(line, "ok ", 3) != 0)
+    if (strncmp(line, "ok", 2) != 0)
         return;
-    line += 3;
-    current = atoi(line);
-    if (current == 0)
-        return;
-    if (current < 0 || current > ts->count) {
+    line = skip_whitespace(line + 2);
+    errno = 0;
+    number = strtol(line, &end, 10);
+    if (errno != 0 || end == line)
+        number = ts->current + 1;
+    current = number;
+    if (number <= 0 || (current > ts->count && ts->plan == PLAN_FIRST)) {
         test_backspace(ts);
-        printf("invalid test number %d\n", current);
+        printf("ABORTED (invalid test number %lu)\n", current);
         ts->aborted = 1;
         ts->reported = 1;
         return;
     }
-    while (isspace((unsigned char)(*line)))
-        line++;
+
+    /* We have a valid test result.  Tweak the results array if needed. */
+    if (ts->plan == PLAN_INIT || ts->plan == PLAN_PENDING) {
+        ts->plan = PLAN_PENDING;
+        if (current > ts->count)
+            ts->count = current;
+        if (current > ts->allocated) {
+            unsigned long n;
+
+            n = (ts->allocated == 0) ? 32 : ts->allocated * 2;
+            if (n < current)
+                n = current;
+            ts->results = xrealloc(ts->results, n * sizeof(enum test_status));
+            for (i = ts->allocated; i < n; i++)
+                ts->results[i] = TEST_INVALID;
+            ts->allocated = n;
+        }
+    }
+
+    /*
+     * Handle directives.  We should probably do something more interesting
+     * with unexpected passes of todo tests.
+     */
     while (isdigit((unsigned char)(*line)))
         line++;
-    while (isspace((unsigned char)(*line)))
-        line++;
+    line = skip_whitespace(line);
     if (*line == '#') {
-        line++;
-        while (isspace((unsigned char)(*line)))
-            line++;
-        if (strncmp(line, "skip", 4) == 0)
+        line = skip_whitespace(line + 1);
+        if (strncasecmp(line, "skip", 4) == 0)
             status = TEST_SKIP;
+        if (strncasecmp(line, "todo", 4) == 0)
+            status = (status == TEST_FAIL) ? TEST_SKIP : TEST_FAIL;
     }
 
     /* Make sure that the test number is in range and not a duplicate. */
     if (ts->results[current - 1] != TEST_INVALID) {
         test_backspace(ts);
-        printf("duplicate test number %d\n", current);
+        printf("ABORTED (duplicate test number %lu)\n", current);
         ts->aborted = 1;
         ts->reported = 1;
         return;
@@ -355,34 +559,35 @@
         case TEST_PASS: ts->passed++;   break;
         case TEST_FAIL: ts->failed++;   break;
         case TEST_SKIP: ts->skipped++;  break;
-        default:                        break;
+        case TEST_INVALID:              break;
     }
     ts->current = current;
     ts->results[current - 1] = status;
     test_backspace(ts);
     if (isatty(STDOUT_FILENO)) {
-        ts->length = printf("%d/%d", current, ts->count);
+        outlen = printf("%lu/%lu", current, ts->count);
+        ts->length = (outlen >= 0) ? outlen : 0;
         fflush(stdout);
     }
 }
 
 
-/* Print out a range of test numbers, returning the number of characters it
-   took up.  Add a comma and a space before the range if chars indicates
-   that something has already been printed on the line, and print
-   ... instead if chars plus the space needed would go over the limit (use a
-   limit of 0 to disable this. */
-static int
-test_print_range(int first, int last, int chars, int limit)
+/*
+ * Print out a range of test numbers, returning the number of characters it
+ * took up.  Takes the first number, the last number, the number of characters
+ * already printed on the line, and the limit of number of characters the line
+ * can hold.  Add a comma and a space before the range if chars indicates that
+ * something has already been printed on the line, and print ... instead if
+ * chars plus the space needed would go over the limit (use a limit of 0 to
+ * disable this).
+ */
+static unsigned int
+test_print_range(unsigned long first, unsigned long last, unsigned int chars,
+                 unsigned int limit)
 {
-    int needed = 0;
-    int out = 0;
-    int n;
+    unsigned int needed = 0;
+    unsigned long n;
 
-    if (chars > 0) {
-        needed += 2;
-        if (!limit || chars <= limit) out += printf(", ");
-    }
     for (n = first; n > 0; n /= 10)
         needed++;
     if (last > first) {
@@ -390,35 +595,48 @@
             needed++;
         needed++;
     }
-    if (limit && chars + needed > limit) {
-        if (chars <= limit)
-            out += printf("...");
+    if (chars > 0)
+        needed += 2;
+    if (limit > 0 && chars + needed > limit) {
+        needed = 0;
+        if (chars <= limit) {
+            if (chars > 0) {
+                printf(", ");
+                needed += 2;
+            }
+            printf("...");
+            needed += 3;
+        }
     } else {
+        if (chars > 0)
+            printf(", ");
         if (last > first)
-            out += printf("%d-", first);
-        out += printf("%d", last);
+            printf("%lu-", first);
+        printf("%lu", last);
     }
-    return out;
+    return needed;
 }
 
 
-/* Summarize a single test set.  The second argument is 0 if the set exited
-   cleanly, a positive integer representing the exit status if it exited
-   with a non-zero status, and a negative integer representing the signal
-   that terminated it if it was killed by a signal. */
+/*
+ * Summarize a single test set.  The second argument is 0 if the set exited
+ * cleanly, a positive integer representing the exit status if it exited
+ * with a non-zero status, and a negative integer representing the signal
+ * that terminated it if it was killed by a signal.
+ */
 static void
 test_summarize(struct testset *ts, int status)
 {
-    int i;
-    int missing = 0;
-    int failed = 0;
-    int first = 0;
-    int last = 0;
+    unsigned long i;
+    unsigned long missing = 0;
+    unsigned long failed = 0;
+    unsigned long first = 0;
+    unsigned long last = 0;
 
     if (ts->aborted) {
-        fputs("aborted", stdout);
+        fputs("ABORTED", stdout);
         if (ts->count > 0)
-            printf(", passed %d/%d", ts->passed, ts->count - ts->skipped);
+            printf(" (passed %lu/%lu)", ts->passed, ts->count - ts->skipped);
     } else {
         for (i = 0; i < ts->count; i++) {
             if (ts->results[i] == TEST_INVALID) {
@@ -462,9 +680,9 @@
             fputs(!status ? "ok" : "dubious", stdout);
             if (ts->skipped > 0) {
                 if (ts->skipped == 1)
-                    printf(" (skipped %d test)", ts->skipped);
+                    printf(" (skipped %lu test)", ts->skipped);
                 else
-                    printf(" (skipped %d tests)", ts->skipped);
+                    printf(" (skipped %lu tests)", ts->skipped);
             }
         }
     }
@@ -477,27 +695,36 @@
 }
 
 
-/* Given a test set, analyze the results, classify the exit status, handle a
-   few special error messages, and then pass it along to test_summarize()
-   for the regular output. */
+/*
+ * Given a test set, analyze the results, classify the exit status, handle a
+ * few special error messages, and then pass it along to test_summarize() for
+ * the regular output.  Returns true if the test set ran successfully and all
+ * tests passed or were skipped, false otherwise.
+ */
 static int
 test_analyze(struct testset *ts)
 {
     if (ts->reported)
         return 0;
-    if (WIFEXITED(ts->status) && WEXITSTATUS(ts->status) != 0) {
+    if (ts->all_skipped) {
+        if (ts->reason == NULL)
+            puts("skipped");
+        else
+            printf("skipped (%s)\n", ts->reason);
+        return 1;
+    } else if (WIFEXITED(ts->status) && WEXITSTATUS(ts->status) != 0) {
         switch (WEXITSTATUS(ts->status)) {
         case CHILDERR_DUP:
             if (!ts->reported)
-                puts("can't dup file descriptors");
+                puts("ABORTED (can't dup file descriptors)");
             break;
         case CHILDERR_EXEC:
             if (!ts->reported)
-                puts("execution failed (not found?)");
+                puts("ABORTED (execution failed -- not found?)");
             break;
         case CHILDERR_STDERR:
             if (!ts->reported)
-                puts("can't open /dev/null");
+                puts("ABORTED (can't open /dev/null)");
             break;
         default:
             test_summarize(ts, WEXITSTATUS(ts->status));
@@ -507,6 +734,10 @@
     } else if (WIFSIGNALED(ts->status)) {
         test_summarize(ts, -WTERMSIG(ts->status));
         return 0;
+    } else if (ts->plan != PLAN_FIRST && ts->plan != PLAN_FINAL) {
+        puts("ABORTED (no valid test plan)");
+        ts->aborted = 1;
+        return 0;
     } else {
         test_summarize(ts, 0);
         return (ts->failed == 0);
@@ -514,55 +745,54 @@
 }
 
 
-/* Runs a single test set, accumulating and then reporting the results.
-   Returns true if the test set was successfully run and all tests passed,
-   false otherwise. */
+/*
+ * Runs a single test set, accumulating and then reporting the results.
+ * Returns true if the test set was successfully run and all tests passed,
+ * false otherwise.
+ */
 static int
 test_run(struct testset *ts)
 {
     pid_t testpid, child;
-    int outfd, i, status;
+    int outfd, status;
+    unsigned long i;
     FILE *output;
     char buffer[BUFSIZ];
-    char *file;
 
-    /* Initialize the test and our data structures, flagging this set in
-       error if the initialization fails. */
-    file = xmalloc(strlen(ts->file) + 3);
-    strcpy(file, ts->file);
-    strcat(file, ".t");
-    testpid = test_start(file, &outfd);
-    free(file);
+    /* Run the test program. */
+    testpid = test_start(ts->path, &outfd);
     output = fdopen(outfd, "r");
     if (!output) {
         puts("ABORTED");
         fflush(stdout);
         sysdie("fdopen failed");
     }
-    if (!fgets(buffer, sizeof(buffer), output))
-        ts->aborted = 1;
-    if (!ts->aborted && !test_init(buffer, ts)) {
-        while (fgets(buffer, sizeof(buffer), output))
-            ;
-        ts->aborted = 1;
-    }
 
     /* Pass each line of output to test_checkline(). */
     while (!ts->aborted && fgets(buffer, sizeof(buffer), output))
         test_checkline(buffer, ts);
-    if (ferror(output))
+    if (ferror(output) || ts->plan == PLAN_INIT)
         ts->aborted = 1;
     test_backspace(ts);
 
-    /* Close the output descriptor, retrieve the exit status, and pass that
-       information to test_analyze() for eventual output. */
+    /*
+     * Consume the rest of the test output, close the output descriptor,
+     * retrieve the exit status, and pass that information to test_analyze()
+     * for eventual output.
+     */
+    while (fgets(buffer, sizeof(buffer), output))
+        ;
     fclose(output);
     child = waitpid(testpid, &ts->status, 0);
     if (child == (pid_t) -1) {
-        puts("ABORTED");
-        fflush(stdout);
+        if (!ts->reported) {
+            puts("ABORTED");
+            fflush(stdout);
+        }
         sysdie("waitpid for %u failed", (unsigned int) testpid);
     }
+    if (ts->all_skipped)
+        ts->aborted = 0;
     status = test_analyze(ts);
 
     /* Convert missing tests to failed tests. */
@@ -582,7 +812,8 @@
 test_fail_summary(const struct testlist *fails)
 {
     struct testset *ts;
-    int i, chars, total, first, last;
+    unsigned int chars;
+    unsigned long i, first, last, total;
 
     puts(header);
 
@@ -591,7 +822,7 @@
     for (; fails; fails = fails->next) {
         ts = fails->ts;
         total = ts->count - ts->skipped;
-        printf("%-26.26s %4d/%-4d %3.0f%% %4d ", ts->file, ts->failed,
+        printf("%-26.26s %4lu/%-4lu %3.0f%% %4lu ", ts->file, ts->failed,
                total, total ? (ts->failed * 100.0) / total : 0,
                ts->skipped);
         if (WIFEXITED(ts->status))
@@ -607,47 +838,104 @@
         last = 0;
         for (i = 0; i < ts->count; i++) {
             if (ts->results[i] == TEST_FAIL) {
-                if (first && i == last)
+                if (first != 0 && i == last)
                     last = i + 1;
                 else {
-                    if (first)
-                        chars += test_print_range(first, last, chars, 20);
+                    if (first != 0)
+                        chars += test_print_range(first, last, chars, 19);
                     first = i + 1;
                     last = i + 1;
                 }
             }
         }
-        if (first)
-            test_print_range(first, last, chars, 20);
+        if (first != 0)
+            test_print_range(first, last, chars, 19);
         putchar('\n');
+        free(ts->file);
+        free(ts->path);
+        free(ts->results);
+        if (ts->reason != NULL)
+            free(ts->reason);
+        free(ts);
     }
 }
 
 
-/* Run a batch of tests from a given file listing each test on a line by
-   itself.  The file must be rewindable.  Returns true iff all tests
-   passed. */
+/*
+ * Given the name of a test, a pointer to the testset struct, and the source
+ * and build directories, find the test.  We try first relative to the current
+ * directory, then in the build directory (if not NULL), then in the source
+ * directory.  In each of those directories, we first try a "-t" extension and
+ * then a ".t" extension.  When we find an executable program, we fill in the
+ * path member of the testset struct.  If none of those paths are executable,
+ * just fill in the name of the test with "-t" appended.
+ *
+ * The caller is responsible for freeing the path member of the testset
+ * struct.
+ */
+static void
+find_test(const char *name, struct testset *ts, const char *source,
+          const char *build)
+{
+    char *path;
+    const char *bases[4];
+    unsigned int i;
+
+    bases[0] = ".";
+    bases[1] = build;
+    bases[2] = source;
+    bases[3] = NULL;
+
+    for (i = 0; bases[i] != NULL; i++) {
+        path = xmalloc(strlen(bases[i]) + strlen(name) + 4);
+        sprintf(path, "%s/%s-t", bases[i], name);
+        if (access(path, X_OK) != 0)
+            path[strlen(path) - 2] = '.';
+        if (access(path, X_OK) == 0)
+            break;
+        free(path);
+        path = NULL;
+    }
+    if (path == NULL) {
+        path = xmalloc(strlen(name) + 3);
+        sprintf(path, "%s-t", name);
+    }
+    ts->path = path;
+}
+
+
+/*
+ * Run a batch of tests from a given file listing each test on a line by
+ * itself.  Takes two additional parameters: the root of the source directory
+ * and the root of the build directory.  Test programs will be first searched
+ * for in the current directory, then the build directory, then the source
+ * directory.  The file must be rewindable.  Returns true iff all tests
+ * passed.
+ */
 static int
-test_batch(const char *testlist)
+test_batch(const char *testlist, const char *source, const char *build)
 {
     FILE *tests;
-    size_t length, i;
-    size_t longest = 0;
+    unsigned int length, i;
+    unsigned int longest = 0;
     char buffer[BUFSIZ];
-    int line;
+    unsigned int line;
     struct testset ts, *tmp;
     struct timeval start, end;
     struct rusage stats;
-    struct testlist *failhead = 0;
-    struct testlist *failtail = 0;
-    int total = 0;
-    int passed = 0;
-    int skipped = 0;
-    int failed = 0;
-    int aborted = 0;
+    struct testlist *failhead = NULL;
+    struct testlist *failtail = NULL;
+    struct testlist *next;
+    unsigned long total = 0;
+    unsigned long passed = 0;
+    unsigned long skipped = 0;
+    unsigned long failed = 0;
+    unsigned long aborted = 0;
 
-    /* Open our file of tests to run and scan it, checking for lines that
-       are too long and searching for the longest line. */
+    /*
+     * Open our file of tests to run and scan it, checking for lines that
+     * are too long and searching for the longest line.
+     */
     tests = fopen(testlist, "r");
     if (!tests)
         sysdie("can't open %s", testlist);
@@ -656,7 +944,7 @@
         line++;
         length = strlen(buffer) - 1;
         if (buffer[length] != '\n') {
-            fprintf(stderr, "%s:%d: line too long\n", testlist, line);
+            fprintf(stderr, "%s:%u: line too long\n", testlist, line);
             exit(1);
         }
         if (length > longest)
@@ -665,8 +953,10 @@
     if (fseek(tests, 0, SEEK_SET) == -1)
         sysdie("can't rewind %s", testlist);
 
-    /* Add two to longest and round up to the nearest tab stop.  This is how
-       wide the column for printing the current test name will be. */
+    /*
+     * Add two to longest and round up to the nearest tab stop.  This is how
+     * wide the column for printing the current test name will be.
+     */
     longest += 2;
     if (longest % 8)
         longest += 8 - (longest % 8);
@@ -674,41 +964,54 @@
     /* Start the wall clock timer. */
     gettimeofday(&start, NULL);
 
-    /* Now, plow through our tests again, running each one.  Check line
-       length again out of paranoia. */
+    /*
+     * Now, plow through our tests again, running each one.  Check line
+     * length again out of paranoia.
+     */
     line = 0;
     while (fgets(buffer, sizeof(buffer), tests)) {
         line++;
         length = strlen(buffer) - 1;
         if (buffer[length] != '\n') {
-            fprintf(stderr, "%s:%d: line too long\n", testlist, line);
+            fprintf(stderr, "%s:%u: line too long\n", testlist, line);
             exit(1);
         }
         buffer[length] = '\0';
         fputs(buffer, stdout);
         for (i = length; i < longest; i++)
             putchar('.');
+        if (isatty(STDOUT_FILENO))
+            fflush(stdout);
         memset(&ts, 0, sizeof(ts));
+        ts.plan = PLAN_INIT;
         ts.file = xstrdup(buffer);
-        if (!test_run(&ts)) {
+        find_test(buffer, &ts, source, build);
+        ts.reason = NULL;
+        if (test_run(&ts)) {
+            free(ts.file);
+            free(ts.path);
+            free(ts.results);
+            if (ts.reason != NULL)
+                free(ts.reason);
+        } else {
             tmp = xmalloc(sizeof(struct testset));
             memcpy(tmp, &ts, sizeof(struct testset));
             if (!failhead) {
                 failhead = xmalloc(sizeof(struct testset));
                 failhead->ts = tmp;
-                failhead->next = 0;
+                failhead->next = NULL;
                 failtail = failhead;
             } else {
                 failtail->next = xmalloc(sizeof(struct testset));
                 failtail = failtail->next;
                 failtail->ts = tmp;
-                failtail->next = 0;
+                failtail->next = NULL;
             }
         }
         aborted += ts.aborted;
-        total += ts.count;
+        total += ts.count + ts.all_skipped;
         passed += ts.passed;
-        skipped += ts.skipped;
+        skipped += ts.skipped + ts.all_skipped;
         failed += ts.failed;
     }
     total -= skipped;
@@ -718,28 +1021,35 @@
     getrusage(RUSAGE_CHILDREN, &stats);
 
     /* Print out our final results. */
-    if (failhead) test_fail_summary(failhead);
+    if (failhead != NULL) {
+        test_fail_summary(failhead);
+        while (failhead != NULL) {
+            next = failhead->next;
+            free(failhead);
+            failhead = next;
+        }
+    }
     putchar('\n');
     if (aborted != 0) {
         if (aborted == 1)
-            printf("Aborted %d test set", aborted);
+            printf("Aborted %lu test set", aborted);
         else
-            printf("Aborted %d test sets", aborted);
-        printf(", passed %d/%d tests", passed, total);
+            printf("Aborted %lu test sets", aborted);
+        printf(", passed %lu/%lu tests", passed, total);
     }
     else if (failed == 0)
         fputs("All tests successful", stdout);
     else
-        printf("Failed %d/%d tests, %.2f%% okay", failed, total,
+        printf("Failed %lu/%lu tests, %.2f%% okay", failed, total,
                (total - failed) * 100.0 / total);
     if (skipped != 0) {
         if (skipped == 1)
-            printf(", %d test skipped", skipped);
+            printf(", %lu test skipped", skipped);
         else
-            printf(", %d tests skipped", skipped);
+            printf(", %lu tests skipped", skipped);
     }
     puts(".");
-    printf("Files=%d,  Tests=%d", line, total);
+    printf("Files=%u,  Tests=%lu", line, total);
     printf(",  %.2f seconds", tv_diff(&end, &start));
     printf(" (%.2f usr + %.2f sys = %.2f CPU)\n",
            tv_seconds(&stats.ru_utime), tv_seconds(&stats.ru_stime),
@@ -748,14 +1058,81 @@
 }
 
 
-/* Main routine.  Given a file listing tests, run each test listed. */
+/*
+ * Run a single test case.  This involves just running the test program after
+ * having done the environment setup and finding the test program.
+ */
+static void
+test_single(const char *program, const char *source, const char *build)
+{
+    struct testset ts;
+
+    memset(&ts, 0, sizeof(ts));
+    find_test(program, &ts, source, build);
+    if (execl(ts.path, ts.path, (char *) 0) == -1)
+        sysdie("cannot exec %s", ts.path);
+}
+
+
+/*
+ * Main routine.  Set the SOURCE and BUILD environment variables and then,
+ * given a file listing tests, run each test listed.
+ */
 int
 main(int argc, char *argv[])
 {
-    if (argc != 2) {
+    int option;
+    int single = 0;
+    char *setting;
+    const char *list;
+    const char *source = SOURCE;
+    const char *build = BUILD;
+
+    while ((option = getopt(argc, argv, "b:os:")) != EOF) {
+        switch (option) {
+        case 'b':
+            build = optarg;
+            break;
+        case 'o':
+            single = 1;
+            break;
+        case 's':
+            source = optarg;
+            break;
+        default:
+            exit(1);
+        }
+    }
+    argc -= optind;
+    argv += optind;
+    if (argc != 1) {
         fprintf(stderr, "Usage: runtests <test-list>\n");
         exit(1);
     }
-    printf(banner, argv[1]);
-    exit(test_batch(argv[1]) ? 0 : 1);
+
+    if (source != NULL) {
+        setting = xmalloc(strlen("SOURCE=") + strlen(source) + 1);
+        sprintf(setting, "SOURCE=%s", source);
+        if (putenv(setting) != 0)
+            sysdie("cannot set SOURCE in the environment");
+    }
+    if (build != NULL) {
+        setting = xmalloc(strlen("BUILD=") + strlen(build) + 1);
+        sprintf(setting, "BUILD=%s", build);
+        if (putenv(setting) != 0)
+            sysdie("cannot set BUILD in the environment");
+    }
+
+    if (single) {
+        test_single(argv[0], source, build);
+        exit(0);
+    } else {
+        list = strrchr(argv[0], '/');
+        if (list == NULL)
+            list = argv[0];
+        else
+            list++;
+        printf(banner, list);
+        exit(test_batch(argv[0], source, build) ? 0 : 1);
+    }
 }




More information about the inn-committers mailing list