ab: Add the -W option to use worker threads.

This allows for multiple CPUs to handle the load, the number of requests and
concurrency level asked are distributed over the configured number of workers,
allowing for as much parallelism.

On unixes (only for now), -W0 will use all the CPUs available on the system.

To avoid synchronization during runtime, the stats and requests times are
gathered per worker and consolidated at the end of the run before being
printed.

Connection closes, keepalives and errors are now handled in a single place,
namely cleanup_connection(), which takes care of the good/bad state of each
request based on the response fully received or not.

When multiple workers are running, SIGINT is handled by the main thread only
and masked in workers, workers are asked to stop and woken up if waiting in
poll().

A single worker is started first to determine the connectivity with the peer,
if that fails (10 tries) ab will stop early still without starting the other
workers, otherwise the first worker will signal the main thread to start the
others.



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1900362 13f79535-47bb-0310-9956-ffa450edef68

1 files changed, 889 insertions, 521 deletions

diff --git a/support/ab.c b/support/ab.c
index d467ffa4a1..7906b84c93 100644
--- a/support/ab.c
+++ b/support/ab.c
@@ -137,6 +137,16 @@
 #include "ap_release.h"
 #include "apr_poll.h"
 
+#include "apr_atomic.h"
+#if APR_HAS_THREADS
+#include "apr_thread_proc.h"
+#include "apr_thread_mutex.h"
+#include "apr_thread_cond.h"
+#if APR_HAVE_PTHREAD_H
+#include <pthread.h>
+#endif
+#endif
+
 #define APR_WANT_STRFUNC
 #include "apr_want.h"
 
@@ -144,6 +154,7 @@
 #ifdef NOT_ASCII
 #include "apr_xlate.h"
 #endif
+
 #if APR_HAVE_STDIO_H
 #include <stdio.h>
 #endif
@@ -241,6 +252,8 @@ typedef STACK_OF(X509) X509_STACK_TYPE;
 /* maximum number of requests on a time limited test */
 #define MAX_REQUESTS (INT_MAX > 50000 ? 50000 : INT_MAX)
 
+#define ROUND_UP(x, y) ((((x) + (y) - 1) / (y)) * (y))
+
 /* connection state
  * don't add enums or rearrange or otherwise change values without
  * visiting set_conn_state()
@@ -260,11 +273,12 @@ typedef enum {
 
 #define CBUFFSIZE (8192)
 
-typedef struct connection connection;
+/* forward declare */
+struct worker;
 
 struct connection {
     APR_RING_ENTRY(connection) delay_list;
-    void (*delay_fn)(struct connection *);
+    struct worker *worker;
     apr_pool_t *ctx;
     apr_socket_t *aprsock;
     apr_pollfd_t pollfd;
@@ -284,10 +298,9 @@ struct connection {
                connect,         /* Connected, start writing */
                endwrite,        /* Request written */
                beginread,       /* First byte of input */
-               done;            /* Connection closed */
+               end;             /* Connection closed */
 
-    apr_uint64_t keptalive;     /* subsequent keepalive requests */
-    int socknum;
+    apr_size_t keptalive;       /* subsequent keepalive requests */
 #ifdef USE_SSL
     SSL *ssl;
 #endif
@@ -300,9 +313,57 @@ struct data {
     apr_interval_time_t time;     /* time for connection */
 };
 
-APR_RING_HEAD(delay_head_t, connection);
+struct metrics {
+    apr_size_t doclen;          /* the length the document should be */
+    apr_int64_t totalread;      /* total number of bytes read */
+    apr_int64_t totalbread;     /* totoal amount of entity body read */
+    apr_int64_t totalposted;    /* total number of bytes posted, inc. headers */
+    apr_int64_t done;           /* number of requests we have done */
+    apr_int64_t doneka;         /* number of keep alive connections done */
+    apr_int64_t good, bad;      /* number of good and bad requests */
+    int epipe;                  /* number of broken pipe writes */
+    int err_length;             /* requests failed due to response length */
+    int err_conn;               /* requests failed due to connection drop */
+    int err_recv;               /* requests failed due to broken read */
+    int err_except;             /* requests failed due to exception */
+    int err_response;           /* requests with invalid or non-200 response */
+    int concurrent;             /* Number of multiple requests actually made */
+#ifdef USE_SSL
+    char ssl_info[128];
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+    char ssl_tmp_key[128];
+#endif
+#endif
+};
+
+APR_RING_HEAD(delayed_ring_t, connection);
+
+struct worker {
+    apr_pool_t *pool;
+#if APR_HAS_THREADS
+    apr_thread_t *thd;
+#endif
+    apr_pollset_t *pollset;
+    apr_sockaddr_t *destsa;
+
+    int slot;
+    int requests;
+    int concurrency;
+    int succeeded_once;  /* response header received once */
+    apr_int64_t started; /* number of requests started, so no excess */
 
-struct delay_head_t delay_head;
+    struct data *stats;
+    struct connection *conns;
+    struct delayed_ring_t delayed_ring;
+
+    struct metrics metrics;
+
+    char buffer[CBUFFSIZE];  /* throw-away buffer to read stuff into */
+};
+
+/* global metrics (consolidated from workers') */
+static struct metrics metrics;
+static void consolidate_metrics(void);
 
 #define ap_min(a,b) (((a)<(b))?(a):(b))
 #define ap_max(a,b) (((a)>(b))?(a):(b))
@@ -318,9 +379,9 @@ enum {NO_METH = 0, GET, HEAD, PUT, POST, CUSTOM_METHOD} method = NO_METH;
 const char *method_str[] = {"bug", "GET", "HEAD", "PUT", "POST", ""};
 int send_body = 0;      /* non-zero if sending body with request */
 int requests = 1;       /* Number of requests to make */
+int num_workers = 1;    /* Number of worker threads to use */
 int heartbeatres = 100; /* How often do we say we're alive */
 int concurrency = 1;    /* Number of multiple requests to make */
-int concurrent = 0;     /* Number of multiple requests actually made */
 int percentile = 1;     /* Show percentile served */
 int nolength = 0;       /* Accept variable document length */
 int confidence = 1;     /* Show confidence estimator and warnings */
@@ -348,8 +409,10 @@ const char *csvperc;          /* CSV Percentile file */
 const char *fullurl;
 const char *colonhost;
 int isproxy = 0;
+apr_interval_time_t hbperiod = 0; /* heartbeat period (when time limited) */
 apr_interval_time_t aprtimeout = apr_time_from_sec(30); /* timeout value */
 apr_interval_time_t ramp = apr_time_from_msec(0); /* ramp delay */
+int pollset_wakeable = 0;
 
 /* overrides for ab-generated common headers */
 const char *opt_host;   /* which optional "Host:" header specified, if any */
@@ -364,30 +427,11 @@ const char *tablestring;
 const char *trstring;
 const char *tdstring;
 
-apr_size_t doclen = 0;     /* the length the document should be */
-apr_int64_t totalread = 0;    /* total number of bytes read */
-apr_int64_t totalbread = 0;   /* totoal amount of entity body read */
-apr_int64_t totalposted = 0;  /* total number of bytes posted, inc. headers */
-int started = 0;           /* number of requests started, so no excess */
-int done = 0;              /* number of requests we have done */
-int doneka = 0;            /* number of keep alive connections done */
-int good = 0, bad = 0;     /* number of good and bad requests */
-int epipe = 0;             /* number of broken pipe writes */
-int err_length = 0;        /* requests failed due to response length */
-int err_conn = 0;          /* requests failed due to connection drop */
-int err_recv = 0;          /* requests failed due to broken read */
-int err_except = 0;        /* requests failed due to exception */
-int err_response = 0;      /* requests with invalid or non-200 response */
-
 #ifdef USE_SSL
 int is_ssl;
 SSL_CTX *ssl_ctx;
 char *ssl_cipher = NULL;
-char *ssl_info = NULL;
 char *ssl_cert = NULL;
-#if OPENSSL_VERSION_NUMBER >= 0x10002000L
-char *ssl_tmp_key = NULL;
-#endif
 BIO *bio_out,*bio_err;
 #ifdef HAVE_TLSEXT
 int tls_use_sni = 1;         /* used by default, -I disables it */
@@ -395,26 +439,22 @@ const char *tls_sni = NULL; /* 'opt_host' if any, 'hostname' otherwise */
 #endif
 #endif
 
-apr_time_t start, lasttime, stoptime;
+apr_time_t start, logtime;
+volatile apr_time_t lasttime, stoptime;
 
 /* global request (and its length) */
 char _request[8192];
 char *request = _request;
 apr_size_t reqlen;
-int requests_initialized = 0;
-
-/* one global throw-away buffer to read stuff into */
-char buffer[8192];
 
 /* interesting percentiles */
 int percs[] = {50, 66, 75, 80, 90, 95, 98, 99, 100};
 
-struct connection *con;     /* connection array */
+struct worker *workers;     /* worker threads */
+struct connection *conns;   /* connection array */
 struct data *stats;         /* data for each request */
 apr_pool_t *cntxt;
 
-apr_pollset_t *readbits;
-
 apr_sockaddr_t *mysa;
 apr_sockaddr_t *destsa;
 
@@ -422,11 +462,42 @@ apr_sockaddr_t *destsa;
 apr_xlate_t *from_ascii, *to_ascii;
 #endif
 
+#if APR_HAS_THREADS
+static apr_thread_mutex_t *workers_mutex;
+static apr_thread_cond_t *workers_can_start;
+#endif
+
+static APR_INLINE int worker_should_exit(struct worker *worker)
+{
+    return (lasttime >= stoptime
+            || (!tlimit && worker->metrics.done >= worker->requests));
+}
+static APR_INLINE int worker_should_stop(struct worker *worker)
+{
+    return (worker_should_exit(worker)
+            || (!tlimit && worker->started >= worker->requests));
+}
+
 static void write_request(struct connection * c);
-static void close_connection(struct connection * c);
+static void retry_connection(struct connection *c, apr_status_t status);
+static void cleanup_connection(struct connection *c, int reuse);
 
+static APR_INLINE void reuse_connection(struct connection *c)
+{
+    cleanup_connection(c, 1);
+}
+static APR_INLINE void close_connection(struct connection *c)
+{
+    cleanup_connection(c, 0);
+}
+static APR_INLINE void abort_connection(struct connection *c)
+{
+    c->gotheader = 0; /* invalidate */
+    close_connection(c);
+}
+
+static void output_results(void);
 static void output_html_results(void);
-static void output_results(int sig);
 
 /* --------------------------------------------------------- */
 
@@ -435,13 +506,15 @@ static void output_results(int sig);
 static void err(const char *s)
 {
     fprintf(stderr, "%s\n", s);
-    if (done)
-        printf("Total of %d requests completed\n" , done);
+    fflush(stderr);
 
+    consolidate_metrics();
+    if (metrics.done)
+        printf("Total of %" APR_INT64_T_FMT " requests completed\n" , metrics.done);
     if (use_html)
         output_html_results();
     else
-        output_results(0);
+        output_results();
 
     exit(1);
 }
@@ -452,53 +525,21 @@ static void apr_err(const char *s, apr_status_t rv)
 {
     char buf[120];
 
-    fprintf(stderr,
-        "%s: %s (%d)\n",
-        s, apr_strerror(rv, buf, sizeof buf), rv);
-    if (done)
-        printf("Total of %d requests completed\n" , done);
+    fprintf(stderr, "%s: %s (%d)\n",
+            s, apr_strerror(rv, buf, sizeof buf), rv);
+    fflush(stderr);
 
+    consolidate_metrics();
+    if (metrics.done)
+        printf("Total of %" APR_INT64_T_FMT " requests completed\n" , metrics.done);
     if (use_html)
         output_html_results();
     else
-        output_results(0);
+        output_results();
 
     exit(rv);
 }
 
-static void *xmalloc(size_t size)
-{
-    void *ret = malloc(size);
-    if (ret == NULL) {
-        fprintf(stderr, "Could not allocate memory (%"
-                APR_SIZE_T_FMT" bytes)\n", size);
-        exit(1);
-    }
-    return ret;
-}
-
-static void *xcalloc(size_t num, size_t size)
-{
-    void *ret = calloc(num, size);
-    if (ret == NULL) {
-        fprintf(stderr, "Could not allocate memory (%"
-                APR_SIZE_T_FMT" bytes)\n", size*num);
-        exit(1);
-    }
-    return ret;
-}
-
-static char *xstrdup(const char *s)
-{
-    char *ret = strdup(s);
-    if (ret == NULL) {
-        fprintf(stderr, "Could not allocate memory (%"
-                APR_SIZE_T_FMT " bytes)\n", strlen(s));
-        exit(1);
-    }
-    return ret;
-}
-
 /*
  * Similar to standard strstr() but we ignore case in this version.
  * Copied from ap_strcasestr().
@@ -549,7 +590,7 @@ static void set_polled_events(struct connection *c, apr_int16_t new_reqevents)
 
     if (c->pollfd.reqevents != new_reqevents) {
         if (c->pollfd.reqevents != 0) {
-            rv = apr_pollset_remove(readbits, &c->pollfd);
+            rv = apr_pollset_remove(c->worker->pollset, &c->pollfd);
             if (rv != APR_SUCCESS) {
                 apr_err("apr_pollset_remove()", rv);
             }
@@ -557,7 +598,7 @@ static void set_polled_events(struct connection *c, apr_int16_t new_reqevents)
 
         if (new_reqevents != 0) {
             c->pollfd.reqevents = new_reqevents;
-            rv = apr_pollset_add(readbits, &c->pollfd);
+            rv = apr_pollset_add(c->worker->pollset, &c->pollfd);
             if (rv != APR_SUCCESS) {
                 apr_err("apr_pollset_add()", rv);
             }
@@ -630,7 +671,6 @@ static int ssl_rand_choosenum(int l, int h)
     int i;
     char buf[50];
 
-    srand((unsigned int)time(NULL));
     apr_snprintf(buf, sizeof(buf), "%.0f",
                  (((double)(rand()%RAND_MAX)/RAND_MAX)*(h-l)));
     i = atoi(buf)+1;
@@ -642,15 +682,15 @@ static int ssl_rand_choosenum(int l, int h)
 static void ssl_rand_seed(void)
 {
     int n, l;
-    time_t t;
+    apr_time_t t;
     pid_t pid;
     unsigned char stackdata[256];
 
     /*
      * seed in the current time (usually just 4 bytes)
      */
-    t = time(NULL);
-    l = sizeof(time_t);
+    t = lasttime;
+    l = sizeof(apr_time_t);
     RAND_seed((unsigned char *)&t, l);
 
     /*
@@ -748,12 +788,15 @@ static void ssl_print_info(struct connection *c)
 
 static void ssl_proceed_handshake(struct connection *c)
 {
-    int do_next = 1;
+    struct worker *worker = c->worker;
+    int again;
 
-    while (do_next) {
+    do {
         int ret, ecode;
         apr_status_t status;
 
+        again = 0; /* until further notice */
+
         ret = SSL_do_handshake(c->ssl);
         ecode = SSL_get_error(c->ssl, ret);
 
@@ -761,7 +804,7 @@ static void ssl_proceed_handshake(struct connection *c)
         case SSL_ERROR_NONE:
             if (verbosity >= 2)
                 ssl_print_info(c);
-            if (ssl_info == NULL) {
+            if (!worker->metrics.ssl_info[0]) {
                 AB_SSL_CIPHER_CONST SSL_CIPHER *ci;
                 X509 *cert;
                 int sk_bits, pk_bits, swork;
@@ -774,24 +817,23 @@ static void ssl_proceed_handshake(struct connection *c)
                 else
                     pk_bits = 0;  /* Anon DH */
 
-                ssl_info = xmalloc(128);
-                apr_snprintf(ssl_info, 128, "%s,%s,%d,%d",
+                apr_snprintf(worker->metrics.ssl_info, sizeof(worker->metrics.ssl_info),
+                             "%s,%s,%d,%d",
                              SSL_get_version(c->ssl),
                              SSL_CIPHER_get_name(ci),
                              pk_bits, sk_bits);
             }
 #if OPENSSL_VERSION_NUMBER >= 0x10002000L
-            if (ssl_tmp_key == NULL) {
+            if (!worker->metrics.ssl_tmp_key[0] && !worker->metrics.ssl_tmp_key[1]) {
                 EVP_PKEY *key;
                 if (SSL_get_server_tmp_key(c->ssl, &key)) {
-                    ssl_tmp_key = xmalloc(128);
                     switch (EVP_PKEY_id(key)) {
                     case EVP_PKEY_RSA:
-                        apr_snprintf(ssl_tmp_key, 128, "RSA %d bits",
+                        apr_snprintf(worker->metrics.ssl_tmp_key, 128, "RSA %d bits",
                                      EVP_PKEY_bits(key));
                         break;
                     case EVP_PKEY_DH:
-                        apr_snprintf(ssl_tmp_key, 128, "DH %d bits",
+                        apr_snprintf(worker->metrics.ssl_tmp_key, 128, "DH %d bits",
                                      EVP_PKEY_bits(key));
                         break;
 #ifndef OPENSSL_NO_EC
@@ -804,38 +846,36 @@ static void ssl_proceed_handshake(struct connection *c)
                         if (!cname)
                             cname = OBJ_nid2sn(nid);
 
-                        apr_snprintf(ssl_tmp_key, 128, "ECDH %s %d bits",
-                                     cname,
-                                     EVP_PKEY_bits(key));
+                        apr_snprintf(worker->metrics.ssl_tmp_key, 128, "ECDH %s %d bits",
+                                     cname, EVP_PKEY_bits(key));
                         break;
                         }
 #endif
                     default:
-                        apr_snprintf(ssl_tmp_key, 128, "%s %d bits",
+                        apr_snprintf(worker->metrics.ssl_tmp_key, 128, "%s %d bits",
                                      OBJ_nid2sn(EVP_PKEY_id(key)),
                                      EVP_PKEY_bits(key));
                         break;
                     }
                     EVP_PKEY_free(key);
                 }
+                else {
+                    /* not available, do not reenter here still */
+                    worker->metrics.ssl_tmp_key[1] = !0;
+                }
             }
 #endif
             write_request(c);
-
-            do_next = 0;
             break;
-        case SSL_ERROR_WANT_READ:
 
+        case SSL_ERROR_WANT_READ:
             set_conn_state(c, STATE_HANDSHAKE, APR_POLLIN);
-
-            do_next = 0;
             break;
-        case SSL_ERROR_WANT_WRITE:
 
+        case SSL_ERROR_WANT_WRITE:
             set_conn_state(c, STATE_HANDSHAKE, APR_POLLOUT);
-
-            do_next = 0;
             break;
+
         case SSL_ERROR_WANT_CONNECT:
         case SSL_ERROR_SSL:
         case SSL_ERROR_SYSCALL:
@@ -844,17 +884,21 @@ static void ssl_proceed_handshake(struct connection *c)
             BIO_printf(bio_err, "SSL handshake failed (%d): %s\n", ecode,
                     apr_psprintf(c->ctx, "%pm", &status));
             ERR_print_errors(bio_err);
-            close_connection(c);
-            do_next = 0;
+            abort_connection(c);
+            break;
+
+        default:
+            again = 1;
             break;
         }
-    }
+    } while (again);
 }
 
 #endif /* USE_SSL */
 
 static void write_request(struct connection * c)
 {
+    struct worker *worker = c->worker;
 
     do {
         apr_time_t tnow;
@@ -867,7 +911,9 @@ static void write_request(struct connection * c)
          * First time round ?
          */
         if (c->rwrite == 0) {
-            apr_socket_timeout_set(c->aprsock, 0);
+            /* zero connect time with keep-alive */
+            if (c->keptalive)
+                c->start = tnow;
             c->connect = tnow;
             c->rwrote = 0;
             c->rwrite = reqlen;
@@ -877,7 +923,7 @@ static void write_request(struct connection * c)
         }
         else if (tnow > c->connect + aprtimeout) {
             printf("Send request timed out!\n");
-            close_connection(c);
+            abort_connection(c);
             return;
         }
 
@@ -887,19 +933,15 @@ static void write_request(struct connection * c)
             if (e <= 0) {
                 switch (SSL_get_error(c->ssl, e)) {
                 case SSL_ERROR_WANT_READ:
-
                     set_conn_state(c, STATE_WRITE, APR_POLLIN);
-
                     break;
                 case SSL_ERROR_WANT_WRITE:
-
                     set_conn_state(c, STATE_WRITE, APR_POLLOUT);
-
                     break;
                 default:
                     BIO_printf(bio_err, "SSL write failed - closing connection\n");
                     ERR_print_errors(bio_err);
-                    close_connection (c);
+                    abort_connection(c);
                     break;
                 }
                 return;
@@ -911,24 +953,24 @@ static void write_request(struct connection * c)
         {
             e = apr_socket_send(c->aprsock, request + c->rwrote, &l);
             if (e != APR_SUCCESS && !l) {
-                if (!APR_STATUS_IS_EAGAIN(e)) {
-                    epipe++;
-                    printf("Send request failed!\n");
-                    close_connection(c);
+                if (APR_STATUS_IS_EAGAIN(e)) {
+                    set_conn_state(c, STATE_WRITE, APR_POLLOUT);
                 }
                 else {
-                    set_conn_state(c, STATE_WRITE, APR_POLLOUT);
+                    worker->metrics.epipe++;
+                    printf("Send request failed!\n");
+                    abort_connection(c);
                 }
                 return;
             }
         }
-        totalposted += l;
+        worker->metrics.totalposted += l;
         c->rwrote += l;
         c->rwrite -= l;
     } while (c->rwrite);
 
     c->endwrite = lasttime = apr_time_now();
-    started++;
+    worker->started++;
 
     set_conn_state(c, STATE_READ, APR_POLLIN);
 }
@@ -975,13 +1017,51 @@ static int compwait(struct data * a, struct data * b)
     return 0;
 }
 
-static void output_results(int sig)
+static void consolidate_metrics(void)
 {
-    double timetaken;
+    int i;
+
+    for (i = 0; i < num_workers; i++) {
+        struct worker *worker = &workers[i];
+
+        metrics.done += worker->metrics.done;
+        metrics.doneka += worker->metrics.doneka;
+        metrics.good += worker->metrics.good;
+        metrics.bad += worker->metrics.bad;
+
+        metrics.epipe += worker->metrics.epipe;
+        metrics.err_length += worker->metrics.err_length;
+        metrics.err_conn += worker->metrics.err_conn;
+        metrics.err_recv += worker->metrics.err_recv;
+        metrics.err_except += worker->metrics.err_except;
+        metrics.err_response += worker->metrics.err_response;
+
+        metrics.concurrent += worker->metrics.concurrent;
+        metrics.totalread += worker->metrics.totalread;
+        metrics.totalbread += worker->metrics.totalbread;
+        metrics.totalposted += worker->metrics.totalposted;
 
-    if (sig) {
-        lasttime = apr_time_now();  /* record final time if interrupted */
+        if (metrics.doclen == 0) {
+            metrics.doclen = worker->metrics.doclen;
+        }
+
+#ifdef USE_SSL
+        if (is_ssl && !metrics.ssl_info[0] && worker->metrics.ssl_info[0]) {
+            apr_cpystrn(metrics.ssl_info, worker->metrics.ssl_info,
+                        sizeof(metrics.ssl_info));
+        }
+        if (is_ssl && !metrics.ssl_tmp_key[0] && worker->metrics.ssl_tmp_key[0]) {
+            apr_cpystrn(metrics.ssl_tmp_key, worker->metrics.ssl_tmp_key,
+                        sizeof(metrics.ssl_tmp_key));
+        }
+#endif
     }
+}
+
+static void output_results(void)
+{
+    double timetaken;
+
     timetaken = (double) (lasttime - start) / APR_USEC_PER_SEC;
 
     printf("\n\n");
@@ -989,12 +1069,12 @@ static void output_results(int sig)
     printf("Server Hostname:        %s\n", hostname);
     printf("Server Port:            %hu\n", port);
 #ifdef USE_SSL
-    if (is_ssl && ssl_info) {
-        printf("SSL/TLS Protocol:       %s\n", ssl_info);
+    if (is_ssl && metrics.ssl_info[0]) {
+        printf("SSL/TLS Protocol:       %s\n", metrics.ssl_info);
     }
 #if OPENSSL_VERSION_NUMBER >= 0x10002000L
-    if (is_ssl && ssl_tmp_key) {
-        printf("Server Temp Key:        %s\n", ssl_tmp_key);
+    if (is_ssl && metrics.ssl_tmp_key[0]) {
+        printf("Server Temp Key:        %s\n", metrics.ssl_tmp_key);
     }
 #endif
 #ifdef HAVE_TLSEXT
@@ -1008,50 +1088,50 @@ static void output_results(int sig)
     if (nolength)
         printf("Document Length:        Variable\n");
     else
-        printf("Document Length:        %" APR_SIZE_T_FMT " bytes\n", doclen);
+        printf("Document Length:        %" APR_SIZE_T_FMT " bytes\n", metrics.doclen);
     printf("\n");
+    printf("Number of workers:      %d\n", num_workers);
     printf("Concurrency Level:      %d\n", concurrency);
-    printf("Concurrency achieved:   %d\n", concurrent);
+    printf("Concurrency achieved:   %d\n", metrics.concurrent);
     printf("Rampup delay:           %" APR_TIME_T_FMT " [ms]\n", apr_time_as_msec(ramp));
     printf("Time taken for tests:   %.3f seconds\n", timetaken);
-    printf("Complete requests:      %d\n", done);
-    printf("Failed requests:        %d\n", bad);
-    if (bad)
+    printf("Complete requests:      %" APR_INT64_T_FMT "\n", metrics.done);
+    printf("Failed requests:        %" APR_INT64_T_FMT "\n", metrics.bad);
+    if (metrics.bad)
         printf("   (Connect: %d, Receive: %d, Length: %d, Exceptions: %d)\n",
-            err_conn, err_recv, err_length, err_except);
-    if (epipe)
-        printf("Write errors:           %d\n", epipe);
-    if (err_response)
-        printf("Non-2xx responses:      %d\n", err_response);
+            metrics.err_conn, metrics.err_recv, metrics.err_length, metrics.err_except);
+    if (metrics.epipe)
+        printf("Write errors:           %d\n", metrics.epipe);
+    if (metrics.err_response)
+        printf("Non-2xx responses:      %d\n", metrics.err_response);
     if (keepalive)
-        printf("Keep-Alive requests:    %d\n", doneka);
-    printf("Total transferred:      %" APR_INT64_T_FMT " bytes\n", totalread);
+        printf("Keep-Alive requests:    %" APR_INT64_T_FMT "\n", metrics.doneka);
+    printf("Total transferred:      %" APR_INT64_T_FMT " bytes\n", metrics.totalread);
     if (send_body)
-        printf("Total body sent:        %" APR_INT64_T_FMT "\n",
-               totalposted);
-    printf("HTML transferred:       %" APR_INT64_T_FMT " bytes\n", totalbread);
+        printf("Total body sent:        %" APR_INT64_T_FMT "\n", metrics.totalposted);
+    printf("HTML transferred:       %" APR_INT64_T_FMT " bytes\n", metrics.totalbread);
 
     /* avoid divide by zero */
-    if (timetaken && done) {
+    if (timetaken && metrics.done) {
         printf("Requests per second:    %.2f [#/sec] (mean)\n",
-               (double) done / timetaken);
+               (double) metrics.done / timetaken);
         printf("Time per request:       %.3f [ms] (mean)\n",
-               (double) concurrency * timetaken * 1000 / done);
+               (double) concurrency * timetaken * 1000 / metrics.done);
         printf("Time per request:       %.3f [ms] (mean, across all concurrent requests)\n",
-               (double) timetaken * 1000 / done);
+               (double) timetaken * 1000 / metrics.done);
         printf("Transfer rate:          %.2f [Kbytes/sec] received\n",
-               (double) totalread / 1024 / timetaken);
+               (double) metrics.totalread / 1024 / timetaken);
         if (send_body) {
             printf("                        %.2f kb/s sent\n",
-               (double) totalposted / 1024 / timetaken);
+               (double) metrics.totalposted / 1024 / timetaken);
             printf("                        %.2f kb/s total\n",
-               (double) (totalread + totalposted) / 1024 / timetaken);
+               (double) (metrics.totalread + metrics.totalposted) / 1024 / timetaken);
         }
     }
 
-    if (done > 0) {
+    if (metrics.done > 0) {
         /* work out connection times */
-        int i;
+        apr_int64_t i, count = ap_min(metrics.done, requests);
         apr_time_t totalcon = 0, total = 0, totald = 0, totalwait = 0;
         apr_time_t meancon, meantot, meand, meanwait;
         apr_interval_time_t mincon = AB_MAX, mintot = AB_MAX, mind = AB_MAX,
@@ -1060,7 +1140,7 @@ static void output_results(int sig)
         apr_interval_time_t mediancon = 0, mediantot = 0, mediand = 0, medianwait = 0;
         double sdtot = 0, sdcon = 0, sdd = 0, sdwait = 0;
 
-        for (i = 0; i < done; i++) {
+        for (i = 0; i < count; i++) {
             struct data *s = &stats[i];
             mincon = ap_min(mincon, s->ctime);
             mintot = ap_min(mintot, s->time);
@@ -1077,13 +1157,13 @@ static void output_results(int sig)
             totald += s->time - s->ctime;
             totalwait += s->waittime;
         }
-        meancon = totalcon / done;
-        meantot = total / done;
-        meand = totald / done;
-        meanwait = totalwait / done;
+        meancon = totalcon / count;
+        meantot = total / count;
+        meand = totald / count;
+        meanwait = totalwait / count;
 
         /* calculating the sample variance: the sum of the squared deviations, divided by n-1 */
-        for (i = 0; i < done; i++) {
+        for (i = 0; i < count; i++) {
             struct data *s = &stats[i];
             double a;
             a = ((double)s->time - meantot);
@@ -1096,44 +1176,44 @@ static void output_results(int sig)
             sdwait += a * a;
         }
 
-        sdtot = (done > 1) ? sqrt(sdtot / (done - 1)) : 0;
-        sdcon = (done > 1) ? sqrt(sdcon / (done - 1)) : 0;
-        sdd = (done > 1) ? sqrt(sdd / (done - 1)) : 0;
-        sdwait = (done > 1) ? sqrt(sdwait / (done - 1)) : 0;
+        sdtot = (count > 1) ? sqrt(sdtot / (count - 1)) : 0;
+        sdcon = (count > 1) ? sqrt(sdcon / (count - 1)) : 0;
+        sdd = (count > 1) ? sqrt(sdd / (count - 1)) : 0;
+        sdwait = (count > 1) ? sqrt(sdwait / (count - 1)) : 0;
 
         /*
          * XXX: what is better; this hideous cast of the compradre function; or
          * the four warnings during compile ? dirkx just does not know and
          * hates both/
          */
-        qsort(stats, done, sizeof(struct data),
+        qsort(stats, count, sizeof(struct data),
               (int (*) (const void *, const void *)) compradre);
-        if ((done > 1) && (done % 2))
-            mediancon = (stats[done / 2].ctime + stats[done / 2 + 1].ctime) / 2;
+        if ((count > 1) && (count % 2))
+            mediancon = (stats[count / 2].ctime + stats[count / 2 + 1].ctime) / 2;
         else
-            mediancon = stats[done / 2].ctime;
+            mediancon = stats[count / 2].ctime;
 
-        qsort(stats, done, sizeof(struct data),
+        qsort(stats, count, sizeof(struct data),
               (int (*) (const void *, const void *)) compri);
-        if ((done > 1) && (done % 2))
-            mediand = (stats[done / 2].time + stats[done / 2 + 1].time \
-            -stats[done / 2].ctime - stats[done / 2 + 1].ctime) / 2;
+        if ((count > 1) && (count % 2))
+            mediand = (stats[count / 2].time + stats[count / 2 + 1].time \
+                       -stats[count / 2].ctime - stats[count / 2 + 1].ctime) / 2;
         else
-            mediand = stats[done / 2].time - stats[done / 2].ctime;
+            mediand = stats[count / 2].time - stats[count / 2].ctime;
 
-        qsort(stats, done, sizeof(struct data),
+        qsort(stats, count, sizeof(struct data),
               (int (*) (const void *, const void *)) compwait);
-        if ((done > 1) && (done % 2))
-            medianwait = (stats[done / 2].waittime + stats[done / 2 + 1].waittime) / 2;
+        if ((count > 1) && (count % 2))
+            medianwait = (stats[count / 2].waittime + stats[count / 2 + 1].waittime) / 2;
         else
-            medianwait = stats[done / 2].waittime;
+            medianwait = stats[count / 2].waittime;
 
-        qsort(stats, done, sizeof(struct data),
+        qsort(stats, count, sizeof(struct data),
               (int (*) (const void *, const void *)) comprando);
-        if ((done > 1) && (done % 2))
-            mediantot = (stats[done / 2].time + stats[done / 2 + 1].time) / 2;
+        if ((count > 1) && (count % 2))
+            mediantot = (stats[count / 2].time + stats[count / 2 + 1].time) / 2;
         else
-            mediantot = stats[done / 2].time;
+            mediantot = stats[count / 2].time;
 
         printf("\nConnection Times (ms)\n");
         /*
@@ -1201,17 +1281,17 @@ static void output_results(int sig)
 
 
         /* Sorted on total connect times */
-        if (percentile && (done > 1)) {
+        if (percentile && (count > 1)) {
             printf("\nPercentage of the requests served within a certain time (ms)\n");
             for (i = 0; i < sizeof(percs) / sizeof(int); i++) {
                 if (percs[i] <= 0)
                     printf(" 0%%  <0> (never)\n");
                 else if (percs[i] >= 100)
                     printf(" 100%%  %5" APR_TIME_T_FMT " (longest request)\n",
-                           ap_round_ms(stats[done - 1].time));
+                           ap_round_ms(stats[count - 1].time));
                 else
                     printf("  %d%%  %5" APR_TIME_T_FMT "\n", percs[i],
-                           ap_round_ms(stats[(unsigned long)done * percs[i] / 100].time));
+                           ap_round_ms(stats[(unsigned long)count * percs[i] / 100].time));
             }
         }
         if (csvperc) {
@@ -1226,10 +1306,10 @@ static void output_results(int sig)
                 if (i == 0)
                     t = ap_double_ms(stats[0].time);
                 else if (i == 100)
-                    t = ap_double_ms(stats[done - 1].time);
+                    t = ap_double_ms(stats[count - 1].time);
                 else
-                    t = ap_double_ms(stats[(unsigned long) (0.5 + (double)done * i / 100.0)].time);
-                fprintf(out, "%d,%.3f\n", i, t);
+                    t = ap_double_ms(stats[(unsigned long) (0.5 + (double)count * i / 100.0)].time);
+                fprintf(out, "%" APR_INT64_T_FMT ",%.3f\n", i, t);
             }
             fclose(out);
         }
@@ -1241,7 +1321,7 @@ static void output_results(int sig)
                 exit(1);
             }
             fprintf(out, "starttime\tseconds\tctime\tdtime\tttime\twait\n");
-            for (i = 0; i < done; i++) {
+            for (i = 0; i < count; i++) {
                 (void) apr_ctime(tmstring, stats[i].starttime);
                 fprintf(out, "%s\t%" APR_TIME_T_FMT "\t%" APR_TIME_T_FMT
                                "\t%" APR_TIME_T_FMT "\t%" APR_TIME_T_FMT
@@ -1255,10 +1335,6 @@ static void output_results(int sig)
             fclose(out);
         }
     }
-
-    if (sig) {
-        exit(1);
-    }
 }
 
 /* --------------------------------------------------------- */
@@ -1289,13 +1365,16 @@ static void output_html_results(void)
     else
         printf("<tr %s><th colspan=2 %s>Document Length:</th>"
             "<td colspan=2 %s>%" APR_SIZE_T_FMT " bytes</td></tr>\n",
-            trstring, tdstring, tdstring, doclen);
+            trstring, tdstring, tdstring, metrics.doclen);
+    printf("<tr %s><th colspan=2 %s>Number of workers:</th>"
+       "<td colspan=2 %s>%d</td></tr>\n",
+       trstring, tdstring, tdstring, num_workers);
     printf("<tr %s><th colspan=2 %s>Concurrency Level:</th>"
        "<td colspan=2 %s>%d</td></tr>\n",
        trstring, tdstring, tdstring, concurrency);
     printf("<tr %s><th colspan=2 %s>Concurrency achieved:</th>"
        "<td colspan=2 %s>%d</td></tr>\n",
-       trstring, tdstring, tdstring, concurrent);
+       trstring, tdstring, tdstring, metrics.concurrent);
     printf("<tr %s><th colspan=2 %s>Rampup delay:</th>"
        "<td colspan=2 %s>%" APR_TIME_T_FMT " [ms]</td></tr>\n",
        trstring, tdstring, tdstring, apr_time_as_msec(ramp));
@@ -1303,61 +1382,60 @@ static void output_html_results(void)
        "<td colspan=2 %s>%.3f seconds</td></tr>\n",
        trstring, tdstring, tdstring, timetaken);
     printf("<tr %s><th colspan=2 %s>Complete requests:</th>"
-       "<td colspan=2 %s>%d</td></tr>\n",
-       trstring, tdstring, tdstring, done);
+       "<td colspan=2 %s>%" APR_INT64_T_FMT "</td></tr>\n",
+       trstring, tdstring, tdstring, metrics.done);
     printf("<tr %s><th colspan=2 %s>Failed requests:</th>"
-       "<td colspan=2 %s>%d</td></tr>\n",
-       trstring, tdstring, tdstring, bad);
-    if (bad)
+       "<td colspan=2 %s>%" APR_INT64_T_FMT "</td></tr>\n",
+       trstring, tdstring, tdstring, metrics.bad);
+    if (metrics.bad)
         printf("<tr %s><td colspan=4 %s >   (Connect: %d, Length: %d, Exceptions: %d)</td></tr>\n",
-           trstring, tdstring, err_conn, err_length, err_except);
-    if (err_response)
+           trstring, tdstring, metrics.err_conn, metrics.err_length, metrics.err_except);
+    if (metrics.err_response)
         printf("<tr %s><th colspan=2 %s>Non-2xx responses:</th>"
            "<td colspan=2 %s>%d</td></tr>\n",
-           trstring, tdstring, tdstring, err_response);
+           trstring, tdstring, tdstring, metrics.err_response);
     if (keepalive)
         printf("<tr %s><th colspan=2 %s>Keep-Alive requests:</th>"
-           "<td colspan=2 %s>%d</td></tr>\n",
-           trstring, tdstring, tdstring, doneka);
+           "<td colspan=2 %s>%" APR_INT64_T_FMT "</td></tr>\n",
+           trstring, tdstring, tdstring, metrics.doneka);
     printf("<tr %s><th colspan=2 %s>Total transferred:</th>"
        "<td colspan=2 %s>%" APR_INT64_T_FMT " bytes</td></tr>\n",
-       trstring, tdstring, tdstring, totalread);
+       trstring, tdstring, tdstring, metrics.totalread);
     if (send_body)
         printf("<tr %s><th colspan=2 %s>Total body sent:</th>"
            "<td colspan=2 %s>%" APR_INT64_T_FMT "</td></tr>\n",
-           trstring, tdstring,
-           tdstring, totalposted);
+           trstring, tdstring, tdstring, metrics.totalposted);
     printf("<tr %s><th colspan=2 %s>HTML transferred:</th>"
        "<td colspan=2 %s>%" APR_INT64_T_FMT " bytes</td></tr>\n",
-       trstring, tdstring, tdstring, totalbread);
+       trstring, tdstring, tdstring, metrics.totalbread);
 
     /* avoid divide by zero */
     if (timetaken) {
         printf("<tr %s><th colspan=2 %s>Requests per second:</th>"
            "<td colspan=2 %s>%.2f</td></tr>\n",
-           trstring, tdstring, tdstring, (double) done / timetaken);
+           trstring, tdstring, tdstring, (double) metrics.done / timetaken);
         printf("<tr %s><th colspan=2 %s>Transfer rate:</th>"
            "<td colspan=2 %s>%.2f kb/s received</td></tr>\n",
-           trstring, tdstring, tdstring, (double) totalread / 1024 / timetaken);
+           trstring, tdstring, tdstring, (double) metrics.totalread / 1024 / timetaken);
         if (send_body) {
             printf("<tr %s><td colspan=2 %s>&nbsp;</td>"
                "<td colspan=2 %s>%.2f kb/s sent</td></tr>\n",
                trstring, tdstring, tdstring,
-               (double) totalposted / 1024 / timetaken);
+               (double) metrics.totalposted / 1024 / timetaken);
             printf("<tr %s><td colspan=2 %s>&nbsp;</td>"
                "<td colspan=2 %s>%.2f kb/s total</td></tr>\n",
                trstring, tdstring, tdstring,
-               (double) (totalread + totalposted) / 1024 / timetaken);
+               (double) (metrics.totalread + metrics.totalposted) / 1024 / timetaken);
         }
     }
     {
         /* work out connection times */
-        int i;
+        apr_int64_t i, count = ap_min(metrics.done, requests);
         apr_interval_time_t totalcon = 0, total = 0;
         apr_interval_time_t mincon = AB_MAX, mintot = AB_MAX;
         apr_interval_time_t maxcon = 0, maxtot = 0;
 
-        for (i = 0; i < done; i++) {
+        for (i = 0; i < count; i++) {
             struct data *s = &stats[i];
             mincon = ap_min(mincon, s->ctime);
             mintot = ap_min(mintot, s->time);
@@ -1376,7 +1454,7 @@ static void output_html_results(void)
         totalcon = ap_round_ms(totalcon);
         total    = ap_round_ms(total);
 
-        if (done > 0) { /* avoid division by zero (if 0 done) */
+        if (count > 0) { /* avoid division by zero (if 0 count) */
             printf("<tr %s><th %s colspan=4>Connection Times (ms)</th></tr>\n",
                trstring, tdstring);
             printf("<tr %s><th %s>&nbsp;</th> <th %s>min</th>   <th %s>avg</th>   <th %s>max</th></tr>\n",
@@ -1385,18 +1463,18 @@ static void output_html_results(void)
                "<td %s>%5" APR_TIME_T_FMT "</td>"
                "<td %s>%5" APR_TIME_T_FMT "</td>"
                "<td %s>%5" APR_TIME_T_FMT "</td></tr>\n",
-               trstring, tdstring, tdstring, mincon, tdstring, totalcon / done, tdstring, maxcon);
+               trstring, tdstring, tdstring, mincon, tdstring, totalcon / count, tdstring, maxcon);
             printf("<tr %s><th %s>Processing:</th>"
                "<td %s>%5" APR_TIME_T_FMT "</td>"
                "<td %s>%5" APR_TIME_T_FMT "</td>"
                "<td %s>%5" APR_TIME_T_FMT "</td></tr>\n",
                trstring, tdstring, tdstring, mintot - mincon, tdstring,
-               (total / done) - (totalcon / done), tdstring, maxtot - maxcon);
+               (total / count) - (totalcon / count), tdstring, maxtot - maxcon);
             printf("<tr %s><th %s>Total:</th>"
                "<td %s>%5" APR_TIME_T_FMT "</td>"
                "<td %s>%5" APR_TIME_T_FMT "</td>"
                "<td %s>%5" APR_TIME_T_FMT "</td></tr>\n",
-               trstring, tdstring, tdstring, mintot, tdstring, total / done, tdstring, maxtot);
+               trstring, tdstring, tdstring, mintot, tdstring, total / count, tdstring, maxtot);
         }
         printf("</table>\n");
     }
@@ -1406,37 +1484,36 @@ static void output_html_results(void)
 
 /* start asnchronous non-blocking connection */
 
-static void start_connect(struct connection * c)
+static void start_connection(struct connection * c)
 {
+    struct worker *worker = c->worker;
     apr_status_t rv;
 
-    if (!(started < requests)) {
+    if (worker_should_stop(worker)) {
         return;
     }
 
-    c->delay = 0;
-    c->delay_fn = NULL;
+    if (c->ctx) {
+        apr_pool_clear(c->ctx);
+    }
+    else {
+        apr_pool_create(&c->ctx, worker->pool);
+        APR_RING_ELEM_INIT(c, delay_list);
+        worker->metrics.concurrent++;
+    }
 
     c->read = 0;
     c->bread = 0;
+    c->length = 0;
     c->keepalive = 0;
     c->cbx = 0;
     c->gotheader = 0;
     c->rwrite = 0;
     c->keptalive = 0;
-    if (c->ctx) {
-        apr_pool_clear(c->ctx);
-    }
-    else {
-        apr_pool_create(&c->ctx, cntxt);
-        concurrent++;
-    }
-
-    APR_RING_ELEM_INIT((c), delay_list);
 
-    if ((rv = apr_socket_create(&c->aprsock, destsa->family,
-                SOCK_STREAM, 0, c->ctx)) != APR_SUCCESS) {
-    apr_err("socket", rv);
+    if ((rv = apr_socket_create(&c->aprsock, worker->destsa->family,
+                                SOCK_STREAM, 0, c->ctx)) != APR_SUCCESS) {
+        apr_err("socket", rv);
     }
 
     if (myhost) {
@@ -1450,8 +1527,7 @@ static void start_connect(struct connection * c)
     c->pollfd.reqevents = 0;
     c->pollfd.client_data = c;
 
-    if ((rv = apr_socket_opt_set(c->aprsock, APR_SO_NONBLOCK, 1))
-         != APR_SUCCESS) {
+    if ((rv = apr_socket_opt_set(c->aprsock, APR_SO_NONBLOCK, 1))) {
         apr_err("socket nonblock", rv);
     }
 
@@ -1468,6 +1544,7 @@ static void start_connect(struct connection * c)
         }
     }
 
+    apr_socket_timeout_set(c->aprsock, 0);
     c->start = lasttime = apr_time_now();
 #ifdef USE_SSL
     if (is_ssl) {
@@ -1502,95 +1579,192 @@ static void start_connect(struct connection * c)
         c->ssl = NULL;
     }
 #endif
-    if ((rv = apr_socket_connect(c->aprsock, destsa)) != APR_SUCCESS) {
+    if ((rv = apr_socket_connect(c->aprsock, worker->destsa))) {
         if (APR_STATUS_IS_EINPROGRESS(rv)) {
             set_conn_state(c, STATE_CONNECTING, APR_POLLOUT);
-            c->rwrite = 0;
-            return;
         }
         else {
-            set_conn_state(c, STATE_UNCONNECTED, 0);
-            apr_socket_close(c->aprsock);
-            if (good == 0 && destsa->next) {
-                destsa = destsa->next;
-                err_conn = 0;
-            }
-            else if (bad++ > 10) {
-                fprintf(stderr,
-                   "\nTest aborted after 10 failures\n\n");
-                apr_err("apr_socket_connect()", rv);
-            }
-            else {
-                err_conn++;
-            }
-
-            start_connect(c);
-            return;
+            retry_connection(c, rv);
         }
+        return;
     }
 
     /* connected first time */
 #ifdef USE_SSL
     if (c->ssl) {
         ssl_proceed_handshake(c);
-    } else
+    }
+    else
 #endif
-    {
-        write_request(c);
+    write_request(c);
+}
+
+/* --------------------------------------------------------- */
+
+/* shutdown the transport layer */
+
+static void shutdown_connection(struct connection *c)
+{
+    set_conn_state(c, STATE_UNCONNECTED, 0);
+#ifdef USE_SSL
+    if (c->ssl) {
+        SSL_shutdown(c->ssl);
+        SSL_free(c->ssl);
+        c->ssl = NULL;
     }
+#endif
+    apr_socket_close(c->aprsock);
 }
 
 /* --------------------------------------------------------- */
 
-/* close down connection and save stats */
+/* retry a connect()ion failure on the next address (if any) */
 
-static void close_connection(struct connection * c)
+static void retry_connection(struct connection *c, apr_status_t status)
 {
+    struct worker *worker = c->worker;
+
+    if (worker->metrics.good == 0 && worker->destsa->next) {
+        worker->destsa = worker->destsa->next;
+        shutdown_connection(c);
+        start_connection(c);
+    }
+    else {
+        worker->metrics.err_conn++;
+        if (worker->metrics.good == 0) {
+            if (worker->metrics.err_conn > 10) {
+                fprintf(stderr,
+                        "\nTest aborted after 10 failures\n\n");
+                apr_err("apr_socket_connect()", status);
+            }
+            worker->destsa = destsa;
+        }
+        abort_connection(c);
+    }
+}
+
+/* --------------------------------------------------------- */
+
+/* reuse or renew the connection, saving stats */
+
+static void cleanup_connection(struct connection *c, int reuse)
+{
+    struct worker *worker = c->worker;
+    int good = (c->gotheader && c->bread >= c->length);
+
+    /* close before measuring, to account for shutdown time */
+    if (!reuse || !good) {
+        shutdown_connection(c);
+        reuse = 0;
+    }
+
     if (c->read == 0 && c->keptalive) {
         /*
-         * server has legitimately shut down an idle keep alive request
-         * as per RFC7230 6.3.1.
+         * server has legitimately shut down an idle keep alive connection
+         * as per RFC7230 6.3.1, revert previous accounting (not an error).
          */
-        if (good)
-            good--;     /* connection never happened */
+        worker->metrics.doneka--;
     }
     else {
-        if (good == 1) {
-            /* first time here */
-            doclen = c->bread;
-        }
-        else if ((c->bread != doclen) && !nolength) {
-            bad++;
-            err_length++;
-        }
         /* save out time */
-        if (done < requests) {
-            struct data *s = &stats[done++];
-            c->done      = lasttime = apr_time_now();
+        if (tlimit || worker->metrics.done < worker->requests) {
+            apr_time_t tnow = lasttime = c->end = apr_time_now();
+            struct data *s = &worker->stats[worker->metrics.done++ % worker->requests];
+
             s->starttime = c->start;
+            s->time      = ap_max(0, c->end - c->start);
             s->ctime     = ap_max(0, c->connect - c->start);
-            s->time      = ap_max(0, c->done - c->start);
             s->waittime  = ap_max(0, c->beginread - c->endwrite);
-            if (heartbeatres && !(done % heartbeatres)) {
-                fprintf(stderr, "Completed %d requests\n", done);
-                fflush(stderr);
+
+            if (heartbeatres) {
+                static apr_int64_t reqs_count64;
+                static apr_uint32_t reqs_count32;
+                int sync = 0, flush = 0;
+                apr_uint32_t n;
+
+#if APR_HAS_THREADS
+                /* use 32bit atomics only to help 32bit systems and support
+                 * earlier APR versions (which lack 64bit atomics).
+                 */
+                if (num_workers > 1)
+                    n = apr_atomic_inc32(&reqs_count32) + 1;
+                else
+#endif
+                    n = ++reqs_count32;
+
+                if (!tlimit && !(n % heartbeatres)) {
+                    sync = 1;
+                }
+                else if (tlimit && tnow >= logtime) {
+                    sync = (logtime != 0);
+                    logtime = tnow + hbperiod;
+                }
+
+                if (sync) {
+#if APR_HAS_THREADS
+                    if (num_workers > 1) {
+                        apr_uint32_t m;
+                        do {
+                            m = apr_atomic_read32(&reqs_count32);
+                        } while (m && apr_atomic_cas32(&reqs_count32, 0, m) != m);
+                        if (m) {
+                            /* races should be quite rare here now */
+                            reqs_count64 += m;
+                            flush = (m >= n);
+                        }
+                    }
+                    else
+#endif
+                    {
+                        reqs_count64 += reqs_count32;
+                        reqs_count32 = 0;
+                        flush = 1;
+                    }
+                }
+                if (flush) {
+                    fprintf(stderr,
+                            "Completed %" APR_INT64_T_FMT " requests\n",
+                            reqs_count64);
+                    fflush(stderr);
+                }
+            }
+        }
+
+        /* update worker's metrics */
+        if (good) {
+            if (worker->metrics.good == 0) {
+                /* first time saves the doclen */
+                worker->metrics.doclen = c->bread;
+            }
+            worker->metrics.good++;
+        }
+        else {
+            if (!nolength && c->bread != worker->metrics.doclen) {
+                worker->metrics.err_length++;
             }
+            worker->metrics.bad++;
         }
     }
 
-    set_conn_state(c, STATE_UNCONNECTED, 0);
-#ifdef USE_SSL
-    if (c->ssl) {
-        SSL_shutdown(c->ssl);
-        SSL_free(c->ssl);
-        c->ssl = NULL;
+    if (!reuse) {
+        start_connection(c); /* nop if worker_should_stop() */
     }
-#endif
-    apr_socket_close(c->aprsock);
+    else if (!worker_should_stop(worker)) {
+        c->read = 0;
+        c->bread = 0;
+        c->length = 0;
+        c->keepalive = 0;
+        c->cbx = 0;
+        c->gotheader = 0;
+        c->rwrite = 0;
 
-    /* connect again */
-    start_connect(c);
-    return;
+        c->keptalive++;
+        worker->metrics.doneka++;
+        write_request(c);
+    }
+    else {
+        shutdown_connection(c);
+    }
 }
 
 /* --------------------------------------------------------- */
@@ -1599,23 +1773,33 @@ static void close_connection(struct connection * c)
 
 static void read_connection(struct connection * c)
 {
+    struct worker *worker = c->worker;
     apr_size_t r;
     apr_status_t status;
     char *part;
     char respcode[4];       /* 3 digits and null */
-    int i;
 
-    r = sizeof(buffer);
 read_more:
+    r = sizeof(worker->buffer);
+    if (c->length && r > c->length - c->bread) {
+        r = c->length - c->bread;
+    }
 #ifdef USE_SSL
     if (c->ssl) {
-        status = SSL_read(c->ssl, buffer, r);
+        status = SSL_read(c->ssl, worker->buffer, r);
         if (status <= 0) {
             int scode = SSL_get_error(c->ssl, status);
 
-            if (scode == SSL_ERROR_ZERO_RETURN) {
-                /* connection closed cleanly: */
-                good++;
+            if (scode == SSL_ERROR_WANT_READ) {
+                set_conn_state(c, STATE_READ, APR_POLLIN);
+            }
+            else if (scode == SSL_ERROR_WANT_WRITE) {
+                set_conn_state(c, STATE_READ, APR_POLLOUT);
+            }
+            else if (scode == SSL_ERROR_ZERO_RETURN) {
+                /* connection closed cleanly:
+                 * let the length check catch any response errors
+                 */
                 close_connection(c);
             }
             else if (scode == SSL_ERROR_SYSCALL
@@ -1625,32 +1809,13 @@ read_more:
                  * some data has already been read; this commonly happens, so
                  * let the length check catch any response errors
                  */
-                good++;
                 close_connection(c);
             }
-            else if (scode == SSL_ERROR_SYSCALL 
-                     && c->read == 0
-                     && destsa->next
-                     && c->state == STATE_CONNECTING
-                     && good == 0) {
-                return;
-            }
-            else if (scode == SSL_ERROR_WANT_READ) {
-
-                set_conn_state(c, STATE_READ, APR_POLLIN);
-
-            }
-            else if (scode == SSL_ERROR_WANT_WRITE) {
-
-                set_conn_state(c, STATE_READ, APR_POLLOUT);
-
-            }
             else {
                 /* some fatal error: */
-                c->read = 0;
                 BIO_printf(bio_err, "SSL read failed (%d) - closing connection\n", scode);
                 ERR_print_errors(bio_err);
-                close_connection(c);
+                abort_connection(c);
             }
             return;
         }
@@ -1659,43 +1824,37 @@ read_more:
     else
 #endif
     {
-        status = apr_socket_recv(c->aprsock, buffer, &r);
+        status = apr_socket_recv(c->aprsock, worker->buffer, &r);
         if (APR_STATUS_IS_EAGAIN(status))
             return;
         else if (r == 0 && APR_STATUS_IS_EOF(status)) {
-            good++;
             close_connection(c);
             return;
         }
         /* catch legitimate fatal apr_socket_recv errors */
         else if (status != APR_SUCCESS) {
+            worker->metrics.err_recv++;
             if (recverrok) {
-                err_recv++;
-                bad++;
-                close_connection(c);
                 if (verbosity >= 1) {
                     char buf[120];
-                    fprintf(stderr,"%s: %s (%d)\n", "apr_socket_recv", apr_strerror(status, buf, sizeof buf), status);
+                    fprintf(stderr,"%s: %s (%d)\n", "apr_socket_recv",
+                            apr_strerror(status, buf, sizeof buf), status);
                 }
-                return;
-            } else if (destsa->next && c->state == STATE_CONNECTING
-                       && c->read == 0 && good == 0) {
-                return;
             }
             else {
-                err_recv++;
                 apr_err("apr_socket_recv", status);
             }
+            abort_connection(c);
+            return;
         }
     }
 
-    totalread += r;
+    worker->metrics.totalread += r;
     if (c->read == 0) {
         c->beginread = apr_time_now();
     }
     c->read += r;
 
-
     if (!c->gotheader) {
         char *s;
         int l = 4;
@@ -1704,7 +1863,7 @@ read_more:
 #ifdef NOT_ASCII
         apr_size_t inbytes_left = space, outbytes_left = space;
 
-        status = apr_xlate_conv_buffer(from_ascii, buffer, &inbytes_left,
+        status = apr_xlate_conv_buffer(from_ascii, worker->buffer, &inbytes_left,
                            c->cbuff + c->cbx, &outbytes_left);
         if (status || inbytes_left || outbytes_left) {
             fprintf(stderr, "only simple translation is supported (%d/%" APR_SIZE_T_FMT
@@ -1712,7 +1871,7 @@ read_more:
             exit(1);
         }
 #else
-        memcpy(c->cbuff + c->cbx, buffer, space);
+        memcpy(c->cbuff + c->cbx, worker->buffer, space);
 #endif              /* NOT_ASCII */
         c->cbx += tocopy;
         space -= tocopy;
@@ -1732,38 +1891,26 @@ read_more:
 
         if (!s) {
             /* read rest next time */
-            if (space) {
-                return;
-            }
-            else {
-            /* header is in invalid or too big - close connection */
-                set_conn_state(c, STATE_UNCONNECTED, 0);
-                apr_socket_close(c->aprsock);
-                err_response++;
-                if (bad++ > 10) {
-                    err("\nTest aborted after 10 failures\n\n");
+            if (!space) {
+                /* header is in invalid or too big - close connection */
+                if (worker->metrics.err_response++ > 10) {
+                    fprintf(stderr,
+                            "\nTest aborted after 10 failures\n\n");
+                    err("Response header too long\n");
                 }
-                start_connect(c);
+                abort_connection(c);
             }
+            return;
         }
         else {
             /* have full header */
-            if (!good) {
-                /*
-                 * this is first time, extract some interesting info
-                 */
-                char *p, *q;
-                size_t len = 0;
-                p = xstrcasestr(c->cbuff, "Server:");
-                q = servername;
-                if (p) {
-                    p += 8;
-                    /* -1 to not overwrite last '\0' byte */
-                    while (*p > 32 && len++ < sizeof(servername) - 1)
-                        *q++ = *p++;
-                }
-                *q = 0;
-            }
+            s[l / 2] = '\0';     /* terminate at end of header */
+            c->gotheader = 1;
+
+            /* account for the body we may have read already */
+            c->bread += c->cbx - (s + l - c->cbuff) + r - tocopy;
+            worker->metrics.totalbread += c->bread;
+
             /*
              * XXX: this parsing isn't even remotely HTTP compliant... but in
              * the interest of speed it doesn't totally have to be, it just
@@ -1782,19 +1929,17 @@ read_more:
             }
 
             if (respcode[0] != '2') {
-                err_response++;
+                worker->metrics.err_response++;
                 if (verbosity >= 2)
                     printf("WARNING: Response code not 2xx (%s)\n", respcode);
             }
             else if (verbosity >= 3) {
                 printf("LOG: Response code = %s\n", respcode);
             }
-            c->gotheader = 1;
-            *s = 0;     /* terminate at end of header */
-            if (keepalive && xstrcasestr(c->cbuff, "Keep-Alive")) {
-                char *cl;
-                c->keepalive = 1;
-                cl = xstrcasestr(c->cbuff, "Content-Length:");
+
+            c->keepalive = (keepalive && xstrcasestr(c->cbuff, "Keep-Alive"));
+            if (c->keepalive) {
+                const char *cl = xstrcasestr(c->cbuff, "Content-Length:");
                 if (cl && method != HEAD) {
                     /* response to HEAD doesn't have entity body */
                     c->length = atoi(cl + 16);
@@ -1803,90 +1948,131 @@ read_more:
                     c->length = 0;
                 }
             }
-            c->bread += c->cbx - (s + l - c->cbuff) + r - tocopy;
-            totalbread += c->bread;
 
             /* We have received the header, so we know this destination socket
-             * address is working, so initialize all remaining requests. */
-            if (!requests_initialized) {
+             * address is working, so schedule all remaining connections. */
+            if (!worker->succeeded_once) {
+                int i;
                 apr_time_t now = apr_time_now();
-                for (i = 1; i < concurrency; i++) {
-                    con[i].socknum = i;
-                    con[i].delay = now + (i * ramp);
-                    con[i].delay_fn = &start_connect;
+                for (i = 1; i < worker->concurrency; i++) {
+                    worker->conns[i].delay = now + (i * ramp);
+                    APR_RING_INSERT_TAIL(&worker->delayed_ring, &worker->conns[i],
+                                         connection, delay_list);
+                }
+                worker->succeeded_once = 1;
+
+                /*
+                 * first time, extract some interesting info
+                 */
+                if (worker->slot == 0) {
+                    char *p, *q;
+                    size_t len = 0;
+                    p = xstrcasestr(c->cbuff, "Server:");
+                    q = servername;
+                    if (p) {
+                        p += 8;
+                        /* -1 to not overwrite last '\0' byte */
+                        while (*p > 32 && len++ < sizeof(servername) - 1)
+                            *q++ = *p++;
+                    }
+                    *q = 0;
+                }
 
-                    APR_RING_INSERT_TAIL(&delay_head, &con[i], connection, delay_list);
+#if APR_HAS_THREADS
+                if (num_workers > 1 && worker->slot == 0) {
+                    apr_status_t rv;
+                    apr_thread_mutex_lock(workers_mutex);
+                    rv = apr_thread_cond_signal(workers_can_start);
+                    if (rv != APR_SUCCESS) {
+                        apr_err("apr_thread_cond_wait()", rv);
+                    }
+                    workers_can_start = NULL; /* one shot */
+                    apr_thread_mutex_unlock(workers_mutex);
                 }
-                requests_initialized = 1;
+#endif
             }
         }
     }
     else {
         /* outside header, everything we have read is entity body */
         c->bread += r;
-        totalbread += r;
+        worker->metrics.totalbread += r;
     }
-    if (r == sizeof(buffer) && c->bread < c->length) {
-        /* read was full, try more immediately (nonblocking already) */
+
+    /* read incomplete or connection terminated by close, continue
+     * reading until we get everything or EOF/EAGAIN.
+     */
+    if (c->bread < c->length || (!c->length && method != HEAD)) {
         goto read_more;
     }
 
-    /* are we done? */
-    if (started >= requests && (c->bread >= c->length)) {
+    /* read complete, reuse/close depending on keepalive */
+    if (c->keepalive) {
+        reuse_connection(c);
+    }
+    else {
         close_connection(c);
     }
+}
 
-    /* are we keepalive? if so, reuse existing connection */
-    else if (c->keepalive && (c->bread >= c->length)) {
-        /* finished a keep-alive connection */
-        good++;
-        /* save out time */
-        if (good == 1) {
-            /* first time here */
-            doclen = c->bread;
-        }
-        else if ((c->bread != doclen) && !nolength) {
-            bad++;
-            err_length++;
-        }
-        if (done < requests) {
-            struct data *s = &stats[done++];
-            doneka++;
-            c->done      = apr_time_now();
-            s->starttime = c->start;
-            s->ctime     = ap_max(0, c->connect - c->start);
-            s->time      = ap_max(0, c->done - c->start);
-            s->waittime  = ap_max(0, c->beginread - c->endwrite);
-            if (heartbeatres && !(done % heartbeatres)) {
-                fprintf(stderr, "Completed %d requests\n", done);
-                fflush(stderr);
-            }
-        }
-        c->keepalive = 0;
-        c->length = 0;
-        c->gotheader = 0;
-        c->cbx = 0;
-        c->read = c->bread = 0;
-        /* zero connect time with keep-alive */
-        c->start = c->connect = lasttime = apr_time_now();
+/* --------------------------------------------------------- */
 
-        c->keptalive++;
+/* run the tests */
 
-        write_request(c);
+static void start_worker(struct worker *worker);
+#if APR_HAS_THREADS
+static void join_worker(struct worker *worker);
+#endif /* APR_HAS_THREADS */
+
+#ifdef SIGINT
+static void workers_may_exit(int sig);
+#endif /* SIGINT */
+
+#define USE_SIGMASK (APR_HAS_THREADS \
+                     && (APR_HAVE_PTHREAD_H \
+                         || defined(SIGPROCMASK_SETS_THREAD_MASK)))
+
+static void init_signals(void)
+{
+#ifdef SIGINT
+#if USE_SIGMASK
+    if (num_workers > 1) {
+        apr_status_t rv;
+        rv = apr_setup_signal_thread();
+        if (rv != APR_SUCCESS) {
+            apr_err("apr_setup_signal_thread()", rv);
+        }
     }
+#endif
+    /* Stop early on SIGINT */
+    apr_signal(SIGINT, workers_may_exit);
+#endif /* SIGINT */
 }
 
-/* --------------------------------------------------------- */
-
-/* run the tests */
+#if APR_HAS_THREADS
+static void block_signals(int block)
+{
+#ifdef SIGINT
+#if USE_SIGMASK
+    if (num_workers > 1) {
+        sigset_t set;
+        sigemptyset(&set);
+        sigaddset(&set, SIGINT);
+#if defined(SIGPROCMASK_SETS_THREAD_MASK)
+        sigprocmask(block ? SIG_BLOCK : SIG_UNBLOCK, &set, NULL);
+#else
+        pthread_sigmask(block ? SIG_BLOCK : SIG_UNBLOCK, &set, NULL);
+#endif
+    }
+#endif /* USE_SIGMASK */
+#endif /* SIGINT */
+}
+#endif /* APR_HAS_THREADS */
 
 static void test(void)
 {
-    apr_time_t stoptime;
-    apr_int16_t rtnev;
     apr_status_t rv;
-    int i;
-    apr_status_t status;
+    int i, j;
     int snprintf_res = 0;
 #ifdef NOT_ASCII
     apr_size_t inbytes_left, outbytes_left;
@@ -1910,19 +2096,6 @@ static void test(void)
         fflush(stdout);
     }
 
-    con = xcalloc(concurrency, sizeof(struct connection));
-
-    /*
-     * XXX: a way to calculate the stats without requiring O(requests) memory
-     * XXX: would be nice.
-     */
-    stats = xcalloc(requests, sizeof(struct data));
-
-    if ((status = apr_pollset_create(&readbits, concurrency, cntxt,
-                                     APR_POLLSET_NOCOPY)) != APR_SUCCESS) {
-        apr_err("apr_pollset_create failed", status);
-    }
-
     /* add default headers if necessary */
     if (!opt_host) {
         /* Host: header not overridden, add default value to hdrs */
@@ -2000,7 +2173,7 @@ static void test(void)
      * Combine headers and (optional) post file into one continuous buffer
      */
     if (send_body) {
-        char *buff = xmalloc(postlen + reqlen + 1);
+        char *buff = apr_palloc(cntxt, postlen + reqlen + 1);
         strcpy(buff, request);
         memcpy(buff + reqlen, postdata, postlen);
         request = buff;
@@ -2008,19 +2181,19 @@ static void test(void)
 
 #ifdef NOT_ASCII
     inbytes_left = outbytes_left = reqlen;
-    status = apr_xlate_conv_buffer(to_ascii, request, &inbytes_left,
+    rv = apr_xlate_conv_buffer(to_ascii, request, &inbytes_left,
                    request, &outbytes_left);
-    if (status || inbytes_left || outbytes_left) {
+    if (rv || inbytes_left || outbytes_left) {
         fprintf(stderr, "only simple translation is supported (%d/%"
                         APR_SIZE_T_FMT "/%" APR_SIZE_T_FMT ")\n",
-                        status, inbytes_left, outbytes_left);
+                        rv, inbytes_left, outbytes_left);
         exit(1);
     }
 #endif              /* NOT_ASCII */
 
     if (myhost) {
         /* This only needs to be done once */
-        if ((rv = apr_sockaddr_info_get(&mysa, myhost, APR_UNSPEC, 0, 0, cntxt)) != APR_SUCCESS) {
+        if ((rv = apr_sockaddr_info_get(&mysa, myhost, APR_UNSPEC, 0, 0, cntxt))) {
             char buf[120];
             apr_snprintf(buf, sizeof(buf),
                          "apr_sockaddr_info_get() for %s", myhost);
@@ -2031,27 +2204,127 @@ static void test(void)
     /* This too */
     if ((rv = apr_sockaddr_info_get(&destsa, connecthost,
                                     myhost ? mysa->family : APR_UNSPEC,
-                                    connectport, 0, cntxt))
-       != APR_SUCCESS) {
+                                    connectport, 0, cntxt))) {
         char buf[120];
         apr_snprintf(buf, sizeof(buf),
                  "apr_sockaddr_info_get() for %s", connecthost);
         apr_err(buf, rv);
     }
 
+    /*
+     * XXX: a way to calculate the stats without requiring O(requests) memory
+     * XXX: would be nice.
+     */
+    stats = apr_pcalloc(cntxt, requests * sizeof(struct data));
+
+    conns = apr_pcalloc(cntxt, concurrency * sizeof(struct connection));
+
+    workers = apr_pcalloc(cntxt, num_workers * sizeof(struct worker));
+    for (i = 0; i < num_workers; i++) {
+        struct worker *worker = &workers[i];
+
+        worker->slot = i;
+        worker->pool = cntxt;
+        worker->destsa = destsa;
+        worker->requests = requests / num_workers;
+        worker->concurrency = concurrency / num_workers;
+        worker->stats = &stats[i * worker->requests];
+        worker->conns = &conns[i * worker->concurrency];
+        for (j = 0; j < worker->concurrency; j++) {
+            worker->conns[j].worker = worker;
+        }
+        APR_RING_INIT(&worker->delayed_ring, connection, delay_list);
+
+#ifdef APR_POLLSET_WAKEABLE
+        rv = apr_pollset_create(&worker->pollset, worker->concurrency,
+                                cntxt, APR_POLLSET_NOCOPY | APR_POLLSET_WAKEABLE);
+        if (rv == APR_SUCCESS)
+            pollset_wakeable = 1;
+        else if (APR_STATUS_IS_ENOTIMPL(rv))
+#endif
+            rv = apr_pollset_create(&worker->pollset, worker->concurrency,
+                                    cntxt, APR_POLLSET_NOCOPY);
+        if (rv != APR_SUCCESS) {
+            apr_err("apr_pollset_create failed", rv);
+        }
+    }
+
+#if APR_HAS_THREADS
+    if (num_workers > 1) {
+        rv = apr_thread_mutex_create(&workers_mutex, APR_THREAD_MUTEX_DEFAULT,
+                                     cntxt);
+        if (rv != APR_SUCCESS) {
+            apr_err("apr_thread_mutex_create()", rv);
+        }
+        rv = apr_thread_cond_create(&workers_can_start, cntxt);
+        if (rv != APR_SUCCESS) {
+            apr_err("apr_thread_cond_create()", rv);
+        }
+    }
+#endif
+
+    init_signals();
+
     /* ok - lets start */
     start = lasttime = apr_time_now();
     stoptime = tlimit ? (start + apr_time_from_sec(tlimit)) : AB_MAX;
 
-#ifdef SIGINT
-    /* Output the results if the user terminates the run early. */
-    apr_signal(SIGINT, output_results);
+    /* let the first worker determine if the connectivity is ok before
+     * starting the others (if any).
+     */
+    start_worker(&workers[0]);
+
+#if APR_HAS_THREADS
+    if (num_workers > 1) {
+        /* wait for the signal of the first worker to continue */
+        apr_thread_mutex_lock(workers_mutex);
+        if (workers_can_start) { /* might have been signaled & NULL-ed already */
+            rv = apr_thread_cond_wait(workers_can_start, workers_mutex);
+            if (rv != APR_SUCCESS) {
+                apr_err("apr_thread_cond_wait()", rv);
+            }
+        }
+        apr_thread_mutex_unlock(workers_mutex);
+
+        /* start the others? */
+        if (workers[0].succeeded_once) {
+            for (i = 1; i < num_workers; i++) {
+                start_worker(&workers[i]);
+            }
+        }
+        /* wait what's started only, join_worker() knows */
+        for (i = 0; i < num_workers; i++) {
+            join_worker(&workers[i]);
+        }
+    }
 #endif
 
+    consolidate_metrics();
+
+    if (heartbeatres)
+        fprintf(stderr, "Finished %" APR_INT64_T_FMT " requests%s\n",
+                metrics.done, stoptime ? "" : " (interrupted)");
+    else if (!stoptime)
+        printf("..interrupted\n");
+    else
+        printf("..done\n");
+
+    if (use_html)
+        output_html_results();
+    else
+        output_results();
+}
+
+static void worker_test(struct worker *worker)
+{
+    apr_status_t rv;
+    struct connection *c;
+    apr_int16_t rtnev;
+    int i;
+
     /* initialise first connection to determine destination socket address
      * which should be used for next connections. */
-    con[0].socknum = 0;
-    start_connect(&con[0]);
+    start_connection(&worker->conns[0]);
 
     do {
         apr_int32_t n;
@@ -2059,37 +2332,33 @@ static void test(void)
         apr_interval_time_t t = aprtimeout;
         apr_time_t now = apr_time_now();
 
-        while (!APR_RING_EMPTY(&delay_head, connection, delay_list)) {
-
-            struct connection *c = APR_RING_FIRST(&delay_head);
-
+        while (!APR_RING_EMPTY(&worker->delayed_ring, connection, delay_list)) {
+            c = APR_RING_FIRST(&worker->delayed_ring);
             if (c->delay <= now) {
                 APR_RING_REMOVE(c, delay_list);
+                APR_RING_ELEM_INIT(c, delay_list);
                 c->delay = 0;
-                c->delay_fn(c);
+                start_connection(c);
             }
             else {
                 t = c->delay - now;
                 break;
             }
-        };
-
-        n = concurrent;
-        do {
-            status = apr_pollset_poll(readbits, t, &n, &pollresults);
-        } while (APR_STATUS_IS_EINTR(status));
-
-        if (APR_STATUS_IS_TIMEUP(status) &&
-                !APR_RING_EMPTY(&delay_head, connection, delay_list)) {
-            continue;
         }
-        else if (status != APR_SUCCESS) {
-            apr_err("apr_pollset_poll", status);
+
+        n = worker->metrics.concurrent;
+        rv = apr_pollset_poll(worker->pollset, t, &n, &pollresults);
+        if (rv != APR_SUCCESS) {
+            if (APR_STATUS_IS_EINTR(rv)
+                || (APR_STATUS_IS_TIMEUP(rv) &&
+                    !APR_RING_EMPTY(&worker->delayed_ring, connection,
+                                    delay_list))) {
+                continue;
+            }
+            apr_err("apr_pollset_poll", rv);
         }
 
         for (i = 0, pollfd = pollresults; i < n; i++, pollfd++) {
-            struct connection *c;
-
             c = pollfd->client_data;
 
             /*
@@ -2119,7 +2388,7 @@ static void test(void)
              * connection is done and we loop here endlessly calling
              * apr_poll().
              */
-            if ((rtnev & APR_POLLIN) || (rtnev & APR_POLLPRI) || (rtnev & APR_POLLHUP)) {
+            if (rtnev & (APR_POLLIN | APR_POLLHUP | APR_POLLPRI)) {
 
                 switch (c->state) {
 #ifdef USE_SSL
@@ -2135,51 +2404,23 @@ static void test(void)
                     break;
                 }
 
-            }
-            if ((rtnev & APR_POLLERR) || (rtnev & APR_POLLNVAL)) {
-                if (destsa->next && c->state == STATE_CONNECTING && good == 0) {
-                    destsa = destsa->next;
-                    start_connect(c);
-                }
-                else {
-                    bad++;
-                    err_except++;
-                    /* avoid apr_poll/EINPROGRESS loop on HP-UX, let recv discover ECONNREFUSED */
-                    if (c->state == STATE_CONNECTING) {
-                        read_connection(c);
-                    }
-                    else {
-                        start_connect(c);
-                    }
-                }
                 continue;
             }
+
             if (rtnev & APR_POLLOUT) {
                 if (c->state == STATE_CONNECTING) {
                     /* call connect() again to detect errors */
-                    rv = apr_socket_connect(c->aprsock, destsa);
+                    rv = apr_socket_connect(c->aprsock, worker->destsa);
                     if (rv != APR_SUCCESS) {
-                        set_conn_state(c, STATE_UNCONNECTED, 0);
-                        apr_socket_close(c->aprsock);
-                        err_conn++;
-                        if (bad++ > 10) {
-                            fprintf(stderr,
-                                    "\nTest aborted after 10 failures\n\n");
-                            apr_err("apr_socket_connect()", rv);
-                        }
-                        start_connect(c);
+                        retry_connection(c, rv);
                         continue;
                     }
-                    else {
-
 #ifdef USE_SSL
-                        if (c->ssl)
-                            ssl_proceed_handshake(c);
-                        else
+                    if (c->ssl)
+                        ssl_proceed_handshake(c);
+                    else
 #endif
-                        write_request(c);
-                    }
-
+                    write_request(c);
                 }
                 else {
 
@@ -2198,20 +2439,104 @@ static void test(void)
                     }
 
                 }
+
+                continue;
+            }
+
+            if (rtnev & (APR_POLLERR | APR_POLLNVAL)) {
+                if (c->state == STATE_CONNECTING) {
+                    retry_connection(c, APR_ENOPOLL);
+                }
+                else {
+                    worker->metrics.err_except++;
+                    abort_connection(c);
+                }
+                continue;
             }
         }
-    } while (lasttime < stoptime && done < requests);
+    } while (!worker_should_exit(worker));
+}
 
-    if (heartbeatres)
-        fprintf(stderr, "Finished %d requests\n", done);
-    else
-        printf("..done\n");
+#if APR_HAS_THREADS
+static void *APR_THREAD_FUNC worker_thread(apr_thread_t *thd, void *arg)
+{
+    struct worker *worker = arg;
+
+    worker->pool = apr_thread_pool_get(thd);
+    worker_test(worker);
+
+    /* unblock the main thread if the first worker could never start successfully */
+    if (num_workers > 1 && worker->slot == 0 && !worker->succeeded_once) {
+        apr_status_t rv;
+        apr_thread_mutex_lock(workers_mutex);
+        rv = apr_thread_cond_signal(workers_can_start);
+        if (rv != APR_SUCCESS) {
+            apr_err("apr_thread_cond_wait()", rv);
+        }
+        workers_can_start = NULL; /* one shot */
+        apr_thread_mutex_unlock(workers_mutex);
+    }
 
-    if (use_html)
-        output_html_results();
+    apr_thread_exit(thd, APR_SUCCESS);
+    return NULL;
+}
+#endif
+
+static void start_worker(struct worker *worker)
+{
+#if APR_HAS_THREADS
+    if (num_workers > 1) {
+        apr_status_t rv;
+        block_signals(1);
+        rv = apr_thread_create(&worker->thd, NULL, worker_thread, worker, cntxt);
+        block_signals(0);
+        if (rv != APR_SUCCESS) {
+            apr_err("apr_thread_create()", rv);
+        }
+    }
     else
-        output_results(0);
+#endif /* APR_HAS_THREADS */
+    worker_test(worker);
+}
+
+#if APR_HAS_THREADS
+static void join_worker(struct worker *worker)
+{
+    apr_thread_t *thd = worker->thd;
+    if (thd) {
+        apr_status_t rv, thread_rv;
+        rv = apr_thread_join(&thread_rv, thd);
+        if (rv != APR_SUCCESS) {
+            apr_err("apr_thread_join()", rv);
+        }
+        worker->thd = NULL;
+    }
+}
+#endif /* APR_HAS_THREADS */
+
+#ifdef SIGINT
+static void workers_may_exit(int sig)
+{
+    lasttime = apr_time_now();      /* record final time if interrupted */
+    if (num_workers > 1) {
+        stoptime = 0;               /* everyone stop now! */
+
+#ifdef APR_POLLSET_WAKEABLE
+        if (pollset_wakeable) {     /* wake up poll()ing workers */
+            int i;
+            for (i = 0; i < num_workers; ++i) {
+                apr_pollset_wakeup(workers[i].pollset);
+            }
+        }
+#endif
+    }
+    else {
+        consolidate_metrics();
+        output_results();
+        exit(1);
+    }
 }
+#endif /* SIGINT */
 
 /* ------------------------------------------------------- */
 
@@ -2246,6 +2571,7 @@ static void usage(const char *progname)
     fprintf(stderr, "Options are:\n");
     fprintf(stderr, "    -n requests     Number of requests to perform\n");
     fprintf(stderr, "    -c concurrency  Number of multiple requests to make at a time\n");
+    fprintf(stderr, "    -W workers      Number of concurrent worker threads\n");
     fprintf(stderr, "    -t timelimit    Seconds to max. to spend on benchmarking\n");
     fprintf(stderr, "                    This implies -n 50000\n");
     fprintf(stderr, "    -s timeout      Seconds to max. wait for each response\n");
@@ -2416,7 +2742,7 @@ static apr_status_t open_postfile(const char *pfile)
         return rv;
     }
     postlen = (apr_size_t)finfo.size;
-    postdata = xmalloc(postlen);
+    postdata = apr_palloc(cntxt, postlen);
     rv = apr_file_read_full(postfd, postdata, postlen, NULL);
     if (rv != APR_SUCCESS) {
         fprintf(stderr, "ab: Could not read POST data file: %s\n",
@@ -2445,6 +2771,8 @@ int main(int argc, const char * const argv[])
     AB_SSL_METHOD_CONST SSL_METHOD *meth = SSLv23_client_method();
 #endif /* USE_SSL */
 
+    srand((unsigned int)apr_time_now());
+
     /* table defaults  */
     tablestring = "";
     trstring = "";
@@ -2479,10 +2807,11 @@ int main(int argc, const char * const argv[])
 
     myhost = NULL; /* 0.0.0.0 or :: */
 
-    APR_RING_INIT(&delay_head, connection, delay_list);
-
     apr_getopt_init(&opt, cntxt, argc, argv);
     while ((status = apr_getopt(opt, "n:c:t:s:b:T:p:u:v:lrkVhwiIx:y:z:C:H:P:A:g:X:de:SqB:m:R:"
+#if APR_HAS_THREADS
+            "W:"
+#endif
 #ifdef USE_SSL
             "Z:f:E:"
 #endif
@@ -2494,6 +2823,14 @@ int main(int argc, const char * const argv[])
                     err("Invalid number of requests\n");
                 }
                 break;
+#if APR_HAS_THREADS
+            case 'W':
+                num_workers = atoi(opt_arg);
+                if (num_workers < 0) {
+                    err("Invalid number of workers\n");
+                }
+                break;
+#endif
             case 'k':
                 keepalive = 1;
                 break;
@@ -2502,6 +2839,9 @@ int main(int argc, const char * const argv[])
                 break;
             case 'c':
                 concurrency = atoi(opt_arg);
+                if (concurrency < 0) {
+                    err("Invalid negative concurrency\n");
+                }
                 break;
             case 'b':
                 windowsize = atoi(opt_arg);
@@ -2512,13 +2852,13 @@ int main(int argc, const char * const argv[])
                 method = HEAD;
                 break;
             case 'g':
-                gnuplot = xstrdup(opt_arg);
+                gnuplot = apr_pstrdup(cntxt, opt_arg);
                 break;
             case 'd':
                 percentile = 0;
                 break;
             case 'e':
-                csvperc = xstrdup(opt_arg);
+                csvperc = apr_pstrdup(cntxt, opt_arg);
                 break;
             case 'S':
                 confidence = 0;
@@ -2558,6 +2898,8 @@ int main(int argc, const char * const argv[])
                 break;
             case 't':
                 tlimit = atoi(opt_arg);
+                if (tlimit < 0)
+                    err("Invalid negative timelimit\n");
                 requests = MAX_REQUESTS;    /* need to size data array on
                                              * something */
                 break;
@@ -2749,19 +3091,45 @@ int main(int argc, const char * const argv[])
         usage(argv[0]);
     }
 
-    if ((concurrency < 0) || (concurrency > MAX_CONCURRENCY)) {
+#if APR_HAS_THREADS
+    if (num_workers == 0) {
+#ifdef _SC_NPROCESSORS_ONLN
+        num_workers = sysconf(_SC_NPROCESSORS_ONLN);
+#else
+        err("-W0 not implemented on this platform\n");
+#endif
+    }
+    if (num_workers > 1) {
+        requests = ROUND_UP(requests, num_workers);
+        concurrency = ROUND_UP(concurrency, num_workers);
+    }
+    else {
+        num_workers = 1;
+    }
+#endif /* APR_HAS_THREADS */
+
+    if (concurrency > ROUND_UP(MAX_CONCURRENCY, num_workers)) {
         fprintf(stderr, "%s: Invalid Concurrency [Range 0..%d]\n",
-                argv[0], MAX_CONCURRENCY);
+                argv[0], ROUND_UP(MAX_CONCURRENCY, num_workers));
         usage(argv[0]);
     }
-
     if (concurrency > requests) {
         fprintf(stderr, "%s: Cannot use concurrency level greater than "
                 "total number of requests\n", argv[0]);
         usage(argv[0]);
     }
 
-    if ((heartbeatres) && (requests > 150)) {
+    if (tlimit) {
+        /* Print line every 10% of time */
+        hbperiod = apr_time_from_sec(tlimit) / 10;
+        if (hbperiod < apr_time_from_sec(1)) {
+            hbperiod = apr_time_from_sec(1);
+        }
+        else if (hbperiod > apr_time_from_sec(60)) {
+            hbperiod = apr_time_from_sec(60);
+        }
+    }
+    else if ((heartbeatres) && (requests > 150)) {
         heartbeatres = requests / 10;   /* Print line every 10% of requests */
         if (heartbeatres < 100)
             heartbeatres = 100; /* but never more often than once every 100