Improve on sending data to the client.

- Attempt to chain as much as we can in the send netbufs
  (keeps down the SSL_write calls, silly seeing it go out with 8 bytes)
- Change NETBUF_SEND_PAYLOAD_MAX to 4K
- Call SSL_write() with max NETBUF_SEND_PAYLOAD_MAX in size
- Go back to flushing the send buffers after each request
- No more need for a fixed pool for nb->buf, go back to the heap for now
- Disable Nagle, we're doing the chaining now anyway

includes/kore.h

@@ -56,9 +56,9 @@
 #define kore_debug(fmt, ...)
 #endif
 
-#define NETBUF_RECV		0
-#define NETBUF_SEND		1
-#define NETBUF_SEND_PAYLOAD_MAX	16384
+#define NETBUF_RECV			0
+#define NETBUF_SEND			1
+#define NETBUF_SEND_PAYLOAD_MAX		4000
 
 #define NETBUF_CALL_CB_ALWAYS	0x01
 #define NETBUF_FORCE_REMOVE	0x02
@@ -113,7 +113,6 @@ LIST_HEAD(listener_head, listener);
 #define CONN_WRITE_BLOCK	0x04
 #define CONN_IDLE_TIMER_ACT	0x10
 #define CONN_READ_BLOCK		0x20
-#define CONN_WILL_FLUSH		0x40
 
 #define KORE_IDLE_TIMER_MAX	20000
 

src/connection.c

@@ -17,6 +17,8 @@
 #include <sys/param.h>
 #include <sys/socket.h>
 
+#include <netinet/tcp.h>
+
 #include <fcntl.h>
 
 #include "kore.h"
@@ -209,9 +211,6 @@ kore_connection_remove(struct connection *c)
 
 	kore_debug("kore_connection_remove(%p)", c);
 
-	/* XXX */
-	net_send_flush(c);
-
 	if (c->ssl != NULL)
 		SSL_free(c->ssl);
 	close(c->fd);
@@ -230,8 +229,7 @@ kore_connection_remove(struct connection *c)
 	for (nb = TAILQ_FIRST(&(c->send_queue)); nb != NULL; nb = next) {
 		next = TAILQ_NEXT(nb, list);
 		TAILQ_REMOVE(&(c->send_queue), nb, list);
-		if (nb->buf != NULL)
-			kore_mem_free(nb->buf);
+		kore_mem_free(nb->buf);
 		kore_pool_put(&nb_pool, nb);
 	}
 
@@ -310,5 +308,12 @@ kore_connection_nonblock(int fd)
 		return (KORE_RESULT_ERROR);
 	}
 
+	flags = 1;
+	if (setsockopt(fd, IPPROTO_TCP,
+	    TCP_NODELAY, (char *)&flags, sizeof(flags)) == -1) {
+		kore_log(LOG_NOTICE,
+		    "failed to set TCP_NODELAY on %d", fd);
+	}
+
 	return (KORE_RESULT_OK);
 }

src/http.c

@@ -112,13 +112,10 @@ http_request_new(struct connection *c, struct spdy_stream *s, char *host,
 void
 http_process(void)
 {
-	struct connection		*c, *cnext;
 	struct kore_module_handle	*hdlr;
-	TAILQ_HEAD(, connection)	flush_list;
 	struct http_request		*req, *next;
 	int				r, (*cb)(struct http_request *);
 
-	TAILQ_INIT(&flush_list);
 	for (req = TAILQ_FIRST(&http_requests); req != NULL; req = next) {
 		next = TAILQ_NEXT(req, list);
 
@@ -147,11 +144,9 @@ http_process(void)
 
 		switch (r) {
 		case KORE_RESULT_OK:
-			if (!(req->owner->flags & CONN_WILL_FLUSH)) {
-				req->owner->flags |= CONN_WILL_FLUSH;
-				TAILQ_INSERT_TAIL(&flush_list,
-				    req->owner, flush_list);
-			}
+			r = net_send_flush(req->owner);
+			if (r == KORE_RESULT_ERROR)
+				kore_connection_disconnect(req->owner);
 			break;
 		case KORE_RESULT_ERROR:
 			kore_connection_disconnect(req->owner);
@@ -168,17 +163,6 @@ http_process(void)
 			http_request_count--;
 		}
 	}
-
-	for (c = TAILQ_FIRST(&flush_list); c != NULL; c = cnext) {
-		cnext = TAILQ_NEXT(c, flush_list);
-		TAILQ_REMOVE(&flush_list, c, flush_list);
-
-		r = net_send_flush(c);
-		if (r == KORE_RESULT_ERROR)
-			kore_connection_disconnect(c);
-		else
-			c->flags &= ~CONN_WILL_FLUSH;
-	}
 }
 
 void

src/net.c

@@ -14,16 +14,16 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
+#include <sys/param.h>
+
 #include "kore.h"
 
 struct kore_pool		nb_pool;
-struct kore_pool		nsb_pool;
 
 void
 net_init(void)
 {
 	kore_pool_init(&nb_pool, "nb_pool", sizeof(struct netbuf), 1000);
-	kore_pool_init(&nsb_pool, "nsb_pool", NETBUF_SEND_PAYLOAD_MAX, 25);
 }
 
 void
@@ -33,31 +33,6 @@ net_send_queue(struct connection *c, u_int8_t *data, u_int32_t len)
 	struct netbuf		*nb;
 	u_int32_t		avail, i, payload;
 
-	if (len > NETBUF_SEND_PAYLOAD_MAX) {
-		p = data;
-		avail = (len / NETBUF_SEND_PAYLOAD_MAX) + 1;
-		for (i = 0; i < avail && len != 0; i++) {
-			if (len > NETBUF_SEND_PAYLOAD_MAX)
-				payload = NETBUF_SEND_PAYLOAD_MAX;
-			else
-				payload = len;
-
-			net_send_queue(c, p, payload);
-
-			p += payload;
-			len -= payload;
-		}
-
-		return;
-	}
-
-	if (len > NETBUF_SEND_PAYLOAD_MAX) {
-		kore_log(LOG_NOTICE,
-		    "net_send_queue(): %d > NETBUF_SEND_PAYLOAD_MAX", len);
-		kore_connection_disconnect(c);
-		return;
-	}
-
 	nb = TAILQ_LAST(&(c->send_queue), netbuf_head);
 	if (nb != NULL && nb->b_len < nb->m_len) {
 		avail = nb->m_len - nb->b_len;
@@ -83,9 +58,13 @@ net_send_queue(struct connection *c, u_int8_t *data, u_int32_t len)
 	nb->s_off = 0;
 	nb->b_len = len;
 	nb->type = NETBUF_SEND;
-	nb->m_len = NETBUF_SEND_PAYLOAD_MAX;
-	nb->buf = kore_pool_get(&nsb_pool);
 
+	if (nb->b_len < NETBUF_SEND_PAYLOAD_MAX)
+		nb->m_len = NETBUF_SEND_PAYLOAD_MAX;
+	else
+		nb->m_len = nb->b_len;
+
+	nb->buf = kore_malloc(nb->m_len);
 	if (len > 0)
 		memcpy(nb->buf, data, nb->b_len);
 
@@ -138,12 +117,14 @@ net_send(struct connection *c)
 {
 	int			r;
 	struct netbuf		*nb;
+	u_int32_t		len;
 
 	while (!TAILQ_EMPTY(&(c->send_queue))) {
 		nb = TAILQ_FIRST(&(c->send_queue));
 		if (nb->b_len != 0) {
-			r = SSL_write(c->ssl,
-			    (nb->buf + nb->s_off), (nb->b_len - nb->s_off));
+			len = MIN(NETBUF_SEND_PAYLOAD_MAX,
+			    nb->b_len - nb->s_off);
+			r = SSL_write(c->ssl, (nb->buf + nb->s_off), len);
 
 			kore_debug("net_send(%d/%d bytes), progress with %d",
 			    nb->s_off, nb->b_len, r);
@@ -168,7 +149,7 @@ net_send(struct connection *c)
 		if (nb->s_off == nb->b_len) {
 			TAILQ_REMOVE(&(c->send_queue), nb, list);
 
-			kore_pool_put(&nsb_pool, nb->buf);
+			kore_mem_free(nb->buf);
 			kore_pool_put(&nb_pool, nb);
 		}
 	}