[config] config options to stream request/response (#949, #376)

This allows admin to configure if response is collected in entirety
prior to sending data to client

For compatibility with existing configs, default is existing behavior:
  buffer entire response prior to sending data to client

The following are config options, though not all implemented yet

// default: buffer entire request body before connecting to backend
server.stream-request-body = 0

// stream request body to backend; buffer to temp files
server.stream-request-body = 1

// stream request body to backend; minimal buffering might block upload
server.stream-request-body = 2

// default: buffer entire response body before sending to client
server.stream-request-body = 0

// stream response body to client; buffer to temp files
server.stream-request-body = 1

// stream response body to client; minimal buffering might block backend
server.stream-request-body = 2

x-ref:
  "fastcgi, cgi, flush, php5 problem."
  https://redmine.lighttpd.net/issues/949
 "Reimplement upload (POST) handling to match apache/zeus/thttpd/boa functionality"
  https://redmine.lighttpd.net/issues/376

src/base.h

@@ -263,6 +263,8 @@ typedef struct {
 	unsigned short use_xattr;
 	unsigned short follow_symlink;
 	unsigned short range_requests;
+	unsigned short stream_request_body;
+	unsigned short stream_response_body;
 
 	/* debug */
 

src/configfile.c

@@ -119,6 +119,8 @@ static int config_insert(server *srv) {
 		{ "server.http-parseopt-host-strict",  NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_SERVER     }, /* 73 */
 		{ "server.http-parseopt-host-normalize",NULL,T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_SERVER     }, /* 74 */
 		{ "server.bsd-accept-filter",          NULL, T_CONFIG_STRING,  T_CONFIG_SCOPE_CONNECTION }, /* 75 */
+		{ "server.stream-request-body",        NULL, T_CONFIG_SHORT,   T_CONFIG_SCOPE_CONNECTION }, /* 76 */
+		{ "server.stream-response-body",       NULL, T_CONFIG_SHORT,   T_CONFIG_SCOPE_CONNECTION }, /* 77 */
 
 		{ "server.host",
 			"use server.bind instead",
@@ -248,6 +250,8 @@ static int config_insert(server *srv) {
 		s->ssl_verifyclient_export_cert = 0;
 		s->ssl_disable_client_renegotiation = 1;
 		s->listen_backlog = (0 == i ? 1024 : srv->config_storage[0]->listen_backlog);
+		s->stream_request_body = 0;
+		s->stream_response_body = 0;
 
 		/* all T_CONFIG_SCOPE_CONNECTION options */
 		cv[2].destination = s->errorfile_prefix;
@@ -310,12 +314,21 @@ static int config_insert(server *srv) {
 	       || defined(__OpenBSD__) || defined(__DragonflyBSD__)
 		cv[75].destination = s->bsd_accept_filter;
 	      #endif
+		cv[76].destination = &(s->stream_request_body);
+		cv[77].destination = &(s->stream_response_body);
 
 		srv->config_storage[i] = s;
 
 		if (0 != (ret = config_insert_values_global(srv, config->value, cv, i == 0 ? T_CONFIG_SCOPE_SERVER : T_CONFIG_SCOPE_CONNECTION))) {
 			break;
 		}
+
+		if (s->stream_request_body & FDEVENT_STREAM_REQUEST_BUFMIN) {
+			s->stream_request_body |= FDEVENT_STREAM_REQUEST;
+		}
+		if (s->stream_response_body & FDEVENT_STREAM_RESPONSE_BUFMIN) {
+			s->stream_response_body |= FDEVENT_STREAM_RESPONSE;
+		}
 	}
 
 	{
@@ -453,6 +466,9 @@ int config_setup_connection(server *srv, connection *con) {
 	PATCH(range_requests);
 	PATCH(force_lowercase_filenames);
 	/*PATCH(listen_backlog);*//*(not necessary; used only at startup)*/
+	PATCH(stream_request_body);
+	PATCH(stream_response_body);
+
 	PATCH(ssl_enabled);
 
 	PATCH(ssl_pemfile);
@@ -563,6 +579,10 @@ int config_patch_connection(server *srv, connection *con) {
 				buffer_copy_buffer(con->server_name, s->server_name);
 			} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("server.tag"))) {
 				PATCH(server_tag);
+			} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("server.stream-request-body"))) {
+				PATCH(stream_request_body);
+			} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("server.stream-response-body"))) {
+				PATCH(stream_response_body);
 			} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("connection.kbytes-per-second"))) {
 				PATCH(kbytes_per_second);
 			} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("debug.log-request-handling"))) {

src/connections-glue.c

@@ -359,6 +359,7 @@ handler_t connection_handle_read_post_state(server *srv, connection *con) {
 
 	if (dst_cq->bytes_in == (off_t)con->request.content_length) {
 		/* Content is ready */
+		con->conf.stream_request_body &= ~FDEVENT_STREAM_REQUEST_POLLIN;
 		connection_set_state(srv, con, CON_STATE_HANDLE_REQUEST);
 		return HANDLER_GO_ON;
 	} else if (is_closed) {
@@ -372,6 +373,7 @@ handler_t connection_handle_read_post_state(server *srv, connection *con) {
 	      #endif
 		return HANDLER_ERROR;
 	} else {
+		con->conf.stream_request_body |= FDEVENT_STREAM_REQUEST_POLLIN;
 		return HANDLER_WAIT_FOR_EVENT;
 	}
 }

src/connections.c

@@ -361,12 +361,6 @@ static int connection_handle_write_prepare(server *srv, connection *con) {
 		}
 	}
 
-	if (con->request.content_length
-	    && (off_t)con->request.content_length > con->request_content_queue->bytes_in) {
-		/* request body is present and has not been read completely */
-		con->keep_alive = 0;
-	}
-
 	if (con->request.http_method == HTTP_METHOD_HEAD) {
 		/**
 		 * a HEAD request has the same as a GET 
@@ -1037,7 +1031,7 @@ int connection_state_machine(server *srv, connection *con) {
 
 			switch (r = http_response_prepare(srv, con)) {
 			case HANDLER_WAIT_FOR_EVENT:
-				if (!con->file_started) break; /* come back here */
+				if (!con->file_started || 0 == con->conf.stream_response_body) break; /* come back here */
 				/* response headers received from backend; fall through to start response */
 			case HANDLER_FINISHED:
 				if (con->error_handler_saved_status > 0) {
@@ -1168,6 +1162,12 @@ int connection_state_machine(server *srv, connection *con) {
 						"state for fd", con->fd, connection_get_state(con->state));
 			}
 
+			if (con->request.content_length
+			    && (off_t)con->request.content_length > con->request_content_queue->bytes_in) {
+				/* request body is present and has not been read completely */
+				con->keep_alive = 0;
+			}
+
 			plugins_call_handle_request_done(srv, con);
 
 			srv->con_written++;
@@ -1441,11 +1441,11 @@ int connection_state_machine(server *srv, connection *con) {
 				connection_get_state(con->state));
 	}
 
+	r = 0;
 	switch(con->state) {
-	case CON_STATE_READ_POST:
 	case CON_STATE_READ:
 	case CON_STATE_CLOSE:
-		fdevent_event_set(srv->ev, &(con->fde_ndx), con->fd, FDEVENT_IN);
+		r = FDEVENT_IN;
 		break;
 	case CON_STATE_WRITE:
 		/* request write-fdevent only if we really need it
@@ -1455,15 +1455,30 @@ int connection_state_machine(server *srv, connection *con) {
 		if (!chunkqueue_is_empty(con->write_queue) &&
 		    (con->is_writable == 0) &&
 		    (con->traffic_limit_reached == 0)) {
-			fdevent_event_set(srv->ev, &(con->fde_ndx), con->fd, FDEVENT_OUT);
-		} else {
-			fdevent_event_set(srv->ev, &(con->fde_ndx), con->fd, 0);
+			r |= FDEVENT_OUT;
+		}
+		/* fall through */
+	case CON_STATE_READ_POST:
+		if (con->conf.stream_request_body & FDEVENT_STREAM_REQUEST_POLLIN) {
+			r |= FDEVENT_IN;
 		}
 		break;
 	default:
-		fdevent_event_set(srv->ev, &(con->fde_ndx), con->fd, 0);
 		break;
 	}
+	if (-1 != con->fd) {
+		const int events = fdevent_event_get_interest(srv->ev, con->fd);
+		if (r != events) {
+			/* update timestamps when enabling interest in events */
+			if ((r & FDEVENT_IN) && !(events & FDEVENT_IN)) {
+				con->read_idle_ts = srv->cur_ts;
+			}
+			if ((r & FDEVENT_OUT) && !(events & FDEVENT_OUT)) {
+				con->write_request_ts = srv->cur_ts;
+			}
+			fdevent_event_set(srv->ev, &con->fde_ndx, con->fd, r);
+		}
+	}
 
 	return 0;
 }

src/fdevent.h

@@ -65,6 +65,13 @@ typedef handler_t (*fdevent_handler)(struct server *srv, void *ctx, int revents)
 #define FDEVENT_HUP    BV(4)
 #define FDEVENT_NVAL   BV(5)
 
+#define FDEVENT_STREAM_REQUEST          BV(0)
+#define FDEVENT_STREAM_REQUEST_BUFMIN   BV(1)
+#define FDEVENT_STREAM_REQUEST_POLLIN   BV(15)
+
+#define FDEVENT_STREAM_RESPONSE         BV(0)
+#define FDEVENT_STREAM_RESPONSE_BUFMIN  BV(1)
+
 typedef enum { FD_EVENT_TYPE_UNSET = -1,
 		FD_EVENT_TYPE_CONNECTION,
 		FD_EVENT_TYPE_FCGI_CONNECTION,
@@ -173,6 +180,8 @@ fdevents *fdevent_init(struct server *srv, size_t maxfds, fdevent_handler_t type
 int fdevent_reset(fdevents *ev); /* "init" after fork() */
 void fdevent_free(fdevents *ev);
 
+#define fdevent_event_get_interest(ev, fd) \
+        (-1 != (fd) ? (ev)->fdarray[(fd)]->events : 0)
 void fdevent_event_set(fdevents *ev, int *fde_ndx, int fd, int events); /* events can be FDEVENT_IN, FDEVENT_OUT or FDEVENT_IN | FDEVENT_OUT */
 void fdevent_event_del(fdevents *ev, int *fde_ndx, int fd);
 int fdevent_event_get_revent(fdevents *ev, size_t ndx);

src/server.c

@@ -1576,15 +1576,13 @@ int main (int argc, char **argv) {
 				 *
 				 */
 				for (ndx = 0; ndx < conns->used; ndx++) {
+					connection * const con = conns->ptr[ndx];
+					const int waitevents = fdevent_event_get_interest(srv->ev, con->fd);
 					int changed = 0;
-					connection *con;
 					int t_diff;
 
-					con = conns->ptr[ndx];
-
-					if (con->state == CON_STATE_READ ||
-					    con->state == CON_STATE_READ_POST) {
-						if (con->request_count == 1 || con->state == CON_STATE_READ_POST) {
+					if (waitevents & FDEVENT_IN) {
+						if (con->request_count == 1 || con->state != CON_STATE_READ) { /* e.g. CON_STATE_READ_POST || CON_STATE_WRITE */
 							if (srv->cur_ts - con->read_idle_ts > con->conf.max_read_idle) {
 								/* time - out */
 								if (con->conf.log_request_handling) {
@@ -1609,6 +1607,11 @@ int main (int argc, char **argv) {
 						}
 					}
 
+					/* max_write_idle timeout currently functions as backend timeout,
+					 * too, after response has been started.
+					 * future: have separate backend timeout, and then change this
+					 * to check for write interest before checking for timeout */
+					/*if (waitevents & FDEVENT_OUT)*/
 					if ((con->state == CON_STATE_WRITE) &&
 					    (con->write_request_ts != 0)) {
 #if 0