* replace strerror with g_strerror, kill errno from logging

* starting chunked encoding filter
* network write cleanup + sendfile backend (default for now)

src/base.h

@@ -6,6 +6,7 @@
 #define CONST_STR_LEN(x) (x), (x) ? sizeof(x) - 1 : 0
 
 #define GSTR_LEN(x) (x) ? (x)->str : "", (x) ? (x)->len : 0
+#define GSTR_SAFE_STR(x) ((x && x->str) ? x->str : "(null)")
 
 /* we don't use ev_init for now (stupid alias warnings), as ev_init
  * just does set some values to zero and calls ev_set_cb.

src/chunk.c

@@ -51,11 +51,19 @@ handler_t chunkfile_open(server *srv, connection *con, chunkfile *cf) {
 	}
 	if (-1 == (cf->fd = open(cf->name->str, O_RDONLY))) {
 		if (EMFILE == errno) return HANDLER_WAIT_FOR_FD;
-		CON_ERROR(srv, con, "Couldn't open file '%s': %s (%i)", cf->name->str, strerror(errno), errno);
+		CON_ERROR(srv, con, "Couldn't open file '%s': %s", GSTR_SAFE_STR(cf->name), g_strerror(errno));
 		return HANDLER_ERROR;
 	}
 #ifdef FD_CLOEXEC
 	fcntl(cf->fd, F_SETFD, FD_CLOEXEC);
+#endif
+#ifdef HAVE_POSIX_FADVISE
+	/* tell the kernel that we want to stream the file */
+	if (-1 == posix_fadvise(cf->fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {
+		if (ENOSYS != errno) {
+			CON_ERROR(srv, con, "posix_fadvise failed for '%s': %s (%i)", GSTR_SAFE_STR(cf->name), g_strerror(errno), cf->fd);
+		}
+	}
 #endif
 	return HANDLER_GO_ON;
 }
@@ -131,13 +139,13 @@ handler_t chunkiter_read(server *srv, connection *con, chunkiter iter, off_t sta
 				if (-1 == lseek(c->file.file->fd, our_start, SEEK_SET)) {
 					/* prefer the error of the first syscall */
 					if (0 != mmap_errno) {
-						CON_ERROR(srv, con, "mmap failed for '%s' (fd = %i): %s (%i)",
-							c->file.file->name ? c->file.file->name->str : "(null)", c->file.file->fd,
-							strerror(mmap_errno), mmap_errno);
+						CON_ERROR(srv, con, "mmap failed for '%s' (fd = %i): %s",
+							GSTR_SAFE_STR(c->file.file->name), c->file.file->fd,
+							g_strerror(mmap_errno));
 					} else {
-						CON_ERROR(srv, con, "lseek failed for '%s' (fd = %i): %s (%i)",
-							c->file.file->name ? c->file.file->name->str : "(null)", c->file.file->fd,
-							strerror(errno), errno);
+						CON_ERROR(srv, con, "lseek failed for '%s' (fd = %i): %s",
+							GSTR_SAFE_STR(c->file.file->name), c->file.file->fd,
+							g_strerror(errno));
 					}
 					g_string_free(c->mem, TRUE);
 					c->mem = NULL;
@@ -148,13 +156,13 @@ read_chunk:
 					if (EINTR == errno) goto read_chunk;
 					/* prefer the error of the first syscall */
 					if (0 != mmap_errno) {
-						CON_ERROR(srv, con, "mmap failed for '%s' (fd = %i): %s (%i)",
-							c->file.file->name ? c->file.file->name->str : "(null)", c->file.file->fd,
-							strerror(mmap_errno), mmap_errno);
+						CON_ERROR(srv, con, "mmap failed for '%s' (fd = %i): %s",
+							GSTR_SAFE_STR(c->file.file->name), c->file.file->fd,
+							g_strerror(mmap_errno));
 					} else {
-						CON_ERROR(srv, con, "read failed for '%s' (fd = %i): %s (%i)",
-							c->file.file->name ? c->file.file->name->str : "(null)", c->file.file->fd,
-							strerror(errno), errno);
+						CON_ERROR(srv, con, "read failed for '%s' (fd = %i): %s",
+							GSTR_SAFE_STR(c->file.file->name), c->file.file->fd,
+							g_strerror(errno));
 					}
 					g_string_free(c->mem, TRUE);
 					c->mem = NULL;
@@ -172,9 +180,9 @@ read_chunk:
 				/* don't advise files < 64Kb */
 				if (c->file.mmap.length > (64*1024) &&
 					0 != madvise(c->file.mmap.data, c->file.mmap.length, MADV_WILLNEED)) {
-					CON_ERROR(srv, con, "madvise failed for '%s' (fd = %i): %s (%i)",
-						c->file.file->name ? c->file.file->name->str : "(null)", c->file.file->fd,
-						strerror(errno), errno);
+					CON_ERROR(srv, con, "madvise failed for '%s' (fd = %i): %s",
+						GSTR_SAFE_STR(c->file.file->name), c->file.file->fd,
+						g_strerror(errno));
 				}
 #endif
 			}

src/filter_chunked.c

@@ -0,0 +1,48 @@
+
+#include "filter_chunked.h"
+
+/* len != 0 */
+static void http_chunk_append_len(chunkqueue *cq, size_t len) {
+	size_t i, olen = len, j;
+	GString *s;
+
+	s = g_string_sized_new(sizeof(len) * 2 + 2);
+
+	for (i = 0; i < 8 && len; i++) {
+		len >>= 4;
+	}
+
+	/* i is the number of hex digits we have */
+	g_string_set_size(s, i);
+
+	for (j = i-1, len = olen; j+1 > 0; j--) {
+		s->str[j] = (len & 0xf) + (((len & 0xf) <= 9) ? '0' : 'a' - 10);
+		len >>= 4;
+	}
+	g_string_append_len(s, CONST_STR_LEN("\r\n"));
+
+	chunkqueue_append_string(cq, s);
+}
+
+
+handler_t filter_chunked_encode(server *srv, connection *con, chunkqueue *out, chunkqueue *in) {
+	UNUSED(srv);
+	UNUSED(con);
+
+	if (in->length > 0) {
+		http_chunk_append_len(out, in->length);
+		chunkqueue_steal_all(out, in);
+	}
+	if (in->is_closed) {
+		if (!out->is_closed) {
+			chunkqueue_append_mem(out, CONST_STR_LEN("0\r\n"));
+			out->is_closed = TRUE;
+		}
+		return HANDLER_FINISHED;
+	}
+	return HANDLER_GO_ON;
+}
+
+handler_t filter_chunked_decode(server *srv, connection *con, chunkqueue *out, chunkqueue *in) {
+	return HANDLER_ERROR;
+}

src/filter_chunked.h

@@ -0,0 +1,9 @@
+#ifndef _LIGHTTPD_FILTER_CHUNKED_H_
+#define _LIGHTTPD_FILTER_CHUNKED_H_
+
+#include "base.h"
+
+LI_API handler_t filter_chunked_encode(server *srv, connection *con, chunkqueue *out, chunkqueue *in);
+LI_API handler_t filter_chunked_decode(server *srv, connection *con, chunkqueue *out, chunkqueue *in);
+
+#endif

src/log.c

@@ -282,7 +282,7 @@ log_t *log_new(server *srv, log_type_t type, GString *path) {
 	}
 
 	if (fd == -1) {
-		g_printerr("failed to open log: %s %d", strerror(errno), errno);
+		g_printerr("failed to open log: %s", g_strerror(errno));
 		return NULL;
 	}
 

src/network.c

@@ -51,7 +51,8 @@ network_status_t network_write(server *srv, connection *con, int fd, chunkqueue
 	}
 #endif
 
-	res = network_backend_writev(srv, con, fd, cq);
+	/* res = network_write_writev(srv, con, fd, cq); */
+	res = network_write_sendfile(srv, con, fd, cq);
 
 #ifdef TCP_CORK
 	if (corked) {
@@ -83,7 +84,7 @@ network_status_t network_read(server *srv, connection *con, int fd, chunkqueue *
 			case ECONNRESET:
 				return NETWORK_STATUS_CONNECTION_CLOSE;
 			default:
-				CON_ERROR(srv, con, "oops, read from fd=%d failed: %s (%d)", fd, strerror(errno), errno );
+				CON_ERROR(srv, con, "oops, read from fd=%d failed: %s", fd, g_strerror(errno) );
 				return NETWORK_STATUS_FATAL_ERROR;
 			}
 		} else if (0 == r) {

src/network.h

@@ -21,13 +21,20 @@ LI_API ssize_t net_read(int fd, void *buf, ssize_t nbyte);
 LI_API network_status_t network_write(server *srv, connection *con, int fd, chunkqueue *cq);
 LI_API network_status_t network_read(server *srv, connection *con, int fd, chunkqueue *cq);
 
+/* use writev for mem chunks, buffered read/write for files */
+LI_API network_status_t network_write_writev(server *srv, connection *con, int fd, chunkqueue *cq);
+
+/* use sendfile for files, writev for mem chunks */
+LI_API network_status_t network_write_sendfile(server *srv, connection *con, int fd, chunkqueue *cq);
+
 /* write backends */
-LI_API network_status_t network_backend_write(server *srv, connection *con, int fd, chunkqueue *cq);
-LI_API network_status_t network_backend_writev(server *srv, connection *con, int fd, chunkqueue *cq);
+LI_API network_status_t network_backend_write(server *srv, connection *con, int fd, chunkqueue *cq, goffset *write_max);
+LI_API network_status_t network_backend_writev(server *srv, connection *con, int fd, chunkqueue *cq, goffset *write_max);
+LI_API network_status_t network_backend_writev(server *srv, connection *con, int fd, chunkqueue *cq, goffset *write_max);
 
-#define NETWORK_FALLBACK(f) do { \
+#define NETWORK_FALLBACK(f, write_max) do { \
 	network_status_t res; \
-	switch(res = f(srv, con, fd, cq)) { \
+	switch(res = f(srv, con, fd, cq, write_max)) { \
 		case NETWORK_STATUS_SUCCESS: \
 			break; \
 		default: \

src/network_linux_sendfile.c

@@ -0,0 +1,92 @@
+
+#include "network.h"
+
+/* first chunk must be a FILE_CHUNK ! */
+network_status_t network_backend_sendfile(server *srv, connection *con, int fd, chunkqueue *cq, goffset *write_max) {
+	off_t file_offset, toSend;
+	ssize_t r;
+	gboolean did_write_something = FALSE;
+	chunkiter ci;
+	chunk *c;
+
+	if (0 == cq->length) return NETWORK_STATUS_FATAL_ERROR;
+
+	do {
+		ci = chunkqueue_iter(cq);
+
+		if (FILE_CHUNK != (c = chunkiter_chunk(ci))->type) {
+			return did_write_something ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_FATAL_ERROR;
+		}
+
+		switch (chunkfile_open(srv, con, c->file.file)) {
+		case HANDLER_GO_ON:
+			break;
+		case HANDLER_WAIT_FOR_FD:
+			return NETWORK_STATUS_WAIT_FOR_FD;
+		default:
+			return NETWORK_STATUS_FATAL_ERROR;
+		}
+
+		file_offset = c->offset + c->file.start;
+		toSend = c->file.length - c->offset;
+		if (toSend > *write_max) toSend = *write_max;
+
+		while (-1 == (r = sendfile(fd, c->file.file->fd, &file_offset, toSend))) {
+			switch (errno) {
+			case EAGAIN:
+#if EWOULDBLOCK != EAGAIN
+			case EWOULDBLOCK
+#endif
+				return did_write_something ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_WAIT_FOR_EVENT;
+			case ECONNRESET:
+			case EPIPE:
+				return NETWORK_STATUS_CONNECTION_CLOSE;
+			case EINTR:
+				break; /* try again */
+			default:
+				CON_ERROR(srv, con, "oops, write to fd=%d failed: %s", fd, g_strerror(errno));
+				return NETWORK_STATUS_FATAL_ERROR;
+			}
+		}
+		if (0 == r) {
+			/* don't care about cached stat - file is open */
+			struct stat st;
+			if (-1 == fstat(fd, &st)) {
+				CON_ERROR(srv, con, "Couldn't fstat file: %s", g_strerror(errno));
+				return NETWORK_STATUS_FATAL_ERROR;
+			}
+
+			if (file_offset > st.st_size) {
+				/* file shrinked, close the connection */
+				CON_ERROR(srv, con, "%s", "File shrinked, aborting");
+				return NETWORK_STATUS_FATAL_ERROR;
+			}
+			return did_write_something ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_WAIT_FOR_EVENT;
+		}
+		chunkqueue_skip(cq, r);
+		*write_max -= r;
+		did_write_something = TRUE;
+		if (0 == cq->length) return NETWORK_STATUS_SUCCESS;
+	} while (r == toSend && *write_max > 0);
+
+	return NETWORK_STATUS_SUCCESS;
+}
+
+network_status_t network_write_sendfile(server *srv, connection *con, int fd, chunkqueue *cq) {
+	goffset write_max = 256*1024; // 256kB //;
+	if (cq->length == 0) return NETWORK_STATUS_FATAL_ERROR;
+	do {
+		switch (chunkqueue_first_chunk(cq)->type) {
+		case MEM_CHUNK:
+			NETWORK_FALLBACK(network_backend_writev, &write_max);
+			break;
+		case FILE_CHUNK:
+			NETWORK_FALLBACK(network_backend_sendfile, &write_max);
+			break;
+		default:
+			return NETWORK_STATUS_FATAL_ERROR;
+		}
+		if (cq->length == 0) return NETWORK_STATUS_SUCCESS;
+	} while (write_max > 0);
+	return NETWORK_STATUS_SUCCESS;
+}

src/network_write.c

@@ -1,24 +1,24 @@
 
 #include "network.h"
 
-network_status_t network_backend_write(server *srv, connection *con, int fd, chunkqueue *cq) {
+network_status_t network_backend_write(server *srv, connection *con, int fd, chunkqueue *cq, goffset *write_max) {
 	const ssize_t blocksize = 16*1024; /* 16k */
-	const off_t max_write = 16 * blocksize; /* 256k */
 	char *block_data;
 	off_t block_len;
 	ssize_t r;
-	off_t len = 0;
+	gboolean did_write_something = FALSE;
 	chunkiter ci;
 
 	do {
-		if (0 == cq->length) return NETWORK_STATUS_SUCCESS;
+		if (0 == cq->length)
+			return did_write_something ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_FATAL_ERROR;
 
 		ci = chunkqueue_iter(cq);
 		switch (chunkiter_read(srv, con, ci, 0, blocksize, &block_data, &block_len)) {
 		case HANDLER_GO_ON:
 			break;
 		case HANDLER_WAIT_FOR_FD:
-			return len ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_WAIT_FOR_EVENT;
+			return did_write_something ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_WAIT_FOR_FD;
 		case HANDLER_ERROR:
 		default:
 			return NETWORK_STATUS_FATAL_ERROR;
@@ -30,20 +30,21 @@ network_status_t network_backend_write(server *srv, connection *con, int fd, chu
 #if EWOULDBLOCK != EAGAIN
 			case EWOULDBLOCK
 #endif
-				return len ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_WAIT_FOR_EVENT;
+				return did_write_something ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_WAIT_FOR_EVENT;
 			case ECONNRESET:
 			case EPIPE:
 				return NETWORK_STATUS_CONNECTION_CLOSE;
 			default:
-				CON_ERROR(srv, con, "oops, write to fd=%d failed: %s (%d)", fd, strerror(errno), errno );
+				CON_ERROR(srv, con, "oops, write to fd=%d failed: %s", fd, g_strerror(errno));
 				return NETWORK_STATUS_FATAL_ERROR;
 			}
 		} else if (0 == r) {
-			return len ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_WAIT_FOR_EVENT;
+			return did_write_something ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_WAIT_FOR_EVENT;
 		}
 		chunkqueue_skip(cq, r);
-		len += r;
-	} while (r == block_len && len < max_write);
+		did_write_something = TRUE;
+		*write_max -= r;
+	} while (r == block_len && *write_max > 0);
 
 	return NETWORK_STATUS_SUCCESS;
 }

src/network_writev.c

@@ -24,9 +24,9 @@
 # endif
 #endif
 
-network_status_t network_backend_writev(server *srv, connection *con, int fd, chunkqueue *cq) {
-	const off_t max_write = 256 * 1024; /* 256k */
-	off_t min_cq_len, max_chunks_len, we_have;
+/* first chunk must be a MEM_CHUNK ! */
+network_status_t network_backend_writev(server *srv, connection *con, int fd, chunkqueue *cq, goffset *write_max) {
+	off_t we_have;
 	ssize_t r;
 	gboolean did_write_something = FALSE;
 	chunkiter ci;
@@ -34,22 +34,15 @@ network_status_t network_backend_writev(server *srv, connection *con, int fd, ch
 
 	GArray *chunks = g_array_sized_new(FALSE, TRUE, sizeof(struct iovec), UIO_MAXIOV);
 
-	/* stop if chunkqueue length gets less or equal than min_cq_len */
-	min_cq_len = cq->length - max_write;
-	if (min_cq_len < 0) min_cq_len = 0;
-
 	do {
-		if (0 == cq->length) return NETWORK_STATUS_SUCCESS;
+		if (0 == cq->length) return did_write_something ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_FATAL_ERROR;
 
 		ci = chunkqueue_iter(cq);
 
 		if (MEM_CHUNK != (c = chunkiter_chunk(ci))->type) {
-			NETWORK_FALLBACK(network_backend_write);
-			did_write_something = TRUE;
-			continue;
+			return did_write_something ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_FATAL_ERROR;
 		}
 
-		max_chunks_len = cq->length - min_cq_len;
 		we_have = 0;
 		do {
 			guint i = chunks->len;
@@ -58,10 +51,10 @@ network_status_t network_backend_writev(server *srv, connection *con, int fd, ch
 			g_array_set_size(chunks, i + 1);
 			v = &g_array_index(chunks, struct iovec, i);
 			v->iov_base = c->mem->str + c->offset;
-			if (len > max_write) len = max_write;
+			if (len > *write_max - we_have) len = *write_max - we_have;
 			v->iov_len = len;
 			we_have += len;
-		} while (we_have < max_chunks_len &&
+		} while (we_have < *write_max &&
 		         chunkiter_next(&ci) &&
 		         MEM_CHUNK == (c = chunkiter_chunk(ci))->type &&
 		         chunks->len < UIO_MAXIOV);
@@ -82,7 +75,7 @@ network_status_t network_backend_writev(server *srv, connection *con, int fd, ch
 				break; /* try again */
 			default:
 				g_array_free(chunks, TRUE);
-				CON_ERROR(srv, con, "oops, write to fd=%d failed: %s (%d)", fd, strerror(errno), errno );
+				CON_ERROR(srv, con, "oops, write to fd=%d failed: %s", fd, g_strerror(errno));
 				return NETWORK_STATUS_FATAL_ERROR;
 			}
 		}
@@ -91,14 +84,34 @@ network_status_t network_backend_writev(server *srv, connection *con, int fd, ch
 			return did_write_something ? NETWORK_STATUS_SUCCESS : NETWORK_STATUS_WAIT_FOR_EVENT;
 		}
 		chunkqueue_skip(cq, r);
+		*write_max -= r;
 		if (r != we_have) {
 			g_array_free(chunks, TRUE);
 			return NETWORK_STATUS_SUCCESS;
 		}
 		did_write_something = TRUE;
 		g_array_set_size(chunks, 0);
-	} while (cq->length > min_cq_len);
+	} while (*write_max > 0);
 
 	g_array_free(chunks, TRUE);
 	return NETWORK_STATUS_SUCCESS;
 }
+
+network_status_t network_write_writev(server *srv, connection *con, int fd, chunkqueue *cq) {
+	goffset write_max = 256*1024; // 256k //;
+	if (cq->length == 0) return NETWORK_STATUS_FATAL_ERROR;
+	do {
+		switch (chunkqueue_first_chunk(cq)->type) {
+		case MEM_CHUNK:
+			NETWORK_FALLBACK(network_backend_writev, &write_max);
+			break;
+		case FILE_CHUNK:
+			NETWORK_FALLBACK(network_backend_write, &write_max);
+			break;
+		default:
+			return NETWORK_STATUS_FATAL_ERROR;
+		}
+		if (cq->length == 0) return NETWORK_STATUS_SUCCESS;
+	} while (write_max > 0);
+	return NETWORK_STATUS_SUCCESS;
+}

src/response.c

@@ -50,8 +50,10 @@ void response_send_headers(server *srv, connection *con) {
 		g_string_printf(srv->tmp_str, "%"L_GOFFSET_FORMAT, con->out->length);
 		http_header_overwrite(con->response.headers, CONST_STR_LEN("Content-Length"), GSTR_LEN(srv->tmp_str));
 	} else if (con->keep_alive && con->request.http_version == HTTP_VERSION_1_1) {
-		con->response.transfer_encoding |= HTTP_TRANSFER_ENCODING_CHUNKED;
-		http_header_overwrite(con->response.headers, CONST_STR_LEN("Transfer-Encoding"), CONST_STR_LEN("chunked"));
+		if (!(con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED)) {
+			con->response.transfer_encoding |= HTTP_TRANSFER_ENCODING_CHUNKED;
+			http_header_append(con->response.headers, CONST_STR_LEN("Transfer-Encoding"), CONST_STR_LEN("chunked"));
+		}
 	} else {
 		/* Unknown content length, no chunked encoding */
 		con->keep_alive = FALSE;

src/server.c

@@ -254,7 +254,7 @@ static void server_listen_cb(struct ev_loop *loop, ev_io *w, int revents) {
 		/* TODO: server_out_of_fds(srv, NULL); */
 		break;
 	default:
-		ERROR(srv, "accept failed on fd=%d with error: (%d) %s", w->fd, errno, strerror(errno));
+		ERROR(srv, "accept failed on fd=%d with error: %s", w->fd, g_strerror(errno));
 		break;
 	}
 }

src/wscript

@@ -15,11 +15,13 @@ common_source='''
 	condition_parsers.rl
 	config_parser.rl
 	connection.c
+	filter_chunked.c
 	http_headers.c
 	http_request_parser.rl
 	log.c
 	network.c
 	network_write.c network_writev.c
+		network_linux_sendfile.c
 	options.c
 	plugin.c
 	request.c