[mod_openssl]: better io event handling (ssl may want to write while lighty only wants to read)

include/lighttpd/connection.h

@@ -80,4 +80,6 @@ LI_API gchar *li_connection_state_str(liConnectionState state);
 /* returns NULL if the vrequest doesn't belong to a liConnection* object */
 LI_API liConnection* li_connection_from_vrequest(liVRequest *vr);
 
+LI_API void connection_handle_io(liConnection *con);
+
 #endif

include/lighttpd/server.h

@@ -15,6 +15,7 @@ typedef gboolean (*liConnectionNewCB)(liConnection *con);
 typedef void (*liConnectionCloseCB)(liConnection *con);
 typedef liNetworkStatus (*liConnectionWriteCB)(liConnection *con, goffset write_max);
 typedef liNetworkStatus (*liConnectionReadCB)(liConnection *con);
+typedef void (*liServerSocketUpdateEventsCB)(liConnection *con, int events);
 typedef void (*liServerSocketReleaseCB)(liServerSocket *srv_sock);
 
 typedef void (*liServerStateWaitCancelled)(liServer *srv, liServerStateWait *w);
@@ -42,6 +43,7 @@ struct liServerSocket {
 	liConnectionNewCB new_cb;
 	liConnectionCloseCB close_cb;
 	liServerSocketReleaseCB release_cb;
+	liServerSocketUpdateEventsCB update_events_cb;
 };
 
 struct liServerStateWait {

src/main/connection.c

@@ -8,17 +8,25 @@ static void li_connection_internal_error(liConnection *con);
 static void update_io_events(liConnection *con) {
 	int events = 0;
 
-	if (!con->can_read && (con->state != LI_CON_STATE_HANDLE_MAINVR || con->mainvr->state >= LI_VRS_READ_CONTENT) && !con->in->is_closed) {
-		events = events | EV_READ;
-	}
+	if (LI_CON_STATE_KEEP_ALIVE == con->state) {
+		events = EV_READ;
+	} else {
+		if (!con->can_read && (con->state != LI_CON_STATE_HANDLE_MAINVR || con->mainvr->state >= LI_VRS_READ_CONTENT) && !con->in->is_closed) {
+			events = events | EV_READ;
+		}
 
-	if (!con->can_write && con->raw_out->length > 0) {
-		if (!con->mainvr->throttled || con->mainvr->throttle.magazine > 0) {
-			events = events | EV_WRITE;
+		if (!con->can_write && con->raw_out->length > 0) {
+			if (!con->mainvr->throttled || con->mainvr->throttle.magazine > 0) {
+				events = events | EV_WRITE;
+			}
 		}
 	}
 
-	li_ev_io_set_events(con->wrk->loop, &con->sock_watcher, events);
+	if (con->srv_sock->update_events_cb) {
+		con->srv_sock->update_events_cb(con, events);
+	} else {
+		li_ev_io_set_events(con->wrk->loop, &con->sock_watcher, events);
+	}
 }
 
 static void parse_request_body(liConnection *con) {
@@ -380,22 +388,28 @@ static gboolean connection_try_write(liConnection *con) {
 	return TRUE;
 }
 
-static void connection_cb(struct ev_loop *loop, ev_io *w, int revents) {
-	liConnection *con = (liConnection*) w->data;
-	UNUSED(loop);
-
+void connection_handle_io(liConnection *con) {
 	/* ensure that the connection is always in the io timeout queue */
 	if (!con->io_timeout_elem.queued)
 		li_waitqueue_push(&con->wrk->io_timeout_queue, &con->io_timeout_elem);
 
-	if (revents & EV_READ) con->can_read = TRUE;
-	if (revents & EV_WRITE) con->can_write = TRUE;
-
 	if (con->can_read)
 		if (!connection_try_read(con)) return;
 	if (con->can_write)
 		if (!connection_try_write(con)) return;
 
+	if (!check_response_done(con)) return;
+
+	update_io_events(con);
+}
+
+static void connection_cb(struct ev_loop *loop, ev_io *w, int revents) {
+	liConnection *con = (liConnection*) w->data;
+	UNUSED(loop);
+
+	if (revents & EV_READ) con->can_read = TRUE;
+	if (revents & EV_WRITE) con->can_write = TRUE;
+
 	if (revents & EV_ERROR) {
 		/* if this happens, we have a serious bug in the event handling */
 		VR_ERROR(con->mainvr, "%s", "EV_ERROR encountered, dropping connection!");
@@ -403,9 +417,7 @@ static void connection_cb(struct ev_loop *loop, ev_io *w, int revents) {
 		return;
 	}
 
-	if (!check_response_done(con)) return;
-
-	update_io_events(con);
+	connection_handle_io(con);
 }
 
 static void connection_keepalive_cb(struct ev_loop *loop, ev_timer *w, int revents) {
@@ -577,6 +589,7 @@ void li_connection_reset(liConnection *con) {
 		}
 	}
 	ev_io_set(&con->sock_watcher, -1, 0);
+	ev_set_cb(&con->sock_watcher, connection_cb);
 
 	li_chunkqueue_reset(con->raw_in);
 	li_chunkqueue_reset(con->raw_out);
@@ -665,7 +678,7 @@ static void li_connection_reset_keep_alive(liConnection *con) {
 	con->response_headers_sent = FALSE;
 	con->expect_100_cont = FALSE;
 
-	li_ev_io_set_events(con->wrk->loop, &con->sock_watcher, EV_READ);
+	update_io_events(con);
 	con->info.keep_alive = TRUE;
 
 	con->raw_out->is_closed = FALSE;

src/modules/mod_openssl.c

@@ -36,12 +36,43 @@ typedef struct openssl_context openssl_context;
 
 struct openssl_connection_ctx {
 	SSL *ssl;
+	liConnection *con;
+
+	int con_events;
+	liJob con_handle_events_job;
 };
 
 struct openssl_context {
 	SSL_CTX *ssl_ctx;
 };
 
+static void openssl_con_handle_events_cb(liJob *job) {
+	openssl_connection_ctx *conctx = LI_CONTAINER_OF(job, openssl_connection_ctx, con_handle_events_job);
+	liConnection *con = conctx->con;
+
+	connection_handle_io(con);
+}
+
+static void openssl_io_cb(struct ev_loop *loop, ev_io *w, int revents) {
+	liConnection *con = (liConnection*) w->data;
+	openssl_connection_ctx *conctx = con->srv_sock_data;
+
+	if (revents & EV_ERROR) {
+		/* if this happens, we have a serious bug in the event handling */
+		VR_ERROR(con->mainvr, "%s", "EV_ERROR encountered, dropping connection!");
+		li_connection_error(con);
+		return;
+	}
+
+	con->can_read = TRUE;
+	con->can_write = TRUE;
+
+	/* disable all events; they will get reactivated later */
+	li_ev_io_set_events(loop, w, 0);
+
+	li_job_now(&con->wrk->jobqueue, &conctx->con_handle_events_job);
+}
+
 static gboolean openssl_con_new(liConnection *con) {
 	liServer *srv = con->srv;
 	openssl_context *ctx = con->srv_sock->data;
@@ -50,6 +81,9 @@ static gboolean openssl_con_new(liConnection *con) {
 	con->srv_sock_data = NULL;
 
 	conctx->ssl = SSL_new(ctx->ssl_ctx);
+	conctx->con = con;
+	li_job_init(&conctx->con_handle_events_job, openssl_con_handle_events_cb);
+	conctx->con_events = 0;
 
 	if (NULL == conctx->ssl) {
 		ERROR(srv, "SSL_new: %s", ERR_error_string(ERR_get_error(), NULL));
@@ -66,6 +100,8 @@ static gboolean openssl_con_new(liConnection *con) {
 	con->srv_sock_data = conctx;
 	con->info.is_ssl = TRUE;
 
+	ev_set_cb(&con->sock_watcher, openssl_io_cb);
+
 	return TRUE;
 
 fail:
@@ -91,10 +127,22 @@ static void openssl_con_close(liConnection *con) {
 
 	con->srv_sock_data = NULL;
 	con->info.is_ssl = FALSE;
+	li_job_clear(&conctx->con_handle_events_job);
 
 	g_slice_free(openssl_connection_ctx, conctx);
 }
 
+static void openssl_update_events(liConnection *con, int events) {
+	openssl_connection_ctx *conctx = con->srv_sock_data;
+
+	/* new events -> add them to socket watcher too */
+	if (0 != (events & ~conctx->con_events)) {
+		li_ev_io_add_events(con->wrk->loop, &con->sock_watcher, events);
+	}
+
+	conctx->con_events = events;
+}
+
 static liNetworkStatus openssl_con_write(liConnection *con, goffset write_max) {
 	const ssize_t blocksize = 16*1024; /* 16k */
 	char *block_data;
@@ -134,7 +182,10 @@ static liNetworkStatus openssl_con_write(liConnection *con, goffset write_max) {
 
 			switch ((ssl_r = SSL_get_error(conctx->ssl, r))) {
 			case SSL_ERROR_WANT_READ:
+				li_ev_io_add_events(con->wrk->loop, &con->sock_watcher, EV_READ);
+				return LI_NETWORK_STATUS_WAIT_FOR_EVENT;
 			case SSL_ERROR_WANT_WRITE:
+				li_ev_io_add_events(con->wrk->loop, &con->sock_watcher, EV_WRITE);
 				return LI_NETWORK_STATUS_WAIT_FOR_EVENT;
 			case SSL_ERROR_SYSCALL:
 				/* perhaps we have error waiting in our error-queue */
@@ -242,9 +293,17 @@ static liNetworkStatus openssl_con_read(liConnection *con) {
 
 			err = SSL_get_error(conctx->ssl, r);
 
-			if (SSL_ERROR_WANT_READ == err || SSL_ERROR_WANT_WRITE == err) {
+			switch (err) {
+			case SSL_ERROR_WANT_READ:
+				li_ev_io_add_events(con->wrk->loop, &con->sock_watcher, EV_READ);
 				/* ignore requirement that we should pass the same buffer again */
 				return (len > 0) ? LI_NETWORK_STATUS_SUCCESS : LI_NETWORK_STATUS_WAIT_FOR_EVENT;
+			case SSL_ERROR_WANT_WRITE:
+				li_ev_io_add_events(con->wrk->loop, &con->sock_watcher, EV_WRITE);
+				/* ignore requirement that we should pass the same buffer again */
+				return (len > 0) ? LI_NETWORK_STATUS_SUCCESS : LI_NETWORK_STATUS_WAIT_FOR_EVENT;
+			default:
+				break;
 			}
 
 			switch (err) {
@@ -361,6 +420,7 @@ static void openssl_setup_listen_cb(liServer *srv, int fd, gpointer data) {
 	srv_sock->new_cb = openssl_con_new;
 	srv_sock->close_cb = openssl_con_close;
 	srv_sock->release_cb = openssl_sock_release;
+	srv_sock->update_events_cb = openssl_update_events;
 }
 
 static gboolean openssl_setup(liServer *srv, liPlugin* p, liValue *val, gpointer userdata) {