Improve balancer handling

include/lighttpd/actions.h

@@ -11,9 +11,8 @@ struct liActionRegexStackElement {
 };
 
 struct liActionStack {
-	GArray* stack;
-	GArray* regex_stack;
-	gboolean backend_failed;
+	GArray *stack, *regex_stack, *backend_stack;
+	gboolean backend_failed, backend_finished;
 	liBackendError backend_error;
 };
 

include/lighttpd/virtualrequest.h

@@ -128,6 +128,7 @@ LI_API void li_vrequest_error(liVRequest *vr);
 LI_API void li_vrequest_backend_overloaded(liVRequest *vr);
 LI_API void li_vrequest_backend_dead(liVRequest *vr);
 LI_API void li_vrequest_backend_error(liVRequest *vr, liBackendError berror);
+LI_API void li_vrequest_backend_finished(liVRequest *vr); /* action.c */
 
 /* resets fields which weren't reset in favor of keep-alive tracking */
 LI_API void li_vrequest_start(liVRequest *vr);

src/main/actions.c

@@ -169,30 +169,54 @@ static void action_stack_element_release(liServer *srv, liVRequest *vr, action_s
 }
 
 void li_action_stack_init(liActionStack *as) {
-	as->stack = g_array_sized_new(FALSE, TRUE, sizeof(action_stack_element), 15);
-	as->regex_stack = g_array_sized_new(FALSE, FALSE, sizeof(liActionRegexStackElement), 15);
-	g_array_set_size(as->regex_stack, 0);
+	as->stack = g_array_sized_new(FALSE, TRUE, sizeof(action_stack_element), 16);
+	as->regex_stack = g_array_sized_new(FALSE, FALSE, sizeof(liActionRegexStackElement), 16);
+	as->backend_stack = g_array_sized_new(FALSE, TRUE, sizeof(action_stack_element), 4);
+}
+
+static void li_action_backend_stack_reset(liVRequest *vr, liActionStack *as) {
+	liServer *srv = vr->wrk->srv;
+	guint i;
+
+	/* index 0 is the "deepest" backend - release it first */
+	for (i = 0; i < as->backend_stack->len; i++ ) {
+		action_stack_element_release(srv, vr, &g_array_index(as->backend_stack, action_stack_element, i));
+	}
+	g_array_set_size(as->backend_stack, 0);
 }
 
 void li_action_stack_reset(liVRequest *vr, liActionStack *as) {
 	liServer *srv = vr->wrk->srv;
 	guint i;
+
 	for (i = as->stack->len; i-- > 0; ) {
 		action_stack_element_release(srv, vr, &g_array_index(as->stack, action_stack_element, i));
 	}
 	g_array_set_size(as->stack, 0);
+
+	li_action_backend_stack_reset(vr, as);
+
 	as->backend_failed = FALSE;
+	as->backend_finished = FALSE;
 }
 
 void li_action_stack_clear(liVRequest *vr, liActionStack *as) {
 	liServer *srv = vr->wrk->srv;
 	guint i;
+
 	for (i = as->stack->len; i-- > 0; ) {
 		action_stack_element_release(srv, vr, &g_array_index(as->stack, action_stack_element, i));
 	}
 	g_array_free(as->stack, TRUE);
+
+	li_action_backend_stack_reset(vr, as);
+	g_array_free(as->backend_stack, TRUE);
+
 	g_array_free(as->regex_stack, TRUE);
-	as->stack = NULL;
+
+	as->stack = as->backend_stack = as->regex_stack = NULL;
+	as->backend_failed = FALSE;
+	as->backend_finished = FALSE;
 }
 
 static action_stack_element *action_stack_top(liActionStack* as) {
@@ -210,7 +234,19 @@ void li_action_enter(liVRequest *vr, liAction *a) {
 }
 
 static void action_stack_pop(liServer *srv, liVRequest *vr, liActionStack *as) {
-	action_stack_element_release(srv, vr, &g_array_index(as->stack, action_stack_element, as->stack->len - 1));
+	action_stack_element *ase;
+
+	if (as->stack->len == 0) return;
+
+	ase = &g_array_index(as->stack, action_stack_element, as->stack->len - 1);
+
+	if (ase->act->type == ACTION_TBALANCER && !as->backend_finished) {
+		/* release later if backend is finished (i.e. "disconnected") */
+		g_array_append_val(as->backend_stack, *ase);
+	} else {
+		action_stack_element_release(srv, vr, ase);
+	}
+
 	g_array_set_size(as->stack, as->stack->len - 1);
 }
 
@@ -224,6 +260,7 @@ liHandlerResult li_action_execute(liVRequest *vr) {
 
 	while (NULL != (ase = action_stack_top(as))) {
 		if (as->backend_failed) {
+			/* set by li_vrequest_backend_error */
 			vr->state = LI_VRS_HANDLE_REQUEST_HEADERS;
 			vr->backend = NULL;
 
@@ -325,6 +362,11 @@ liHandlerResult li_action_execute(liVRequest *vr) {
 			}
 			break;
 		case ACTION_TBALANCER:
+			/* skip balancer if request is already handled */
+			if (li_vrequest_is_handled(vr)) {
+				ase->finished = TRUE;
+				break;
+			}
 			res = a->data.balancer.select(vr, ase->backlog_provided, a->data.balancer.param, &ase->data.context);
 			switch (res) {
 			case LI_HANDLER_GO_ON:
@@ -345,3 +387,9 @@ liHandlerResult li_action_execute(liVRequest *vr) {
 	}
 	return LI_HANDLER_GO_ON;
 }
+
+void li_vrequest_backend_finished(liVRequest *vr) {
+	if (!li_vrequest_is_handled(vr)) return;
+	vr->action_stack.backend_finished = TRUE;
+	li_action_backend_stack_reset(vr, &vr->action_stack);
+}

src/modules/mod_balancer.c

@@ -114,6 +114,8 @@ static liHandlerResult balancer_act_select(liVRequest *vr, gboolean backlog_prov
 
 	/* TODO implement some selection algorithms */
 
+	VR_ERROR(vr, "balancer select: %i", be_ndx);
+
 	be->load++;
 	li_action_enter(vr, be->act);
 	*context = GINT_TO_POINTER(be_ndx);
@@ -124,12 +126,14 @@ static liHandlerResult balancer_act_select(liVRequest *vr, gboolean backlog_prov
 static liHandlerResult balancer_act_fallback(liVRequest *vr, gboolean backlog_provided, gpointer param, gpointer *context, liBackendError error) {
 	balancer *b = (balancer*) param;
 	gint be_ndx = GPOINTER_TO_INT(*context);
-	backend *be = &g_array_index(b->backends, backend, be_ndx);
+	backend *be;
 
 	UNUSED(backlog_provided);
-	UNUSED(error);
 
 	if (be_ndx < 0) return LI_HANDLER_GO_ON;
+	be = &g_array_index(b->backends, backend, be_ndx);
+
+	VR_ERROR(vr, "balancer fallback: %i (error: %i)", be_ndx, error);
 
 	/* TODO implement fallback/backlog */
 
@@ -142,11 +146,14 @@ static liHandlerResult balancer_act_fallback(liVRequest *vr, gboolean backlog_pr
 static liHandlerResult balancer_act_finished(liVRequest *vr, gpointer param, gpointer context) {
 	balancer *b = (balancer*) param;
 	gint be_ndx = GPOINTER_TO_INT(context);
-	backend *be = &g_array_index(b->backends, backend, be_ndx);
+	backend *be;
 
 	UNUSED(vr);
 
 	if (be_ndx < 0) return LI_HANDLER_GO_ON;
+	be = &g_array_index(b->backends, backend, be_ndx);
+
+	VR_ERROR(vr, "balancer finished: %i", be_ndx);
 
 	/* TODO implement backlog */
 
@@ -160,7 +167,6 @@ static void balancer_act_free(liServer *srv, gpointer param) {
 
 static liAction* balancer_rr(liServer *srv, liPlugin* p, liValue *val, gpointer userdata) {
 	balancer *b;
-	liAction *a;
 	UNUSED(p); UNUSED(userdata);
 
 	if (!val) {
@@ -174,8 +180,7 @@ static liAction* balancer_rr(liServer *srv, liPlugin* p, liValue *val, gpointer
 		return NULL;
 	}
 
-	a = li_action_new_balancer(balancer_act_select, balancer_act_fallback, balancer_act_finished, balancer_act_free, b, TRUE);
-	return a;
+	return li_action_new_balancer(balancer_act_select, balancer_act_fallback, balancer_act_finished, balancer_act_free, b, TRUE);
 }
 
 

src/modules/mod_fastcgi.c

@@ -187,6 +187,7 @@ static void fastcgi_connection_free(fastcgi_connection *fcon) {
 	ev_io_stop(vr->wrk->loop, &fcon->fd_watcher);
 	fastcgi_context_release(fcon->ctx);
 	if (fcon->fd != -1) close(fcon->fd);
+	li_vrequest_backend_finished(vr);
 
 	li_chunkqueue_free(fcon->fcgi_in);
 	li_chunkqueue_free(fcon->fcgi_out);
@@ -574,6 +575,7 @@ static void fastcgi_fd_cb(struct ev_loop *loop, ev_io *w, int revents) {
 				ev_io_stop(loop, w);
 				close(fcon->fd);
 				fcon->fd = -1;
+				li_vrequest_backend_finished(fcon->vr);
 				break;
 			case LI_NETWORK_STATUS_WAIT_FOR_EVENT:
 				break;
@@ -599,6 +601,7 @@ static void fastcgi_fd_cb(struct ev_loop *loop, ev_io *w, int revents) {
 				ev_io_stop(loop, w);
 				close(fcon->fd);
 				fcon->fd = -1;
+				li_vrequest_backend_finished(fcon->vr);
 				break;
 			case LI_NETWORK_STATUS_WAIT_FOR_EVENT:
 				break;

src/modules/mod_proxy.c

@@ -127,6 +127,7 @@ static void proxy_connection_free(proxy_connection *pcon) {
 	ev_io_stop(vr->wrk->loop, &pcon->fd_watcher);
 	proxy_context_release(pcon->ctx);
 	if (pcon->fd != -1) close(pcon->fd);
+	li_vrequest_backend_finished(vr);
 
 	li_chunkqueue_free(pcon->proxy_in);
 	li_chunkqueue_free(pcon->proxy_out);
@@ -251,6 +252,7 @@ static void proxy_fd_cb(struct ev_loop *loop, ev_io *w, int revents) {
 				ev_io_stop(loop, w);
 				close(pcon->fd);
 				pcon->fd = -1;
+				li_vrequest_backend_finished(pcon->vr);
 				break;
 			case LI_NETWORK_STATUS_WAIT_FOR_EVENT:
 				break;
@@ -276,6 +278,7 @@ static void proxy_fd_cb(struct ev_loop *loop, ev_io *w, int revents) {
 				ev_io_stop(loop, w);
 				close(pcon->fd);
 				pcon->fd = -1;
+				li_vrequest_backend_finished(pcon->vr);
 				break;
 			case LI_NETWORK_STATUS_WAIT_FOR_EVENT:
 				break;

src/modules/mod_scgi.c

@@ -122,6 +122,7 @@ static void scgi_connection_free(scgi_connection *scon) {
 	ev_io_stop(vr->wrk->loop, &scon->fd_watcher);
 	scgi_context_release(scon->ctx);
 	if (scon->fd != -1) close(scon->fd);
+	li_vrequest_backend_finished(vr);
 
 	li_chunkqueue_free(scon->scgi_in);
 	li_chunkqueue_free(scon->scgi_out);
@@ -340,6 +341,7 @@ static void scgi_fd_cb(struct ev_loop *loop, ev_io *w, int revents) {
 				ev_io_stop(loop, w);
 				close(scon->fd);
 				scon->fd = -1;
+				li_vrequest_backend_finished(scon->vr);
 				break;
 			case LI_NETWORK_STATUS_WAIT_FOR_EVENT:
 				break;
@@ -365,6 +367,7 @@ static void scgi_fd_cb(struct ev_loop *loop, ev_io *w, int revents) {
 				ev_io_stop(loop, w);
 				close(scon->fd);
 				scon->fd = -1;
+				li_vrequest_backend_finished(scon->vr);
 				break;
 			case LI_NETWORK_STATUS_WAIT_FOR_EVENT:
 				break;