diff --git a/src/gw_backend.c b/src/gw_backend.c index e3067aae..12837756 100644 --- a/src/gw_backend.c +++ b/src/gw_backend.c @@ -1289,6 +1289,15 @@ int gw_set_defaults_backend(server *srv, gw_plugin_data *p, const array *a, gw_p ,{ CONST_STR_LEN("tcp-fin-propagate"), T_CONFIG_BOOL, T_CONFIG_SCOPE_CONNECTION } + ,{ CONST_STR_LEN("connect-timeout"), + T_CONFIG_INT, + T_CONFIG_SCOPE_CONNECTION } + ,{ CONST_STR_LEN("write-timeout"), + T_CONFIG_INT, + T_CONFIG_SCOPE_CONNECTION } + ,{ CONST_STR_LEN("read-timeout"), + T_CONFIG_INT, + T_CONFIG_SCOPE_CONNECTION } ,{ NULL, 0, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET } @@ -1471,6 +1480,15 @@ int gw_set_defaults_backend(server *srv, gw_plugin_data *p, const array *a, gw_p case 22:/* tcp-fin-propagate */ host->tcp_fin_propagate = (0 != cpv->v.u); break; + case 23:/* connect-timeout */ + host->connect_timeout = cpv->v.u; + break; + case 24:/* write-timeout */ + host->write_timeout = cpv->v.u; + break; + case 25:/* read-timeout */ + host->read_timeout = cpv->v.u; + break; default: break; } @@ -1763,6 +1781,34 @@ void gw_set_transparent(gw_handler_ctx *hctx) { } +static void gw_host_hctx_enq(gw_handler_ctx * const hctx) { + gw_host * const host = hctx->host; + /*if (__builtin_expect( (host == NULL), 0)) return;*/ + + hctx->prev = NULL; + hctx->next = host->hctxs; + if (hctx->next) + hctx->next->prev = hctx; + host->hctxs = hctx; +} + + +static void gw_host_hctx_deq(gw_handler_ctx * const hctx) { + /*if (__builtin_expect( (hctx->host == NULL), 0)) return;*/ + + if (hctx->prev) + hctx->prev->next = hctx->next; + else + hctx->host->hctxs= hctx->next; + + if (hctx->next) + hctx->next->prev = hctx->prev; + + hctx->next = NULL; + hctx->prev = NULL; +} + + static void gw_backend_close(gw_handler_ctx * const hctx, request_st * const r) { if (hctx->fd >= 0) { fdevent_fdnode_event_del(hctx->ev, hctx->fdn); @@ -1770,6 +1816,7 @@ static void gw_backend_close(gw_handler_ctx * const hctx, request_st * const r) fdevent_sched_close(hctx->ev, hctx->fd, 1); hctx->fdn = NULL; hctx->fd = -1; + gw_host_hctx_deq(hctx); } if (hctx->host) { @@ -1885,6 +1932,7 @@ static handler_t gw_write_request(gw_handler_ctx * const hctx, request_st * cons } hctx->write_ts = log_monotonic_secs; + gw_host_hctx_enq(hctx); switch (gw_establish_connection(r, hctx->host, hctx->proc, hctx->pid, hctx->fd, hctx->conf.debug)) { case 1: /* connection is in progress */ @@ -2595,17 +2643,86 @@ handler_t gw_check_extension(request_st * const r, gw_plugin_data * const p, int return HANDLER_GO_ON; } +__attribute_cold__ +__attribute_noinline__ +static void gw_handle_trigger_hctx_timeout(gw_handler_ctx * const hctx, const char * const msg) { + + request_st * const r = hctx->r; + joblist_append(r->con); + + if (*msg == 'c') { /* "connect" */ + /* temporarily disable backend proc */ + gw_proc_connect_error(r, hctx->host, hctx->proc, hctx->pid, + ETIMEDOUT, hctx->conf.debug); + /* cleanup this request and let request handler start request again */ + /* retry only once since request already waited write_timeout secs */ + if (hctx->reconnects++ < 1) { + gw_reconnect(hctx, r); + return; + } + r->http_status = 503; /* Service Unavailable */ + } + else { /* "read" or "write" */ + /* blocked waiting to send (more) data to or to receive response + * (neither are a definite indication that the proc is no longer + * responsive on other socket connections; not marking proc overloaded) + * (If connect() to backend succeeded, then we began sending + * request and filled kernel socket buffers, so request is + * in progress and it is not safe or possible to retry) */ + /*if (hctx->conf.debug)*/ + log_error(r->conf.errh, __FILE__, __LINE__, + "%s timeout on socket: %s (fd: %d)", + msg, hctx->proc->connection_name->ptr, hctx->fd); + + if (*msg == 'w') { /* "write" */ + gw_write_error(hctx, r); /*(calls gw_backend_error())*/ + if (r->http_status == 503) r->http_status = 504; /*Gateway Timeout*/ + return; + } /* else "read" */ + } + gw_backend_error(hctx, r); + if (r->http_status == 500 && !r->resp_body_started && !r->handler_module) + r->http_status = 504; /*Gateway Timeout*/ +} + +__attribute_noinline__ +static void gw_handle_trigger_host_timeouts(gw_host * const host) { + + if (NULL == host->hctxs) return; + const unix_time64_t rsecs = (unix_time64_t)host->read_timeout; + const unix_time64_t wsecs = (unix_time64_t)host->write_timeout; + const unix_time64_t csecs = (unix_time64_t)host->connect_timeout; + if (!rsecs && !wsecs && !csecs) + return; /*(no timeout policy (default))*/ + + const unix_time64_t mono = log_monotonic_secs; /*(could have callers pass)*/ + for (gw_handler_ctx *hctx = host->hctxs, *next; hctx; hctx = next) { + /* if timeout occurs, hctx might be invalidated and removed from list, + * so next element must be store before checking for timeout */ + next = hctx->next; + + if (hctx->state == GW_STATE_CONNECT_DELAYED) { + if (mono - hctx->write_ts > csecs && csecs) /*(waiting for write)*/ + gw_handle_trigger_hctx_timeout(hctx, "connect"); + continue; /*(do not apply wsecs below to GW_STATE_CONNECT_DELAYED)*/ + } + + const int events = fdevent_fdnode_interest(hctx->fdn); + if ((events & FDEVENT_IN) && mono - hctx->read_ts > rsecs && rsecs) { + gw_handle_trigger_hctx_timeout(hctx, "read"); + continue; + } + if ((events & FDEVENT_OUT) && mono - hctx->write_ts > wsecs && wsecs) { + gw_handle_trigger_hctx_timeout(hctx, "write"); + continue; + } + } +} + static void gw_handle_trigger_host(gw_host * const host, log_error_st * const errh, const int debug) { - /* - * TODO: - * - * - add timeout for a connect to a non-gw process - * (use state_timestamp + state) - * - * perhaps we should kill a connect attempt after 10-15 seconds - * - * currently we wait for the TCP timeout which is 180 seconds on Linux - */ + + /* check for socket timeouts on active requests to backend host */ + gw_handle_trigger_host_timeouts(host); /* check each child proc to detect if proc exited */ @@ -2679,6 +2796,7 @@ static void gw_handle_trigger_exts_wkr(gw_exts *exts, log_error_st *errh) { gw_extension * const ex = exts->exts+j; for (uint32_t n = 0; n < ex->used; ++n) { gw_host * const host = ex->hosts[n]; + gw_handle_trigger_host_timeouts(host); for (gw_proc *proc = host->first; proc; proc = proc->next) { if (proc->state == PROC_STATE_OVERLOADED) gw_proc_check_enable(host, proc, errh); diff --git a/src/gw_backend.h b/src/gw_backend.h index a948fc88..dda7d11f 100644 --- a/src/gw_backend.h +++ b/src/gw_backend.h @@ -44,6 +44,8 @@ typedef struct gw_proc { unsigned short port; /* config.port + pno */ } gw_proc; +struct gw_handler_ctx; /* declaration */ + typedef struct { /* list of processes handling this extension * sorted by lowest load @@ -113,13 +115,18 @@ typedef struct { unsigned short disable_time; + unsigned short read_timeout; + unsigned short write_timeout; + unsigned short connect_timeout; + struct gw_handler_ctx *hctxs; + /* * some gw processes get a little bit larger * than wanted. max_requests_per_proc kills a * process after a number of handled requests. * */ - uint32_t max_requests_per_proc; + /*uint32_t max_requests_per_proc;*//* not implemented */ /* config */ @@ -327,6 +334,8 @@ typedef struct gw_handler_ctx { unix_time64_t write_ts; handler_t(*stdin_append)(struct gw_handler_ctx *hctx); handler_t(*create_env)(struct gw_handler_ctx *hctx); + struct gw_handler_ctx *prev; + struct gw_handler_ctx *next; void(*backend_error)(struct gw_handler_ctx *hctx); void(*handler_ctx_free)(void *hctx); } gw_handler_ctx;