diff --git a/src/Makefile.am b/src/Makefile.am index ae8761c4..80153d6f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -78,7 +78,7 @@ lib_LTLIBRARIES += mod_cml.la mod_cml_la_SOURCES = mod_cml.c mod_cml_lua.c mod_cml_funcs.c mod_cml_la_CFLAGS = $(AM_CFLAGS) $(LUA_CFLAGS) mod_cml_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined -mod_cml_la_LIBADD = $(MEMCACHE_LIB) $(common_libadd) $(LUA_LIBS) +mod_cml_la_LIBADD = $(MEMCACHE_LIB) $(common_libadd) $(LUA_LIBS) -lm lib_LTLIBRARIES += mod_trigger_b4_dl.la mod_trigger_b4_dl_la_SOURCES = mod_trigger_b4_dl.c diff --git a/src/base.h b/src/base.h index d725f828..40fcc950 100644 --- a/src/base.h +++ b/src/base.h @@ -157,7 +157,6 @@ typedef struct { array *headers; /* CONTENT */ - buffer *content; size_t content_length; /* returned by strtoul() */ /* internal representation */ @@ -330,8 +329,9 @@ typedef struct { int file_started; int file_finished; - chunkqueue *write_queue; - chunkqueue *read_queue; + chunkqueue *write_queue; /* a large queue for low-level write ( HTTP response ) [ file, mem ] */ + chunkqueue *read_queue; /* a small queue for low-level read ( HTTP request ) [ mem ] */ + chunkqueue *request_content_queue; /* takes request-content into tempfile if necessary [ tempfile, mem ]*/ int traffic_limit_reached; diff --git a/src/connections.c b/src/connections.c index 904582bf..590f0087 100644 --- a/src/connections.c +++ b/src/connections.c @@ -193,10 +193,11 @@ static void dump_packet(const unsigned char *data, size_t len) { static int connection_handle_read(server *srv, connection *con) { int len; buffer *b; + int toread; #ifdef USE_OPENSSL server_socket *srv_sock = con->srv_socket; #endif - + b = chunkqueue_get_append_buffer(con->read_queue); buffer_prepare_copy(b, 4096); @@ -204,11 +205,27 @@ static int connection_handle_read(server *srv, connection *con) { if (srv_sock->is_ssl) { len = SSL_read(con->ssl, b->ptr, b->size - 1); } else { + if (ioctl(con->fd, FIONREAD, &toread)) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "unexpected end-of-file:", + con->fd); + return -1; + } + buffer_prepare_copy(b, toread); + len = read(con->fd, b->ptr, b->size - 1); } #elif defined(__WIN32) len = recv(con->fd, b->ptr, b->size - 1, 0); #else + if (ioctl(con->fd, FIONREAD, &toread)) { + log_error_write(srv, __FILE__, __LINE__, "sd", + "unexpected end-of-file:", + con->fd); + return -1; + } + buffer_prepare_copy(b, toread); + len = read(con->fd, b->ptr, b->size - 1); #endif @@ -538,7 +555,6 @@ connection *connection_init(server *srv) { CLEAN(request.request_line); CLEAN(request.request); CLEAN(request.pathinfo); - CLEAN(request.content); CLEAN(request.orig_uri); @@ -563,6 +579,7 @@ connection *connection_init(server *srv) { #undef CLEAN con->write_queue = chunkqueue_init(); con->read_queue = chunkqueue_init(); + con->request_content_queue = chunkqueue_init(); con->request.headers = array_init(); con->response.headers = array_init(); con->environment = array_init(); @@ -588,6 +605,7 @@ void connections_free(server *srv) { chunkqueue_free(con->write_queue); chunkqueue_free(con->read_queue); + chunkqueue_free(con->request_content_queue); array_free(con->request.headers); array_free(con->response.headers); array_free(con->environment); @@ -599,7 +617,6 @@ void connections_free(server *srv) { CLEAN(request.request_line); CLEAN(request.request); CLEAN(request.pathinfo); - CLEAN(request.content); CLEAN(request.orig_uri); @@ -669,7 +686,6 @@ int connection_reset(server *srv, connection *con) { CLEAN(request.uri); CLEAN(request.request_line); CLEAN(request.pathinfo); - CLEAN(request.content); CLEAN(request.request); CLEAN(request.orig_uri); @@ -712,6 +728,7 @@ int connection_reset(server *srv, connection *con) { array_reset(con->environment); chunkqueue_reset(con->write_queue); + chunkqueue_reset(con->request_content_queue); /* the plugins should cleanup themself */ for (i = 0; i < srv->plugins.used; i++) { @@ -791,6 +808,8 @@ int connection_handle_read_state(server *srv, connection *con) { char *h_term = NULL; chunk *c; chunkqueue *cq = con->read_queue; + chunkqueue *dst_cq = con->request_content_queue; + size_t memusage; if (con->is_readable) { con->read_idle_ts = srv->cur_ts; @@ -905,66 +924,86 @@ int connection_handle_read_state(server *srv, connection *con) { c->offset = c->mem->used - 1; } } - - if (c->offset + 1 == c->mem->used) { - /* chunk is empty, move it to unused */ - cq->first = c->next; - c->next = cq->unused; - cq->unused = c; - - if (cq->first == NULL) cq->last = NULL; - - assert(c != c->next); - } - + /* con->request.request is setup up */ if (h_term) { connection_set_state(srv, con, CON_STATE_REQUEST_END); - } else if (chunkqueue_length(cq) > 64 * 1024) { + } else if (con->request.request->used > 64 * 1024) { log_error_write(srv, __FILE__, __LINE__, "sd", "http-header larger then 64k -> disconnected", chunkqueue_length(cq)); connection_set_state(srv, con, CON_STATE_ERROR); } break; case CON_STATE_READ_POST: - for (c = cq->first; c && (con->request.content->used != con->request.content_length + 1); c = cq->first) { + for (c = cq->first; c && (dst_cq->bytes_in != con->request.content_length); c = c->next) { off_t weWant, weHave, toRead; + int buffer_to_file = 0; - weWant = con->request.content_length - (con->request.content->used ? con->request.content->used - 1 : 0); - /* without the terminating \0 */ + weWant = con->request.content_length - dst_cq->bytes_in; assert(c->mem->used); weHave = c->mem->used - c->offset - 1; toRead = weHave > weWant ? weWant : weHave; - - buffer_append_string_len(con->request.content, c->mem->ptr + c->offset, toRead); - - c->offset += toRead; - - if (c->offset + 1 >= c->mem->used) { - /* chunk is empty, move it to unused */ - - cq->first = c->next; - c->next = cq->unused; - cq->unused = c; - - if (cq->first == NULL) cq->last = NULL; - - assert(c != c->next); + + /* the new way, copy everything into a chunkqueue whcih might use tempfiles */ + if (con->request.content_length > 64 * 1024) { + chunk *dst_c = NULL; + /* copy everything to max 1Mb sized tempfiles */ + + /* + * if the last chunk is + * - smaller than 1Mb (size < 1Mb) + * - not read yet (offset == 0) + * -> append to it + * otherwise + * -> create a new chunk + * + * */ + + if (dst_cq->last && + dst_cq->last->type == FILE_CHUNK && + dst_cq->last->file.is_temp && + dst_cq->last->offset == 0 && + dst_cq->last->file.length < 1 * 1024 * 1024) { + /* ok, take the last chunk for our job */ + dst_c = dst_cq->last; + dst_c->file.fd = open(dst_c->file.name->ptr, O_WRONLY | O_APPEND); + } else { + dst_c = chunkqueue_get_append_tempfile(dst_cq); + } + + /* we have a chunk, let's write to it */ + + assert(dst_c->file.fd != -1); + + assert(toRead == write(dst_c->file.fd, c->mem->ptr + c->offset, toRead)); + + dst_c->file.length += toRead; + + close(dst_c->file.fd); + dst_c->file.fd = -1; } else { - assert(toRead); + buffer *b; + + b = chunkqueue_get_append_buffer(dst_cq); + buffer_copy_string_len(b, c->mem->ptr + c->offset, toRead); } + + c->offset += toRead; + dst_cq->bytes_in += toRead; } - + /* Content is ready */ - if (con->request.content->used == con->request.content_length + 1) { + if (dst_cq->bytes_in == con->request.content_length) { connection_set_state(srv, con, CON_STATE_HANDLE_REQUEST); } break; } - + + chunkqueue_remove_finished_chunks(cq); + return 0; } diff --git a/src/mod_cgi.c b/src/mod_cgi.c index 368c1311..9804a5d9 100644 --- a/src/mod_cgi.c +++ b/src/mod_cgi.c @@ -4,6 +4,8 @@ #else #include #include +#include +#include #include @@ -956,8 +958,78 @@ static int cgi_create_env(server *srv, connection *con, plugin_data *p, buffer * handler_ctx *hctx; /* father */ - if (con->request.content->used) { - write(to_cgi_fds[1], con->request.content->ptr, con->request.content_length); + if (con->request.content_length) { + chunkqueue *cq = con->request_content_queue; + chunk *c; + + assert(chunkqueue_length(cq) == con->request.content_length); + + /* there is content to send */ + for (c = cq->first; c; c = cq->first) { + int r = 0; + + /* copy all chunks */ + switch(c->type) { + case FILE_CHUNK: + + if (c->file.mmap.start == MAP_FAILED) { + if (-1 == c->file.fd && /* open the file if not already open */ + -1 == (c->file.fd = open(c->file.name->ptr, O_RDONLY))) { + log_error_write(srv, __FILE__, __LINE__, "ss", "open failed: ", strerror(errno)); + + return -1; + } + + c->file.mmap.length = c->file.length; + + if (MAP_FAILED == (c->file.mmap.start = mmap(0, c->file.mmap.length, PROT_READ, MAP_SHARED, c->file.fd, 0))) { + log_error_write(srv, __FILE__, __LINE__, "ssbd", "mmap failed: ", + strerror(errno), c->file.name, c->file.fd); + + return -1; + } + + close(c->file.fd); + c->file.fd = -1; + + /* chunk_reset() or chunk_free() will cleanup for us */ + } + + if ((r = write(to_cgi_fds[1], c->file.mmap.start + c->offset, c->file.length - c->offset)) < 0) { + switch(errno) { + case ENOSPC: + con->http_status = 507; + + break; + default: + con->http_status = 403; + break; + } + } + break; + case MEM_CHUNK: + if ((r = write(to_cgi_fds[1], c->mem->ptr + c->offset, c->mem->used - c->offset - 1)) < 0) { + switch(errno) { + case ENOSPC: + con->http_status = 507; + + break; + default: + con->http_status = 403; + break; + } + } + break; + } + + if (r > 0) { + c->offset += r; + cq->bytes_out += r; + } else { + break; + } + chunkqueue_remove_finished_chunks(cq); + } } close(from_cgi_fds[1]); diff --git a/src/mod_fastcgi.c b/src/mod_fastcgi.c index dd23340e..78cfff84 100644 --- a/src/mod_fastcgi.c +++ b/src/mod_fastcgi.c @@ -23,6 +23,7 @@ #include "inet_ntop_cache.h" #include "stat_cache.h" +#include "network_backends.h" #include #include @@ -318,10 +319,8 @@ typedef struct { int reconnects; /* number of reconnect attempts */ - buffer *write_buffer; - size_t write_offset; - - chunkqueue *rb; + chunkqueue *rb; /* read queue */ + chunkqueue *wb; /* write queue */ buffer *response_header; @@ -359,7 +358,6 @@ static handler_ctx * handler_ctx_init() { hctx->fde_ndx = -1; hctx->response_header = buffer_init(); - hctx->write_buffer = buffer_init(); hctx->request_id = 0; hctx->state = FCGI_STATE_INIT; @@ -371,15 +369,16 @@ static handler_ctx * handler_ctx_init() { hctx->send_content_body = 1; hctx->rb = chunkqueue_init(); + hctx->wb = chunkqueue_init(); return hctx; } static void handler_ctx_free(handler_ctx *hctx) { buffer_free(hctx->response_header); - buffer_free(hctx->write_buffer); chunkqueue_free(hctx->rb); + chunkqueue_free(hctx->wb); free(hctx); } @@ -1618,9 +1617,9 @@ static int fcgi_env_add_request_headers(server *srv, connection *con, plugin_dat static int fcgi_create_env(server *srv, handler_ctx *hctx, size_t request_id) { FCGI_BeginRequestRecord beginRecord; FCGI_Header header; + buffer *b; char buf[32]; - size_t offset; const char *s; #ifdef HAVE_IPV6 char b2[INET6_ADDRSTRLEN + 1]; @@ -1642,8 +1641,10 @@ static int fcgi_create_env(server *srv, handler_ctx *hctx, size_t request_id) { beginRecord.body.roleB1 = 0; beginRecord.body.flags = 0; memset(beginRecord.body.reserved, 0, sizeof(beginRecord.body.reserved)); + + b = chunkqueue_get_append_buffer(hctx->wb); - buffer_copy_memory(hctx->write_buffer, (const char *)&beginRecord, sizeof(beginRecord)); + buffer_copy_memory(b, (const char *)&beginRecord, sizeof(beginRecord)); /* send FCGI_PARAMS */ buffer_prepare_copy(p->fcgi_env, 1024); @@ -1800,29 +1801,139 @@ static int fcgi_create_env(server *srv, handler_ctx *hctx, size_t request_id) { fcgi_env_add_request_headers(srv, con, p); fcgi_header(&(header), FCGI_PARAMS, request_id, p->fcgi_env->used, 0); - buffer_append_memory(hctx->write_buffer, (const char *)&header, sizeof(header)); - buffer_append_memory(hctx->write_buffer, (const char *)p->fcgi_env->ptr, p->fcgi_env->used); + buffer_append_memory(b, (const char *)&header, sizeof(header)); + buffer_append_memory(b, (const char *)p->fcgi_env->ptr, p->fcgi_env->used); fcgi_header(&(header), FCGI_PARAMS, request_id, 0, 0); - buffer_append_memory(hctx->write_buffer, (const char *)&header, sizeof(header)); - - /* send FCGI_STDIN */ - - /* something to send ? */ - for (offset = 0; offset != con->request.content_length; ) { - /* send chunks of 1024 bytes */ - size_t toWrite = con->request.content_length - offset > 4096 ? 4096 : con->request.content_length - offset; - - fcgi_header(&(header), FCGI_STDIN, request_id, toWrite, 0); - buffer_append_memory(hctx->write_buffer, (const char *)&header, sizeof(header)); - buffer_append_memory(hctx->write_buffer, (const char *)(con->request.content->ptr + offset), toWrite); - - offset += toWrite; + buffer_append_memory(b, (const char *)&header, sizeof(header)); + + b->used++; /* add virtual \0 */ + hctx->wb->bytes_in += b->used - 1; + + if (con->request.content_length) { + chunkqueue *req_cq = con->request_content_queue; + chunk *req_c; + size_t offset; + + /* something to send ? */ + for (offset = 0, req_c = req_cq->first; offset != req_cq->bytes_in; ) { + size_t weWant = req_cq->bytes_in - offset > FCGI_MAX_LENGTH ? FCGI_MAX_LENGTH : req_cq->bytes_in - offset; + size_t written = 0; + size_t weHave = 0; + + /* we announce toWrite octects + * now take all the request_content chunk that we need to fill this request + * */ + + b = chunkqueue_get_append_buffer(hctx->wb); + fcgi_header(&(header), FCGI_STDIN, request_id, weWant, 0); + buffer_copy_memory(b, (const char *)&header, sizeof(header)); + hctx->wb->bytes_in += sizeof(header); + + if (p->conf.debug > 10) { + fprintf(stderr, "%s.%d: tosend: %d / %Ld\n", __FILE__, __LINE__, offset, req_cq->bytes_in); + } + + for (written = 0; written != weWant; ) { + if (p->conf.debug > 10) { + fprintf(stderr, "%s.%d: chunk: %d / %d\n", __FILE__, __LINE__, written, weWant); + } + + switch (req_c->type) { + case FILE_CHUNK: + weHave = req_c->file.length - req_c->offset; + + if (weHave > weWant - written) weHave = weWant - written; + + if (p->conf.debug > 10) { + fprintf(stderr, "%s.%d: sending %d bytes from (%Ld / %Ld) %s\n", + __FILE__, __LINE__, + weHave, + req_c->offset, + req_c->file.length, + req_c->file.name->ptr); + } + + assert(weHave != 0); + + chunkqueue_append_file(hctx->wb, req_c->file.name, req_c->offset, weHave); + + req_c->offset += weHave; + req_cq->bytes_out += weHave; + written += weHave; + + hctx->wb->bytes_in += weHave; + + /* steal the tempfile + * + * This is tricky: + * - we reference the tempfile from the request-content-queue several times + * if the req_c is larger than FCGI_MAX_LENGTH + * - we can't simply cleanup the request-content-queue as soon as possible + * as it would remove the tempfiles + * - the idea is to 'steal' the tempfiles and attach the is_temp flag to the last + * referencing chunk of the fastcgi-write-queue + * + * */ + + if (req_c->offset == req_c->file.length) { + chunk *c; + + if (p->conf.debug > 10) { + fprintf(stderr, "%s.%d: next chunk\n", __FILE__, __LINE__); + } + c = hctx->wb->last; + + assert(c->type == FILE_CHUNK); + assert(req_c->file.is_temp == 1); + + c->file.is_temp = 1; + req_c->file.is_temp = 0; + + chunkqueue_remove_finished_chunks(req_cq); + + req_c = req_cq->first; + } + + break; + case MEM_CHUNK: + /* append to the buffer */ + weHave = req_c->mem->used - 1 - req_c->offset; + + if (weHave > weWant - written) weHave = weWant - written; + + buffer_append_memory(b, req_c->mem->ptr + req_c->offset, weHave); + + req_c->offset += weHave; + req_cq->bytes_out += weHave; + written += weHave; + + hctx->wb->bytes_in += weHave; + + if (req_c->offset == req_c->mem->used - 1) { + chunkqueue_remove_finished_chunks(req_cq); + + req_c = req_cq->first; + } + + break; + default: + break; + } + } + + b->used++; /* add virtual \0 */ + offset += weWant; + } } + b = chunkqueue_get_append_buffer(hctx->wb); /* terminate STDIN */ fcgi_header(&(header), FCGI_STDIN, request_id, 0, 0); - buffer_append_memory(hctx->write_buffer, (const char *)&header, sizeof(header)); + buffer_copy_memory(b, (const char *)&header, sizeof(header)); + b->used++; /* add virtual \0 */ + + hctx->wb->bytes_in += sizeof(header); #if 0 for (i = 0; i < hctx->write_buffer->used; i++) { @@ -2449,7 +2560,7 @@ static handler_t fcgi_write_request(server *srv, handler_ctx *hctx) { fcgi_extension_host *host= hctx->host; connection *con = hctx->remote_conn; - int r; + int ret; /* sanity check */ if (!host || @@ -2466,9 +2577,9 @@ static handler_t fcgi_write_request(server *srv, handler_ctx *hctx) { switch(hctx->state) { case FCGI_STATE_INIT: - r = host->unixsocket->used ? AF_UNIX : AF_INET; + ret = host->unixsocket->used ? AF_UNIX : AF_INET; - if (-1 == (hctx->fd = socket(r, SOCK_STREAM, 0))) { + if (-1 == (hctx->fd = socket(ret, SOCK_STREAM, 0))) { if (errno == EMFILE || errno == EINTR) { log_error_write(srv, __FILE__, __LINE__, "sd", @@ -2588,25 +2699,34 @@ static handler_t fcgi_write_request(server *srv, handler_ctx *hctx) { fcgi_create_env(srv, hctx, hctx->request_id); fcgi_set_state(srv, hctx, FCGI_STATE_WRITE); - hctx->write_offset = 0; /* fall through */ case FCGI_STATE_WRITE: - /* why aren't we using the network_ interface here ? */ - - r = write(hctx->fd, - hctx->write_buffer->ptr + hctx->write_offset, - hctx->write_buffer->used - hctx->write_offset); + +#if defined USE_LINUX_SENDFILE + ret = network_write_chunkqueue_linuxsendfile(srv, con, hctx->fd, hctx->wb); +#elif defined USE_FREEBSD_SENDFILE + ret = network_write_chunkqueue_freebsdsendfile(srv, con, hctx->fd, hctx->wb); +#elif defined USE_SOLARIS_SENDFILEV + ret = network_write_chunkqueue_solarissendfilev(srv, con, hctx->fd, hctx->wb); +#elif defined USE_WRITEV + ret = network_write_chunkqueue_writev(srv, con, hctx->fd, hctx->wb); +#else + ret = network_write_chunkqueue_write(srv, con, hctx->fd, hctx->wb); +#endif + + chunkqueue_remove_finished_chunks(hctx->wb); - if (-1 == r) { - if (errno == ENOTCONN) { + if (ret < 0) { + switch(errno) { + case ENOTCONN: /* the connection got dropped after accept() * * this is most of the time a PHP which dies * after PHP_FCGI_MAX_REQUESTS * */ - if (hctx->write_offset == 0 && + if (hctx->wb->bytes_out == 0 && hctx->reconnects < 5) { usleep(10000); /* take away the load of the webserver * to let the php a chance to restart @@ -2625,35 +2745,34 @@ static handler_t fcgi_write_request(server *srv, handler_ctx *hctx) { log_error_write(srv, __FILE__, __LINE__, "ssdsd", "[REPORT ME] connection was dropped after accept(). reconnect() denied:", - "write-offset:", hctx->write_offset, + "write-offset:", hctx->wb->bytes_out, "reconnect attempts:", hctx->reconnects); return HANDLER_ERROR; - } - - if ((errno != EAGAIN) && - (errno != EINTR)) { + case EAGAIN: + case EINTR: + fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT); + return HANDLER_WAIT_FOR_EVENT; + default: log_error_write(srv, __FILE__, __LINE__, "ssd", "write failed:", strerror(errno), errno); return HANDLER_ERROR; - } else { - fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT); - - return HANDLER_WAIT_FOR_EVENT; } } - - hctx->write_offset += r; - - if (hctx->write_offset == hctx->write_buffer->used) { + + if (hctx->wb->bytes_out == hctx->wb->bytes_in) { /* we don't need the out event anymore */ fdevent_event_del(srv->ev, &(hctx->fde_ndx), hctx->fd); fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_IN); fcgi_set_state(srv, hctx, FCGI_STATE_READ); + } else { + fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT); + + return HANDLER_WAIT_FOR_EVENT; } - + break; case FCGI_STATE_READ: /* waiting for a response */ @@ -2735,7 +2854,7 @@ SUBREQUEST_FUNC(mod_fastcgi_handle_subrequest) { buffer_reset(con->physical.path); con->mode = DIRECT; - joblist_append(srv, con); + joblist_append(srv, con); /* really ? */ /* mis-using HANDLER_WAIT_FOR_FD to break out of the loop * and hope that the childs will be restarted @@ -2758,6 +2877,7 @@ SUBREQUEST_FUNC(mod_fastcgi_handle_subrequest) { buffer_reset(con->physical.path); con->mode = DIRECT; con->http_status = 503; + joblist_append(srv, con); /* really ? */ return HANDLER_FINISHED; } @@ -2865,11 +2985,11 @@ static handler_t fcgi_handle_fdevent(void *s, void *ctx, int revents) { if (con->file_started == 0) { /* nothing has been send out yet, try to use another child */ - if (hctx->write_offset == 0 && + if (hctx->wb->bytes_out == 0 && hctx->reconnects < 5) { fcgi_reconnect(srv, hctx); - log_error_write(srv, __FILE__, __LINE__, "sdsdsd", + log_error_write(srv, __FILE__, __LINE__, "ssdsd", "response not sent, request not sent, reconnection.", "connection-fd:", con->fd, "fcgi-fd:", hctx->fd); @@ -2877,8 +2997,8 @@ static handler_t fcgi_handle_fdevent(void *s, void *ctx, int revents) { return HANDLER_WAIT_FOR_FD; } - log_error_write(srv, __FILE__, __LINE__, "sdsdsd", - "response not sent, request sent:", hctx->write_offset, + log_error_write(srv, __FILE__, __LINE__, "sosdsd", + "response not sent, request sent:", hctx->wb->bytes_out, "connection-fd:", con->fd, "fcgi-fd:", hctx->fd); diff --git a/src/mod_proxy.c b/src/mod_proxy.c index 6cb4340f..f22e140d 100644 --- a/src/mod_proxy.c +++ b/src/mod_proxy.c @@ -23,6 +23,7 @@ #include "inet_ntop_cache.h" #include "crc32.h" +#include "network_backends.h" #include @@ -98,11 +99,9 @@ typedef struct { buffer *response; buffer *response_header; - - buffer *write_buffer; - size_t write_offset; - + chunkqueue *wb; + int fd; /* fd to the proxy process */ int fde_ndx; /* index into the fd-event buffer */ @@ -128,7 +127,7 @@ static handler_ctx * handler_ctx_init() { hctx->response = buffer_init(); hctx->response_header = buffer_init(); - hctx->write_buffer = buffer_init(); + hctx->wb = chunkqueue_init(); hctx->fd = -1; hctx->fde_ndx = -1; @@ -139,7 +138,7 @@ static handler_ctx * handler_ctx_init() { static void handler_ctx_free(handler_ctx *hctx) { buffer_free(hctx->response); buffer_free(hctx->response_header); - buffer_free(hctx->write_buffer); + chunkqueue_free(hctx->wb); free(hctx); } @@ -402,18 +401,19 @@ static int proxy_create_env(server *srv, handler_ctx *hctx) { size_t i; connection *con = hctx->remote_conn; + buffer *b; UNUSED(srv); /* build header */ - - buffer_reset(hctx->write_buffer); + + b = chunkqueue_get_append_buffer(hctx->wb); /* request line */ - buffer_copy_string(hctx->write_buffer, get_http_method_name(con->request.http_method)); - BUFFER_APPEND_STRING_CONST(hctx->write_buffer, " "); + buffer_copy_string(b, get_http_method_name(con->request.http_method)); + BUFFER_APPEND_STRING_CONST(b, " "); - buffer_append_string_buffer(hctx->write_buffer, con->request.uri); - BUFFER_APPEND_STRING_CONST(hctx->write_buffer, " HTTP/1.0\r\n"); + buffer_append_string_buffer(b, con->request.uri); + BUFFER_APPEND_STRING_CONST(b, " HTTP/1.0\r\n"); /* request header */ for (i = 0; i < con->request.headers->used; i++) { @@ -424,25 +424,73 @@ static int proxy_create_env(server *srv, handler_ctx *hctx) { if (ds->value->used && ds->key->used) { if (0 == strcmp(ds->key->ptr, "Connection")) continue; - buffer_append_string_buffer(hctx->write_buffer, ds->key); - BUFFER_APPEND_STRING_CONST(hctx->write_buffer, ": "); - buffer_append_string_buffer(hctx->write_buffer, ds->value); - BUFFER_APPEND_STRING_CONST(hctx->write_buffer, "\r\n"); + buffer_append_string_buffer(b, ds->key); + BUFFER_APPEND_STRING_CONST(b, ": "); + buffer_append_string_buffer(b, ds->value); + BUFFER_APPEND_STRING_CONST(b, "\r\n"); } } - BUFFER_APPEND_STRING_CONST(hctx->write_buffer, "X-Forwarded-For: "); - buffer_append_string(hctx->write_buffer, inet_ntop_cache_get_ip(srv, &(con->dst_addr))); - BUFFER_APPEND_STRING_CONST(hctx->write_buffer, "\r\n"); + BUFFER_APPEND_STRING_CONST(b, "X-Forwarded-For: "); + buffer_append_string(b, inet_ntop_cache_get_ip(srv, &(con->dst_addr))); + BUFFER_APPEND_STRING_CONST(b, "\r\n"); - BUFFER_APPEND_STRING_CONST(hctx->write_buffer, "\r\n"); + BUFFER_APPEND_STRING_CONST(b, "\r\n"); + hctx->wb->bytes_in += b->used - 1; /* body */ if (con->request.content_length) { - /* the buffer-string functions add an extra \0 at the end the memory-function don't */ - hctx->write_buffer->used--; - buffer_append_memory(hctx->write_buffer, con->request.content->ptr, con->request.content_length); + chunkqueue *req_cq = con->request_content_queue; + chunk *req_c; + size_t offset; + + /* something to send ? */ + for (offset = 0, req_c = req_cq->first; offset != req_cq->bytes_in; req_c = req_c->next) { + size_t weWant = req_cq->bytes_in - offset; + size_t weHave = 0; + + /* we announce toWrite octects + * now take all the request_content chunk that we need to fill this request + * */ + + switch (req_c->type) { + case FILE_CHUNK: + weHave = req_c->file.length - req_c->offset; + + if (weHave > weWant) weHave = weWant; + + chunkqueue_append_file(hctx->wb, req_c->file.name, req_c->offset, weHave); + + req_c->offset += weHave; + req_cq->bytes_out += weHave; + + hctx->wb->bytes_in += weHave; + + break; + case MEM_CHUNK: + /* append to the buffer */ + weHave = req_c->mem->used - 1 - req_c->offset; + + if (weHave > weWant) weHave = weWant; + + b = chunkqueue_get_append_buffer(hctx->wb); + buffer_append_memory(b, req_c->mem->ptr + req_c->offset, weHave); + b->used++; /* add virtual \0 */ + + req_c->offset += weHave; + req_cq->bytes_out += weHave; + + hctx->wb->bytes_in += weHave; + + break; + default: + break; + } + + offset += weHave; + } + } return 0; @@ -652,17 +700,16 @@ static int proxy_demux_response(server *srv, handler_ctx *hctx) { static handler_t proxy_write_request(server *srv, handler_ctx *hctx) { data_proxy *host= hctx->host; plugin_data *p = hctx->plugin_data; + connection *con = hctx->remote_conn; - int r; + int ret; if (!host || (!host->host->used || !host->port)) return -1; switch(hctx->state) { case PROXY_STATE_INIT: - r = AF_INET; - - if (-1 == (hctx->fd = socket(r, SOCK_STREAM, 0))) { + if (-1 == (hctx->fd = socket(AF_INET, SOCK_STREAM, 0))) { log_error_write(srv, __FILE__, __LINE__, "ss", "socket failed: ", strerror(errno)); return HANDLER_ERROR; } @@ -734,33 +781,45 @@ static handler_t proxy_write_request(server *srv, handler_ctx *hctx) { proxy_create_env(srv, hctx); proxy_set_state(srv, hctx, PROXY_STATE_WRITE); - hctx->write_offset = 0; - fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT); - /* fall through */ - case PROXY_STATE_WRITE: - /* continue with the code after the switch */ - if (-1 == (r = write(hctx->fd, - hctx->write_buffer->ptr + hctx->write_offset, - hctx->write_buffer->used - hctx->write_offset))) { + case PROXY_STATE_WRITE:; +#if defined USE_LINUX_SENDFILE + ret = network_write_chunkqueue_linuxsendfile(srv, con, hctx->fd, hctx->wb); +#elif defined USE_FREEBSD_SENDFILE + ret = network_write_chunkqueue_freebsdsendfile(srv, con, hctx->fd, hctx->wb); +#elif defined USE_SOLARIS_SENDFILEV + ret = network_write_chunkqueue_solarissendfilev(srv, con, hctx->fd, hctx->wb); +#elif defined USE_WRITEV + ret = network_write_chunkqueue_writev(srv, con, hctx->fd, hctx->wb); +#else + ret = network_write_chunkqueue_write(srv, con, hctx->fd, hctx->wb); +#endif + + chunkqueue_remove_finished_chunks(hctx->wb); + + if (-1 == ret) { if (errno != EAGAIN && errno != EINTR) { - log_error_write(srv, __FILE__, __LINE__, "ssd", "write failed:", strerror(errno), r); + log_error_write(srv, __FILE__, __LINE__, "ssd", "write failed:", strerror(errno), errno); return HANDLER_ERROR; } else { + fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT); + return HANDLER_WAIT_FOR_EVENT; } } - - hctx->write_offset += r; - - if (hctx->write_offset == hctx->write_buffer->used) { + + if (hctx->wb->bytes_out == hctx->wb->bytes_in) { proxy_set_state(srv, hctx, PROXY_STATE_READ); fdevent_event_del(srv->ev, &(hctx->fde_ndx), hctx->fd); fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_IN); + } else { + fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT); + + return HANDLER_WAIT_FOR_EVENT; } return HANDLER_WAIT_FOR_EVENT; diff --git a/src/mod_scgi.c b/src/mod_scgi.c index c1172c1b..baf8fb85 100644 --- a/src/mod_scgi.c +++ b/src/mod_scgi.c @@ -22,6 +22,7 @@ #include "plugin.h" #include "inet_ntop_cache.h" +#include "network_backends.h" #include @@ -298,10 +299,8 @@ typedef struct { int reconnects; /* number of reconnect attempts */ - buffer *write_buffer; - size_t write_offset; - read_buffer *rb; + chunkqueue *wb; buffer *response_header; @@ -338,7 +337,6 @@ static handler_ctx * handler_ctx_init() { hctx->response = buffer_init(); hctx->response_header = buffer_init(); - hctx->write_buffer = buffer_init(); hctx->request_id = 0; hctx->state = FCGI_STATE_INIT; @@ -350,6 +348,8 @@ static handler_ctx * handler_ctx_init() { hctx->fd = -1; hctx->reconnects = 0; + + hctx->wb = chunkqueue_init(); return hctx; } @@ -357,7 +357,8 @@ static handler_ctx * handler_ctx_init() { static void handler_ctx_free(handler_ctx *hctx) { buffer_free(hctx->response); buffer_free(hctx->response_header); - buffer_free(hctx->write_buffer); + + chunkqueue_free(hctx->wb); if (hctx->rb) { if (hctx->rb->ptr) free(hctx->rb->ptr); @@ -1409,11 +1410,11 @@ static int scgi_env_add_request_headers(server *srv, connection *con, plugin_dat static int scgi_create_env(server *srv, handler_ctx *hctx) { char buf[32]; - size_t offset; const char *s; #ifdef HAVE_IPV6 char b2[INET6_ADDRSTRLEN + 1]; #endif + buffer *b; plugin_data *p = hctx->plugin_data; scgi_extension_host *host= hctx->host; @@ -1564,23 +1565,66 @@ static int scgi_create_env(server *srv, handler_ctx *hctx) { #endif scgi_env_add_request_headers(srv, con, p); + + b = chunkqueue_get_append_buffer(hctx->wb); - buffer_append_long(hctx->write_buffer, p->scgi_env->used); - buffer_append_string_len(hctx->write_buffer, CONST_STR_LEN(":")); - buffer_append_string_len(hctx->write_buffer, (const char *)p->scgi_env->ptr, p->scgi_env->used); - buffer_append_string_len(hctx->write_buffer, CONST_STR_LEN(",")); - hctx->write_buffer->used--; - - /* send FCGI_STDIN */ + buffer_append_long(b, p->scgi_env->used); + buffer_append_string_len(b, CONST_STR_LEN(":")); + buffer_append_string_len(b, (const char *)p->scgi_env->ptr, p->scgi_env->used); + buffer_append_string_len(b, CONST_STR_LEN(",")); + + hctx->wb->bytes_in += b->used - 1; - /* something to send ? */ - for (offset = 0; offset != con->request.content_length; ) { - /* send chunks of 1024 bytes */ - size_t toWrite = con->request.content_length - offset > 4096 ? 4096 : con->request.content_length - offset; - - buffer_append_memory(hctx->write_buffer, (const char *)(con->request.content->ptr + offset), toWrite); - - offset += toWrite; + if (con->request.content_length) { + chunkqueue *req_cq = con->request_content_queue; + chunk *req_c; + size_t offset; + + /* something to send ? */ + for (offset = 0, req_c = req_cq->first; offset != req_cq->bytes_in; req_c = req_c->next) { + size_t weWant = req_cq->bytes_in - offset; + size_t weHave = 0; + + /* we announce toWrite octects + * now take all the request_content chunk that we need to fill this request + * */ + + switch (req_c->type) { + case FILE_CHUNK: + weHave = req_c->file.length - req_c->offset; + + if (weHave > weWant) weHave = weWant; + + chunkqueue_append_file(hctx->wb, req_c->file.name, req_c->offset, weHave); + + req_c->offset += weHave; + req_cq->bytes_out += weHave; + + hctx->wb->bytes_in += weHave; + + break; + case MEM_CHUNK: + /* append to the buffer */ + weHave = req_c->mem->used - 1 - req_c->offset; + + if (weHave > weWant) weHave = weWant; + + b = chunkqueue_get_append_buffer(hctx->wb); + buffer_append_memory(b, req_c->mem->ptr + req_c->offset, weHave); + b->used++; /* add virtual \0 */ + + req_c->offset += weHave; + req_cq->bytes_out += weHave; + + hctx->wb->bytes_in += weHave; + + break; + default: + break; + } + + offset += weHave; + } } #if 0 @@ -2078,7 +2122,7 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) { scgi_extension_host *host= hctx->host; connection *con = hctx->remote_conn; - int r; + int ret; /* sanity check */ if (!host || @@ -2095,9 +2139,9 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) { switch(hctx->state) { case FCGI_STATE_INIT: - r = host->unixsocket->used ? AF_UNIX : AF_INET; + ret = host->unixsocket->used ? AF_UNIX : AF_INET; - if (-1 == (hctx->fd = socket(r, SOCK_STREAM, 0))) { + if (-1 == (hctx->fd = socket(ret, SOCK_STREAM, 0))) { if (errno == EMFILE || errno == EINTR) { log_error_write(srv, __FILE__, __LINE__, "sd", @@ -2210,17 +2254,25 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) { scgi_create_env(srv, hctx); scgi_set_state(srv, hctx, FCGI_STATE_WRITE); - hctx->write_offset = 0; /* fall through */ case FCGI_STATE_WRITE: /* why aren't we using the network_ interface here ? */ - - r = write(hctx->fd, - hctx->write_buffer->ptr + hctx->write_offset, - hctx->write_buffer->used - hctx->write_offset); - - if (-1 == r) { +#if defined USE_LINUX_SENDFILE + ret = network_write_chunkqueue_linuxsendfile(srv, con, hctx->fd, hctx->wb); +#elif defined USE_FREEBSD_SENDFILE + ret = network_write_chunkqueue_freebsdsendfile(srv, con, hctx->fd, hctx->wb); +#elif defined USE_SOLARIS_SENDFILEV + ret = network_write_chunkqueue_solarissendfilev(srv, con, hctx->fd, hctx->wb); +#elif defined USE_WRITEV + ret = network_write_chunkqueue_writev(srv, con, hctx->fd, hctx->wb); +#else + ret = network_write_chunkqueue_write(srv, con, hctx->fd, hctx->wb); +#endif + + chunkqueue_remove_finished_chunks(hctx->wb); + + if (-1 == ret) { if (errno == ENOTCONN) { /* the connection got dropped after accept() * @@ -2228,7 +2280,7 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) { * after PHP_FCGI_MAX_REQUESTS * */ - if (hctx->write_offset == 0 && + if (hctx->wb->bytes_out == 0 && hctx->reconnects < 5) { usleep(10000); /* take away the load of the webserver * to let the php a chance to restart @@ -2247,7 +2299,7 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) { log_error_write(srv, __FILE__, __LINE__, "ssdsd", "[REPORT ME] connection was dropped after accept(). reconnect() denied:", - "write-offset:", hctx->write_offset, + "write-offset:", hctx->wb->bytes_out, "reconnect attempts:", hctx->reconnects); return HANDLER_ERROR; @@ -2267,13 +2319,15 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) { } } - hctx->write_offset += r; - - if (hctx->write_offset == hctx->write_buffer->used) { + if (hctx->wb->bytes_out == hctx->wb->bytes_in) { /* we don't need the out event anymore */ fdevent_event_del(srv->ev, &(hctx->fde_ndx), hctx->fd); fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_IN); scgi_set_state(srv, hctx, FCGI_STATE_READ); + } else { + fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT); + + return HANDLER_WAIT_FOR_EVENT; } break; @@ -2481,7 +2535,7 @@ static handler_t scgi_handle_fdevent(void *s, void *ctx, int revents) { if (con->file_started == 0) { /* nothing has been send out yet, try to use another child */ - if (hctx->write_offset == 0 && + if (hctx->wb->bytes_out == 0 && hctx->reconnects < 5) { scgi_reconnect(srv, hctx); @@ -2494,7 +2548,7 @@ static handler_t scgi_handle_fdevent(void *s, void *ctx, int revents) { } log_error_write(srv, __FILE__, __LINE__, "sdsdsd", - "response not sent, request sent:", hctx->write_offset, + "response not sent, request sent:", hctx->wb->bytes_out, "connection-fd:", con->fd, "fcgi-fd:", hctx->fd); diff --git a/src/mod_status.c b/src/mod_status.c index eaf0acd2..5b80f681 100644 --- a/src/mod_status.c +++ b/src/mod_status.c @@ -478,7 +478,7 @@ static handler_t mod_status_handle_server_status_html(server *srv, connection *c BUFFER_APPEND_STRING_CONST(b, ""); if (con->request.content_length) { - buffer_append_long(b, c->request.content->used); + buffer_append_long(b, c->request_content_queue->bytes_in); BUFFER_APPEND_STRING_CONST(b, "/"); buffer_append_long(b, c->request.content_length); } else { diff --git a/src/mod_webdav.c b/src/mod_webdav.c index f3499c12..be5f93e5 100644 --- a/src/mod_webdav.c +++ b/src/mod_webdav.c @@ -8,8 +8,13 @@ #include #include #include +#include +#include +#ifdef HAVE_CONFIG_H #include "config.h" +#endif + #if defined(HAVE_LIBXML_H) && defined(HAVE_SQLITE3_H) #define USE_PROPPATCH #include @@ -18,7 +23,6 @@ #include #endif - #include "base.h" #include "log.h" #include "buffer.h" @@ -1224,31 +1228,91 @@ URIHANDLER_FUNC(mod_webdav_subrequest_handler) { return HANDLER_FINISHED; case HTTP_METHOD_PUT: { int fd; + chunkqueue *cq = con->request_content_queue; if (p->conf.is_readonly) { con->http_status = 403; return HANDLER_FINISHED; } - + + assert(chunkqueue_length(cq) == con->request.content_length); + /* taken what we have in the request-body and write it to a file */ if (-1 == (fd = open(con->physical.path->ptr, O_WRONLY|O_CREAT|O_TRUNC, 0600))) { /* we can't open the file */ con->http_status = 403; } else { + chunk *c; + con->http_status = 201; /* created */ - if (-1 == (write(fd, con->request.content->ptr, con->request.content->used - 1))) { - switch(errno) { - case ENOSPC: - con->http_status = 507; + for (c = cq->first; c; c = cq->first) { + int r = 0; + + /* copy all chunks */ + switch(c->type) { + case FILE_CHUNK: + + if (c->file.mmap.start == MAP_FAILED) { + if (-1 == c->file.fd && /* open the file if not already open */ + -1 == (c->file.fd = open(c->file.name->ptr, O_RDONLY))) { + log_error_write(srv, __FILE__, __LINE__, "ss", "open failed: ", strerror(errno)); + + return -1; + } + + if (MAP_FAILED == (c->file.mmap.start = mmap(0, c->file.length, PROT_READ, MAP_SHARED, c->file.fd, 0))) { + log_error_write(srv, __FILE__, __LINE__, "ssbd", "mmap failed: ", + strerror(errno), c->file.name, c->file.fd); + + return -1; + } + + c->file.mmap.length = c->file.length; + close(c->file.fd); + c->file.fd = -1; + + /* chunk_reset() or chunk_free() will cleanup for us */ + } + + if ((r = write(fd, c->file.mmap.start + c->offset, c->file.length - c->offset)) < 0) { + switch(errno) { + case ENOSPC: + con->http_status = 507; + + break; + default: + con->http_status = 403; + break; + } + } break; - default: - con->http_status = 403; + case MEM_CHUNK: + if ((r = write(fd, c->mem->ptr + c->offset, c->mem->used - c->offset - 1)) < 0) { + switch(errno) { + case ENOSPC: + con->http_status = 507; + + break; + default: + con->http_status = 403; + break; + } + } break; } + + if (r > 0) { + c->offset += r; + cq->bytes_out += r; + } else { + break; + } + chunkqueue_remove_finished_chunks(cq); } close(fd); + } return HANDLER_FINISHED; }