lighttpd
/
lighttpd1.4
2
0
Fork 0
Code Issues Releases Wiki Activity

[multiple] chunkqueue_write_chunk() 

create API in chunk.[ch] for writing a chunk to an fd

(pull similar code from mod_cgi and mod_webdav)

This new API is intended for use on request body input, which is
written to size-limited temporary files controlled by lighttpd and
written to files or pipes.

(network_backend_write() is for writing chunkqueues to sockets)
This commit is contained in:
Glenn Strauss 2020-12-27 07:57:31 -05:00
parent 15bfe5ef0e
commit 2639e5ae43
4 changed files with 208 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

185
src/chunk.c
View File

@ -885,16 +885,6 @@ void chunkqueue_compact_mem(chunkqueue *cq, size_t clen) {
}
static int chunk_open_file_chunk(chunk * const restrict c, log_error_st * const restrict errh) {
off_t offset, toSend;
struct stat st;
force_assert(NULL != c);
force_assert(FILE_CHUNK == c->type);
force_assert(c->offset >= 0 && c->offset <= c->file.length);
offset = c->offset;
toSend = c->file.length - c->offset;
if (-1 == c->file.fd) {
/* (permit symlinks; should already have been checked. However, TOC-TOU remains) */
if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, 1, O_RDONLY, 0))) {
@ -906,11 +896,17 @@ static int chunk_open_file_chunk(chunk * const restrict c, log_error_st * const
/*(skip file size checks if file is temp file created by lighttpd)*/
if (c->file.is_temp) return 0;
force_assert(FILE_CHUNK == c->type);
force_assert(c->offset >= 0 && c->offset <= c->file.length);
struct stat st;
if (-1 == fstat(c->file.fd, &st)) {
log_perror(errh, __FILE__, __LINE__, "fstat failed");
return -1;
}
const off_t offset = c->offset;
const off_t toSend = c->file.length - c->offset;
if (offset > st.st_size || toSend > st.st_size || offset > st.st_size - toSend) {
log_error(errh, __FILE__, __LINE__, "file shrunk: %s", c->mem->ptr);
return -1;
@ -924,6 +920,165 @@ int chunkqueue_open_file_chunk(chunkqueue * const restrict cq, log_error_st * co
}
#if defined(HAVE_MMAP)
__attribute_cold__
#endif
__attribute_noinline__
static ssize_t
chunkqueue_write_chunk_file_intermed (const int fd, chunk * const restrict c, log_error_st * const errh)
{
char buf[16384];
char *data = buf;
const off_t count = c->file.length - c->offset;
uint32_t dlen = count < (off_t)sizeof(buf) ? (uint32_t)count : sizeof(buf);
chunkqueue cq = {c,c,0,0,0,0,0}; /*(fake cq for chunkqueue_peek_data())*/
if (0 != chunkqueue_peek_data(&cq, &data, &dlen, errh) && 0 == dlen)
return -1;
ssize_t wr;
do { wr = write(fd, data, dlen); } while (-1 == wr && errno == EINTR);
return wr;
}
#if defined(HAVE_MMAP)
/*(improved from network_write_mmap.c)*/
static off_t
mmap_align_offset (off_t start)
{
static off_t pagemask = 0;
if (0 == pagemask) {
long pagesize = sysconf(_SC_PAGESIZE);
if (-1 == pagesize) pagesize = 4096;
pagemask = ~((off_t)pagesize - 1); /* pagesize always power-of-2 */
}
return (start & pagemask);
}
#endif
#if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \
&& (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \
&& defined(__linux__) && !defined HAVE_SENDFILE_BROKEN
#include <sys/sendfile.h>
#include <stdint.h>
#endif
static ssize_t
chunkqueue_write_chunk_file (const int fd, chunk * const restrict c, log_error_st * const errh)
{
/*(similar to network_write_file_chunk_mmap(), but does not use send() on
* Windows because fd is expected to be file or pipe here, not socket)*/
if (0 != chunk_open_file_chunk(c, errh))
return -1;
const off_t count = c->file.length - c->offset;
if (0 == count) return 0; /*(sanity check)*/
ssize_t wr;
#if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \
&& (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \
&& defined(__linux__) && !defined HAVE_SENDFILE_BROKEN
/* Linux kernel >= 2.6.33 supports sendfile() between most fd types */
off_t offset = c->offset;
wr = sendfile(fd, c->file.fd, &offset, count<INT32_MAX ? count : INT32_MAX);
if (wr >= 0) return wr;
if (wr < 0 && (errno == EINVAL || errno == ENOSYS))
#endif
{
#if defined(HAVE_MMAP)
/*(caller is responsible for handling SIGBUS if chunkqueue might contain
* untrusted file, i.e. any file other than lighttpd-created tempfile)*/
/*(tempfiles are expected for input, MAP_PRIVATE used for portability)*/
/*(mmaps and writes complete chunk instead of only small parts; files
* are expected to be temp files with reasonable chunk sizes)*/
/* (re)mmap the buffer if range is not covered completely */
if (MAP_FAILED == c->file.mmap.start
|| c->offset < c->file.mmap.offset
|| c->file.length
> (off_t)(c->file.mmap.offset + c->file.mmap.length)) {
if (MAP_FAILED != c->file.mmap.start) {
munmap(c->file.mmap.start, c->file.mmap.length);
c->file.mmap.start = MAP_FAILED;
}
c->file.mmap.offset = mmap_align_offset(c->offset);
c->file.mmap.length = c->file.length - c->file.mmap.offset;
c->file.mmap.start =
mmap(NULL, c->file.mmap.length, PROT_READ, MAP_PRIVATE,
c->file.fd, c->file.mmap.offset);
#if 0
/* close() fd as soon as fully mmap() rather than when done w/ chunk
* (possibly worthwhile to keep active fd count lower) */
if (c->file.is_temp && !c->file.refchg) {
close(c->file.fd);
c->file.fd = -1;
}
#endif
}
if (MAP_FAILED != c->file.mmap.start) {
const char * const data =
c->file.mmap.start + c->offset - c->file.mmap.offset;
do { wr = write(fd,data,count); } while (-1 == wr && errno==EINTR);
}
else
#endif
wr = chunkqueue_write_chunk_file_intermed(fd, c, errh);
}
return wr;
}
static ssize_t
chunkqueue_write_chunk_mem (const int fd, const chunk * const restrict c)
{
const void * const buf = c->mem->ptr + c->offset;
const size_t count = chunk_buffer_string_length(c->mem) - (size_t)c->offset;
ssize_t wr;
do { wr = write(fd, buf, count); } while (-1 == wr && errno == EINTR);
return wr;
}
ssize_t
chunkqueue_write_chunk (const int fd, chunkqueue * const restrict cq, log_error_st * const restrict errh)
{
/*(note: expects non-empty cq->first)*/
chunk * const c = cq->first;
switch (c->type) {
case MEM_CHUNK:
return chunkqueue_write_chunk_mem(fd, c);
case FILE_CHUNK:
return chunkqueue_write_chunk_file(fd, c, errh);
default:
errno = EINVAL;
return -1;
}
}
ssize_t
chunkqueue_write_chunk_to_pipe (const int fd, chunkqueue * const restrict cq, log_error_st * const restrict errh)
{
/*(note: expects non-empty cq->first)*/
#ifdef SPLICE_F_NONBLOCK /* splice() temp files to pipe on Linux */
chunk * const c = cq->first;
if (c->type == FILE_CHUNK) {
loff_t abs_offset = c->offset;
return (0 == chunk_open_file_chunk(c, errh))
? splice(c->file.fd, &abs_offset, fd, NULL,
(size_t)(c->file.length - c->offset), SPLICE_F_NONBLOCK)
: -1;
}
#endif
return chunkqueue_write_chunk(fd, cq, errh);
}
void
chunkqueue_small_resp_optim (chunkqueue * const restrict cq)
{
@ -1004,12 +1159,16 @@ chunkqueue_peek_data (chunkqueue * const cq,
toSend = (off_t)space;
if (-1 == lseek(c->file.fd, offset, SEEK_SET)) {
log_perror(errh, __FILE__, __LINE__, "lseek");
log_perror(errh, __FILE__, __LINE__, "lseek(\"%s\")",
c->mem->ptr);
return -1;
}
toSend = read(c->file.fd, data_in + *dlen, (size_t)toSend);
do {
toSend = read(c->file.fd, data_in + *dlen, (size_t)toSend);
} while (-1 == toSend && errno == EINTR);
if (toSend <= 0) { /* -1 error; 0 EOF (unexpected) */
log_perror(errh, __FILE__, __LINE__, "read");
log_perror(errh, __FILE__, __LINE__, "read(\"%s\")",
c->mem->ptr);
return -1;
}

3
src/chunk.h
View File

@ -127,6 +127,9 @@ void chunkqueue_compact_mem(chunkqueue *cq, size_t clen);
void chunkqueue_small_resp_optim (chunkqueue * restrict cq);
ssize_t chunkqueue_write_chunk (int fd, chunkqueue * restrict cq, struct log_error_st * restrict errh);
ssize_t chunkqueue_write_chunk_to_pipe (int fd, chunkqueue * restrict cq, struct log_error_st * restrict errh);
int chunkqueue_peek_data (chunkqueue *cq, char **data, uint32_t *dlen, struct log_error_st * restrict errh);
int chunkqueue_read_data (chunkqueue *cq, char *data, uint32_t dlen, struct log_error_st * restrict errh);

178
src/mod_cgi.c
View File

@ -12,7 +12,6 @@
#include "plugin.h"
#include <sys/types.h>
#include "sys-mmap.h"
#include "sys-socket.h"
#ifdef HAVE_SYS_WAIT_H
#include <sys/wait.h>
@ -528,187 +527,50 @@ static int cgi_env_add(void *venv, const char *key, size_t key_len, const char *
return 0;
}
#ifndef SPLICE_F_NONBLOCK
/*(improved from network_write_mmap.c)*/
static off_t mmap_align_offset(off_t start) {
static off_t pagemask = 0;
if (0 == pagemask) {
long pagesize = sysconf(_SC_PAGESIZE);
if (-1 == pagesize) pagesize = 4096;
pagemask = ~((off_t)pagesize - 1); /* pagesize always power-of-2 */
}
return (start & pagemask);
}
#endif
/* returns: 0: continue, -1: fatal error, -2: connection reset */
/* similar to network_write_file_chunk_mmap, but doesn't use send on windows (because we're on pipes),
* also mmaps and sends complete chunk instead of only small parts - the files
* are supposed to be temp files with reasonable chunk sizes.
*
* Also always use mmap; the files are "trusted", as we created them.
*/
static ssize_t cgi_write_file_chunk_mmap(request_st * const r, int fd, chunkqueue *cq) {
chunk* const c = cq->first;
off_t offset, toSend;
ssize_t wr;
force_assert(NULL != c);
force_assert(FILE_CHUNK == c->type);
force_assert(c->offset >= 0 && c->offset <= c->file.length);
offset = c->offset;
toSend = c->file.length - c->offset;
if (0 == toSend) {
chunkqueue_remove_finished_chunks(cq);
return 0;
}
/*(simplified from chunk.c:chunkqueue_open_file_chunk())*/
if (-1 == c->file.fd) {
if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, r->conf.follow_symlink, O_RDONLY, 0))) {
log_perror(r->conf.errh, __FILE__, __LINE__, "open failed: %s", c->mem->ptr);
return -1;
}
}
#ifdef SPLICE_F_NONBLOCK
loff_t abs_offset = offset;
wr = splice(c->file.fd, &abs_offset, fd, NULL,
(size_t)toSend, SPLICE_F_NONBLOCK);
#else
size_t mmap_offset, mmap_avail;
char *data = NULL;
off_t file_end = c->file.length; /* offset to file end in this chunk */
/* (re)mmap the buffer if range is not covered completely */
if (MAP_FAILED == c->file.mmap.start
|| offset < c->file.mmap.offset
|| file_end > (off_t)(c->file.mmap.offset + c->file.mmap.length)) {
if (MAP_FAILED != c->file.mmap.start) {
munmap(c->file.mmap.start, c->file.mmap.length);
c->file.mmap.start = MAP_FAILED;
}
c->file.mmap.offset = mmap_align_offset(offset);
c->file.mmap.length = file_end - c->file.mmap.offset;
if (MAP_FAILED == (c->file.mmap.start = mmap(NULL, c->file.mmap.length, PROT_READ, MAP_PRIVATE, c->file.fd, c->file.mmap.offset))) {
if (toSend > 65536) toSend = 65536;
data = malloc(toSend);
force_assert(data);
if (-1 == lseek(c->file.fd, offset, SEEK_SET)
|| 0 >= (toSend = read(c->file.fd, data, toSend))) {
if (-1 == toSend) {
log_perror(r->conf.errh, __FILE__, __LINE__,
"lseek/read %s %d %lld failed:",
c->mem->ptr, c->file.fd, (long long)offset);
} else { /*(0 == toSend)*/
log_error(r->conf.errh, __FILE__, __LINE__,
"unexpected EOF (input truncated?): %s %d %lld",
c->mem->ptr, c->file.fd, (long long)offset);
}
free(data);
return -1;
}
}
}
if (MAP_FAILED != c->file.mmap.start) {
force_assert(offset >= c->file.mmap.offset);
mmap_offset = offset - c->file.mmap.offset;
force_assert(c->file.mmap.length > mmap_offset);
mmap_avail = c->file.mmap.length - mmap_offset;
force_assert(toSend <= (off_t) mmap_avail);
data = c->file.mmap.start + mmap_offset;
}
wr = write(fd, data, toSend);
if (MAP_FAILED == c->file.mmap.start) free(data);
#endif
if (wr < 0) {
switch (errno) {
case EAGAIN:
case EINTR:
return 0;
case EPIPE:
case ECONNRESET:
return -2;
default:
log_perror(r->conf.errh, __FILE__, __LINE__,
"write %d failed", fd);
return -1;
}
}
chunkqueue_mark_written(cq, wr);
return wr;
}
static int cgi_write_request(handler_ctx *hctx, int fd) {
request_st * const r = hctx->r;
chunkqueue *cq = &r->reqbody_queue;
chunk *c;
chunkqueue_remove_finished_chunks(cq); /* unnecessary? */
/* old comment: windows doesn't support select() on pipes - wouldn't be easy to fix for all platforms.
* solution: if this is still a problem on windows, then substitute
* socketpair() for pipe() and closesocket() for close() on windows.
*/
for (c = cq->first; c; c = cq->first) {
ssize_t wr = -1;
switch(c->type) {
case FILE_CHUNK:
wr = cgi_write_file_chunk_mmap(r, fd, cq);
break;
case MEM_CHUNK:
if ((wr = write(fd, c->mem->ptr + c->offset, buffer_string_length(c->mem) - c->offset)) < 0) {
ssize_t wr = chunkqueue_write_chunk_to_pipe(fd, cq, r->conf.errh);
if (wr > 0) {
chunkqueue_mark_written(cq, wr);
/* continue if wrote whole chunk or wrote 16k block
* (see chunkqueue_write_chunk_file_intermed()) */
if (c != cq->first || wr == 16384)
continue;
/*(else partial write)*/
}
else if (wr < 0) {
switch(errno) {
case EAGAIN:
case EINTR:
/* ignore and try again */
wr = 0;
/* ignore and try again later */
break;
case EPIPE:
case ECONNRESET:
/* connection closed */
wr = -2;
log_error(r->conf.errh, __FILE__, __LINE__,
"failed to send post data to cgi, connection closed by CGI");
/* skip all remaining data */
chunkqueue_mark_written(cq, chunkqueue_length(cq));
break;
default:
/* fatal error */
log_perror(r->conf.errh, __FILE__, __LINE__, "write() failed");
wr = -1;
break;
return -1;
}
} else if (wr > 0) {
chunkqueue_mark_written(cq, wr);
}
break;
}
if (0 == wr) break; /*(might block)*/
switch (wr) {
case -1:
/* fatal error */
return -1;
case -2:
/* connection reset */
log_error(r->conf.errh, __FILE__, __LINE__,
"failed to send post data to cgi, connection closed by CGI");
/* skip all remaining data */
chunkqueue_mark_written(cq, chunkqueue_length(cq));
break;
default:
break;
}
/*if (0 == wr) break;*/ /*(might block)*/
break;
}
if (cq->bytes_out == (off_t)r->reqbody_length && !hctx->conf.upgrade) {

114
src/mod_webdav.c
View File

@ -3553,36 +3553,13 @@ webdav_parse_chunkqueue (request_st * const r,
weHave = c->file.length - c->offset;
}
else {
switch (errno) {
case ENOSYS: case ENODEV: case EINVAL: break;
default:
log_perror(r->conf.errh, __FILE__, __LINE__,
"open() or mmap() '%*.s'",
BUFFER_INTLEN_PTR(c->mem));
}
if (webdav_open_chunk_file_rd(c) < 0) {
log_perror(r->conf.errh, __FILE__, __LINE__,
"open() '%*.s'",
BUFFER_INTLEN_PTR(c->mem));
err = XML_IO_UNKNOWN;
break;
}
ssize_t rd = -1;
do {
if (-1 == lseek(c->file.fd, c->offset, SEEK_SET))
break;
off_t len = c->file.length - c->offset;
if (len > (off_t)sizeof(buf)) len = (off_t)sizeof(buf);
rd = read(c->file.fd, buf, (size_t)len);
} while (-1 == rd && errno == EINTR);
if (rd >= 0) {
xmlstr = buf;
weHave = (size_t)rd;
char *data = buf;
uint32_t dlen = sizeof(buf);
if (0 == chunkqueue_peek_data(cq, &data, &dlen, r->conf.errh)) {
xmlstr = data;
weHave = dlen;
}
else {
log_perror(r->conf.errh, __FILE__, __LINE__,
"read() '%*.s'",
BUFFER_INTLEN_PTR(c->mem));
err = XML_IO_UNKNOWN;
break;
}
@ -4229,85 +4206,20 @@ mod_webdav_delete (request_st * const r, const plugin_config * const pconf)
}
static ssize_t
mod_webdav_write_cq_first_chunk (request_st * const r, chunkqueue * const cq,
const int fd)
{
/* (Note: copying might take some time, temporarily pausing server) */
chunk *c = cq->first;
ssize_t wr = 0;
switch(c->type) {
case FILE_CHUNK:
if (NULL != webdav_mmap_file_chunk(c)) {
do {
wr = write(fd,
c->file.mmap.start + c->offset - c->file.mmap.offset,
c->file.length - c->offset);
} while (-1 == wr && errno == EINTR);
break;
}
else {
switch (errno) {
case ENOSYS: case ENODEV: case EINVAL: break;
default:
log_perror(r->conf.errh, __FILE__, __LINE__,
"open() or mmap() '%*.s'",
BUFFER_INTLEN_PTR(c->mem));
}
if (webdav_open_chunk_file_rd(c) < 0) {
http_status_set_error(r, 500); /* Internal Server Error */
return -1;
}
ssize_t rd = -1;
char buf[16384];
do {
if (-1 == lseek(c->file.fd, c->offset, SEEK_SET))
break;
off_t len = c->file.length - c->offset;
if (len > (off_t)sizeof(buf)) len = (off_t)sizeof(buf);
rd = read(c->file.fd, buf, (size_t)len);
} while (-1 == rd && errno == EINTR);
if (rd >= 0) {
do {
wr = write(fd, buf, (size_t)rd);
} while (-1 == wr && errno == EINTR);
break;
}
else {
log_perror(r->conf.errh, __FILE__, __LINE__,
"read() '%*.s'",
BUFFER_INTLEN_PTR(c->mem));
http_status_set_error(r, 500); /* Internal Server Error */
return -1;
}
}
case MEM_CHUNK:
do {
wr = write(fd, c->mem->ptr + c->offset,
buffer_string_length(c->mem) - c->offset);
} while (-1 == wr && errno == EINTR);
break;
}
if (wr > 0) {
chunkqueue_mark_written(cq, wr);
}
else if (wr < 0)
http_status_set_error(r, (errno == ENOSPC) ? 507 : 403);
return wr;
}
__attribute_noinline__
static int
mod_webdav_write_cq (request_st * const r, chunkqueue * const cq, const int fd)
{
/* (Note: copying might take some time, temporarily pausing server) */
chunkqueue_remove_finished_chunks(cq);
while (!chunkqueue_is_empty(cq)) {
if (mod_webdav_write_cq_first_chunk(r, cq, fd) < 0) return 0;
ssize_t wr = chunkqueue_write_chunk(fd, cq, r->conf.errh);
if (wr > 0)
chunkqueue_mark_written(cq, wr);
else if (wr < 0) {
http_status_set_error(r, (errno == ENOSPC) ? 507 : 403);
return 0;
}
}
return 1;
}