Browse Source

[mod_deflate] skip deflate if loadavg too high (fixes #1505)

[mod_deflate] skip deflate if 1 min loadavg too high
deflate.max-loadavg  = "3.50"  # express value as string of float num

[mod_compress] skip compression if 1 min loadavg too high
compress.max-loadavg = "3.50"  # express value as string of float num

Feature available on BSD-like systems which have getloadavg() in libc

Note: load average calculations are different on different operating
systems and different types of system loads, so there is no value that
can be recommended for one-size-fits-all.

x-ref:
  "Enable mod_compress to abandon compression when load average is too high"
  https://redmine.lighttpd.net/issues/1505
personal/stbuehler/mod-csrf
Glenn Strauss 5 years ago
parent
commit
1f3ad401ba
  1. 2
      SConstruct
  2. 2
      configure.ac
  3. 1
      src/CMakeLists.txt
  4. 2
      src/base.h
  5. 19
      src/mod_compress.c
  6. 16
      src/mod_deflate.c
  7. 12
      src/server.c

2
SConstruct

@ -221,7 +221,7 @@ if 1:
getuid select signal pathconf madvise prctl\
writev sigaction sendfile64 send_file kqueue port_create localtime_r posix_fadvise issetugid inet_pton \
memset_s explicit_bzero clock_gettime \
getentropy arc4random jrand48 srandom'))
getentropy arc4random jrand48 srandom getloadavg'))
checkFunc(autoconf, 'getrandom', 'linux/random.h')
checkTypes(autoconf, Split('pid_t size_t off_t'))

2
configure.ac

@ -772,7 +772,7 @@ AC_CHECK_FUNCS([dup2 getcwd inet_ntoa inet_ntop inet_pton issetugid memset mmap
getuid select signal pathconf madvise posix_fadvise posix_madvise \
writev sigaction sendfile64 send_file kqueue port_create localtime_r gmtime_r \
memset_s explicit_bzero clock_gettime \
getentropy arc4random jrand48 srandom])
getentropy arc4random jrand48 srandom getloadavg])
AC_CHECK_HEADERS([linux/random.h],[
AC_CHECK_FUNC([getrandom], AC_DEFINE([HAVE_GETRANDOM], [1], [getrandom]))
])

1
src/CMakeLists.txt

@ -134,6 +134,7 @@ check_function_exists(chroot HAVE_CHROOT)
check_function_exists(epoll_ctl HAVE_EPOLL_CTL)
check_function_exists(fork HAVE_FORK)
check_function_exists(getentropy HAVE_GETENTROPY)
check_function_exists(getloadavg HAVE_GETLOADAVG)
check_function_exists(getrlimit HAVE_GETRLIMIT)
check_function_exists(getuid HAVE_GETUID)
check_function_exists(gmtime_r HAVE_GMTIME_R)

2
src/base.h

@ -563,6 +563,8 @@ typedef struct {
unsigned short http_host_strict;
unsigned short http_host_normalize;
unsigned short high_precision_timestamps;
time_t loadts;
double loadavg[3];
} server_config;
typedef struct server_socket {

19
src/mod_compress.c

@ -71,6 +71,7 @@ typedef struct {
array *compress;
off_t compress_max_filesize; /** max filesize in kb */
int allowed_encodings;
double max_loadavg;
} plugin_config;
typedef struct {
@ -177,6 +178,7 @@ SETDEFAULTS_FUNC(mod_compress_setdefaults) {
{ "compress.filetype", NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION },
{ "compress.max-filesize", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION },
{ "compress.allowed-encodings", NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION },
{ "compress.max-loadavg", NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION },
{ NULL, NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET }
};
@ -192,11 +194,14 @@ SETDEFAULTS_FUNC(mod_compress_setdefaults) {
s->compress = array_init();
s->compress_max_filesize = 0;
s->allowed_encodings = 0;
s->max_loadavg = 0.0;
cv[0].destination = s->compress_cache_dir;
cv[1].destination = s->compress;
cv[2].destination = &(s->compress_max_filesize);
cv[3].destination = encodings_arr; /* temp array for allowed encodings list */
cv[4].destination = srv->tmp_buf;
buffer_string_set_length(srv->tmp_buf, 0);
p->config_storage[i] = s;
@ -204,6 +209,10 @@ SETDEFAULTS_FUNC(mod_compress_setdefaults) {
return HANDLER_ERROR;
}
if (!buffer_string_is_empty(srv->tmp_buf)) {
s->max_loadavg = strtod(srv->tmp_buf->ptr, NULL);
}
if (encodings_arr->used) {
size_t j = 0;
for (j = 0; j < encodings_arr->used; j++) {
@ -509,6 +518,10 @@ static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, bu
return 0;
}
if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < srv->srvconf.loadavg[0]) {
return -1;
}
if (-1 == mkdir_for_file(p->ofn->ptr)) {
log_error_write(srv, __FILE__, __LINE__, "sb", "couldn't create directory for file", p->ofn);
return -1;
@ -656,6 +669,9 @@ static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p,
if (sce->st.st_size > 128 * 1024 * 1024) return -1;
if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < srv->srvconf.loadavg[0]) {
return -1;
}
if (-1 == (ifd = open(fn->ptr, O_RDONLY | O_BINARY))) {
log_error_write(srv, __FILE__, __LINE__, "sbss", "opening plain-file", fn, "failed", strerror(errno));
@ -745,6 +761,7 @@ static int mod_compress_patch_connection(server *srv, connection *con, plugin_da
PATCH(compress);
PATCH(compress_max_filesize);
PATCH(allowed_encodings);
PATCH(max_loadavg);
/* skip the first, the global context */
for (i = 1; i < srv->config_context->used; i++) {
@ -766,6 +783,8 @@ static int mod_compress_patch_connection(server *srv, connection *con, plugin_da
PATCH(compress_max_filesize);
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.allowed-encodings"))) {
PATCH(allowed_encodings);
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("compress.max-loadavg"))) {
PATCH(max_loadavg);
}
}
}

16
src/mod_deflate.c

@ -174,6 +174,7 @@ typedef struct {
unsigned short work_block_size;
unsigned short sync_flush;
short compression_level;
double max_loadavg;
} plugin_config;
typedef struct {
@ -274,6 +275,7 @@ SETDEFAULTS_FUNC(mod_deflate_setdefaults) {
{ "deflate.compression-level", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION },
{ "deflate.output-buffer-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION },
{ "deflate.work-block-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION },
{ "deflate.max-loadavg", NULL, T_CONFIG_STRING,T_CONFIG_SCOPE_CONNECTION },
{ NULL, NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET }
};
@ -291,6 +293,7 @@ SETDEFAULTS_FUNC(mod_deflate_setdefaults) {
s->work_block_size = 2048;
s->sync_flush = 0;
s->compression_level = -1;
s->max_loadavg = 0.0;
array_reset(p->encodings); /* temp array for allowed encodings list */
@ -301,6 +304,8 @@ SETDEFAULTS_FUNC(mod_deflate_setdefaults) {
cv[4].destination = &(s->compression_level);
cv[5].destination = &(s->output_buffer_size);
cv[6].destination = &(s->work_block_size);
cv[7].destination = p->tmp_buf;
buffer_string_set_length(p->tmp_buf, 0);
p->config_storage[i] = s;
@ -315,6 +320,10 @@ SETDEFAULTS_FUNC(mod_deflate_setdefaults) {
return HANDLER_ERROR;
}
if (!buffer_string_is_empty(p->tmp_buf)) {
s->max_loadavg = strtod(p->tmp_buf->ptr, NULL);
}
if (p->encodings->used) {
size_t j = 0;
for (j = 0; j < p->encodings->used; j++) {
@ -929,6 +938,7 @@ static int mod_deflate_patch_connection(server *srv, connection *con, plugin_dat
PATCH(compression_level);
PATCH(output_buffer_size);
PATCH(work_block_size);
PATCH(max_loadavg);
/* skip the first, the global context */
for (i = 1; i < srv->config_context->used; i++) {
@ -956,6 +966,8 @@ static int mod_deflate_patch_connection(server *srv, connection *con, plugin_dat
PATCH(output_buffer_size);
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.work-block-size"))) {
PATCH(work_block_size);
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.max-loadavg"))) {
PATCH(max_loadavg);
}
}
}
@ -1142,6 +1154,10 @@ CONNECTION_FUNC(mod_deflate_handle_response_start) {
}
}
if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < srv->srvconf.loadavg[0]) {
return HANDLER_GO_ON;
}
/* update ETag, if ETag response header is set */
if (etaglen) {
/* modify ETag response header in-place to remove '"' and append '-label"' */

12
src/server.c

@ -269,6 +269,9 @@ static server *server_init(void) {
srv->srvconf.http_host_normalize = 0;
srv->srvconf.high_precision_timestamps = 0;
srv->srvconf.max_request_field_size = 8192;
srv->srvconf.loadavg[0] = 0.0;
srv->srvconf.loadavg[1] = 0.0;
srv->srvconf.loadavg[2] = 0.0;
/* use syslog */
srv->errorlog_fd = STDERR_FILENO;
@ -1583,6 +1586,15 @@ int main (int argc, char **argv) {
graceful_shutdown = 2; /* value 2 indicates idle timeout */
}
#ifdef HAVE_GETLOADAVG
/* refresh loadavg data every 30 seconds */
if (srv->srvconf.loadts + 30 < min_ts) {
if (-1 != getloadavg(srv->srvconf.loadavg, 3)) {
srv->srvconf.loadts = min_ts;
}
}
#endif
/* cleanup stat-cache */
stat_cache_trigger_cleanup(srv);
/**

Loading…
Cancel
Save