Added mod_cache_disk_etag: caches produced content on disk and checks for it via etag.

personal/stbuehler/wip
Stefan Bühler 14 years ago
parent 87cd65e98f
commit bedcc3d46a
  1. 12
      include/lighttpd/virtualrequest.h
  2. 1
      src/CMakeLists.txt
  3. 2
      src/modules/mod_accesslog.c
  4. 356
      src/modules/mod_cache_disk_etag.c
  5. 31
      src/virtualrequest.c
  6. 1
      src/wscript

@ -32,14 +32,16 @@ typedef enum {
VRS_ERROR
} vrequest_state;
typedef handler_t (*filter_handler)(vrequest *vr, filter *f, plugin *p);
typedef handler_t (*filter_handler)(vrequest *vr, filter *f);
typedef void (*filter_free)(vrequest *vr, filter *f);
typedef handler_t (*vrequest_handler)(vrequest *vr);
typedef handler_t (*vrequest_plugin_handler)(vrequest *vr, plugin *p);
struct filter {
chunkqueue *in, *out;
plugin *p;
filter_handler handle;
filter_handler handle_data;
filter_free handle_free;
gpointer param;
};
struct filters {
@ -99,8 +101,8 @@ LI_API vrequest* vrequest_new(struct connection *con, vrequest_handler handle_re
LI_API void vrequest_free(vrequest *vr);
LI_API void vrequest_reset(vrequest *vr);
LI_API void vrequest_add_filter_in(vrequest *vr, plugin *p, filter_handler handle);
LI_API void vrequest_add_filter_out(vrequest *vr, plugin *p, filter_handler handle);
LI_API void vrequest_add_filter_in(vrequest *vr, filter_handler handle_data, filter_free handle_free, gpointer param);
LI_API void vrequest_add_filter_out(vrequest *vr, filter_handler handle_data, filter_free handle_free, gpointer param);
/* Signals an internal error; handles the error in the _next_ loop */
LI_API void vrequest_error(vrequest *vr);

@ -350,6 +350,7 @@ SET(COMMON_CFLAGS "${LUA_CFLAGS} ${EV_CFLAGS} ${GTHREAD_CFLAGS} ${GMODULE_CFLAGS
ADD_AND_INSTALL_LIBRARY(mod_accesslog "modules/mod_accesslog.c")
ADD_AND_INSTALL_LIBRARY(mod_balancer "modules/mod_balancer.c")
ADD_AND_INSTALL_LIBRARY(mod_cache_disk_etag "modules/mod_cache_disk_etag.c")
ADD_AND_INSTALL_LIBRARY(mod_dirlist "modules/mod_dirlist.c")
ADD_AND_INSTALL_LIBRARY(mod_fastcgi "modules/mod_fastcgi.c")
ADD_AND_INSTALL_LIBRARY(mod_fortune "modules/mod_fortune.c")

@ -449,8 +449,6 @@ static const plugin_option options[] = {
};
static const plugin_action actions[] = {
{ "al", NULL },
{ NULL, NULL }
};

@ -0,0 +1,356 @@
/*
* mod_cache_disk_etag - cache generated content on disk if etag header is set
*
* Description:
* cache generated content on disk if etag header is set
*
* Setups:
* none
* Options:
* none
* Actions:
* cache.disk.etag <path> - cache in specified directory
* path: string
* This blocks action progress until the response headers are
* done (i.e. there has to be a content generator before it (like fastcgi/static file)
* You could insert it multiple times of course (e.g. before and after deflate).
*
* Example config:
* cache.disk.etag "/var/lib/lighttpd/cache_etag"
*
* Todo:
* - use stat cache
*
* Author:
* Copyright (c) 2009 Stefan Bühler
*/
#include <lighttpd/base.h>
#include <lighttpd/plugin_core.h>
#include <sys/stat.h>
#include <fcntl.h>
LI_API gboolean mod_cache_disk_etag_init(modules *mods, module *mod);
LI_API gboolean mod_cache_disk_etag_free(modules *mods, module *mod);
struct cache_etag_context;
typedef struct cache_etag_context cache_etag_context;
struct cache_etag_context {
GString *path;
};
struct cache_etag_file;
typedef struct cache_etag_file cache_etag_file;
struct cache_etag_file {
GString *filename, *tmpfilename;
int fd;
/* cache hit */
int hit_fd;
goffset hit_length;
};
static cache_etag_file* cache_etag_file_create(GString *filename) {
cache_etag_file *cfile = g_slice_new0(cache_etag_file);
cfile->filename = filename;
cfile->fd = -1;
cfile->hit_fd = -1;
return cfile;
}
static gboolean mkdir_for_file(vrequest *vr, char *filename) {
char *p = filename;
if (!filename || !filename[0])
return FALSE;
while ((p = strchr(p + 1, '/')) != NULL) {
*p = '\0';
if ((mkdir(filename, 0700) != 0) && (errno != EEXIST)) {
VR_ERROR(vr, "creating cache-directory '%s' failed: %s", filename, g_strerror(errno));
*p = '/';
return FALSE;
}
*p++ = '/';
if (!*p) {
VR_ERROR(vr, "unexpected trailing slash for filename '%s'", filename);
return FALSE;
}
}
return TRUE;
}
static gboolean cache_etag_file_start(vrequest *vr, cache_etag_file *cfile) {
cfile->tmpfilename = g_string_sized_new(cfile->filename->len + 7);
g_string_append_len(cfile->tmpfilename, GSTR_LEN(cfile->filename));
g_string_append_len(cfile->tmpfilename, CONST_STR_LEN("-XXXXXX"));
if (!mkdir_for_file(vr, cfile->tmpfilename->str)) {
return FALSE;
}
errno = 0; /* posix doesn't define any errors */
if (-1 == (cfile->fd = mkstemp(cfile->tmpfilename->str))) {
VR_ERROR(vr, "Couldn't create cache tempfile '%s': %s", cfile->tmpfilename->str, g_strerror(errno));
return FALSE;
}
#ifdef FD_CLOEXEC
fcntl(cfile->fd, F_SETFD, FD_CLOEXEC);
#endif
return TRUE;
}
static void cache_etag_file_free(cache_etag_file *cfile) {
if (!cfile) return;
if (cfile->fd != -1) {
close(cfile->fd);
unlink(cfile->tmpfilename->str);
}
if (cfile->hit_fd != -1) close(cfile->hit_fd);
if (cfile->filename) g_string_free(cfile->filename, TRUE);
if (cfile->tmpfilename) g_string_free(cfile->tmpfilename, TRUE);
g_slice_free(cache_etag_file, cfile);
}
static void cache_etag_file_finish(vrequest *vr, cache_etag_file *cfile) {
close(cfile->fd);
cfile->fd = -1;
if (-1 == rename(cfile->tmpfilename->str, cfile->filename->str)) {
VR_ERROR(vr, "Couldn't move temporary cache file '%s': '%s'", cfile->tmpfilename->str, g_strerror(errno));
unlink(cfile->tmpfilename->str);
}
cache_etag_file_free(cfile);
}
/**********************************************************************************/
static void cache_etag_filter_free(vrequest *vr, filter *f) {
cache_etag_file *cfile = (cache_etag_file*) f->param;
UNUSED(vr);
cache_etag_file_free(cfile);
}
static handler_t cache_etag_filter_hit(vrequest *vr, filter *f) {
cache_etag_file *cfile = (cache_etag_file*) f->param;
UNUSED(vr);
if (!cfile) return HANDLER_GO_ON;
f->in->is_closed = TRUE;
chunkqueue_append_file_fd(f->out, NULL, 0, cfile->hit_length, cfile->hit_fd);
cfile->hit_fd = -1;
cache_etag_file_free(cfile);
f->param = NULL;
f->out->is_closed = TRUE;
return HANDLER_GO_ON;
}
static handler_t cache_etag_filter_miss(vrequest *vr, filter *f) {
cache_etag_file *cfile = (cache_etag_file*) f->param;
ssize_t res;
gchar *buf;
goffset buflen;
chunkiter citer = chunkqueue_iter(f->in);
UNUSED(vr);
if (0 == f->in->length) return HANDLER_GO_ON;
if (!cfile) { /* somehow we lost the file */
chunkqueue_steal_all(f->out, f->in);
if (f->in->is_closed) f->out->is_closed = TRUE;
return HANDLER_GO_ON;
}
if (HANDLER_GO_ON != chunkiter_read(vr, citer, 0, 64*1024, &buf, &buflen)) {
VR_ERROR(vr, "%s", "Couldn't read data from chunkqueue");
cache_etag_file_free(cfile);
f->param = NULL;
chunkqueue_steal_all(f->out, f->in);
if (f->in->is_closed) f->out->is_closed = TRUE;
return HANDLER_GO_ON;
}
res = write(cfile->fd, buf, buflen);
if (res < 0) {
switch (errno) {
case EINTR:
case EAGAIN:
break; /* come back later */
default:
VR_ERROR(vr, "Couldn't write to temporary cache file '%s': %s",
cfile->tmpfilename->str, g_strerror(errno));
cache_etag_file_free(cfile);
f->param = NULL;
chunkqueue_steal_all(f->out, f->in);
if (f->in->is_closed) f->out->is_closed = TRUE;
return HANDLER_GO_ON;
}
} else {
chunkqueue_steal_len(f->out, f->in, res);
if (f->in->length == 0 && f->in->is_closed) {
cache_etag_file_finish(vr, cfile);
f->param = NULL;
f->out->is_closed = TRUE;
return HANDLER_GO_ON;
}
}
return f->in->length ? HANDLER_COMEBACK : HANDLER_GO_ON;
}
static GString* createFileName(vrequest *vr, GString *path, http_header *etagheader) {
GString *file = g_string_sized_new(256);
gchar* etag_base64 = g_base64_encode(
etagheader->data->str + (etagheader->keylen + 2),
etagheader->data->len - (etagheader->keylen + 2));
g_string_append_len(file, GSTR_LEN(path));
g_string_append_len(file, GSTR_LEN(vr->request.uri.path));
g_string_append_len(file, CONST_STR_LEN("-"));
g_string_append(file, etag_base64);
g_free(etag_base64);
return file;
}
static handler_t cache_etag_cleanup(vrequest *vr, gpointer param, gpointer context) {
cache_etag_file *cfile = (cache_etag_file*) context;
UNUSED(vr);
UNUSED(param);
cache_etag_file_free(cfile);
return HANDLER_GO_ON;
}
static handler_t cache_etag_handle(vrequest *vr, gpointer param, gpointer *context) {
cache_etag_context *ctx = (cache_etag_context*) param;
cache_etag_file *cfile = (cache_etag_file*) *context;
GList *etag_entry;
http_header *etag;
struct stat st;
if (!cfile) {
if (vr->request.http_method != HTTP_METHOD_GET) return HANDLER_GO_ON;
VREQUEST_WAIT_FOR_RESPONSE_HEADERS(vr);
if (vr->response.http_status != 200) return HANDLER_GO_ON;
/* Don't cache static files */
if (vr->out->is_closed && 0 == vr->out->mem_usage) return HANDLER_GO_ON;
etag_entry = http_header_find_first(vr->response.headers, CONST_STR_LEN("etag"));
if (http_header_find_next(etag_entry, CONST_STR_LEN("etag"))) {
VR_ERROR(vr, "%s", "duplicate etag header in response, will not cache it");
return HANDLER_GO_ON;
}
etag = (http_header*) etag_entry->data;
cfile = cache_etag_file_create(createFileName(vr, ctx->path, etag));
*context = cfile;
}
/* TODO use async stat cache*/
if (0 == stat(cfile->filename->str, &st)) {
if (!S_ISREG(st.st_mode)) {
VR_ERROR(vr, "Unexpected file type for cache file '%s' (mode %o)", cfile->filename->str, (unsigned int) st.st_mode);
return HANDLER_GO_ON; /* no caching */
}
if (-1 == (cfile->hit_fd = open(cfile->filename->str, O_RDONLY))) {
if (EMFILE == errno) {
server_out_of_fds(vr->con->srv);
}
VR_ERROR(vr, "Couldn't open cache file '%s': %s", cfile->filename->str, g_strerror(errno));
return HANDLER_GO_ON; /* no caching */
}
#ifdef FD_CLOEXEC
fcntl(cfile->hit_fd, F_SETFD, FD_CLOEXEC);
#endif
if (CORE_OPTION(CORE_OPTION_DEBUG_REQUEST_HANDLING).boolean) {
VR_DEBUG(vr, "cache hit for '%s'", vr->request.uri.path->str);
}
cfile->hit_length = st.st_size;
vrequest_add_filter_out(vr, cache_etag_filter_hit, cache_etag_filter_free, cfile);
*context = NULL;
return HANDLER_GO_ON;
}
if (CORE_OPTION(CORE_OPTION_DEBUG_REQUEST_HANDLING).boolean) {
VR_DEBUG(vr, "cache miss for '%s'", vr->request.uri.path->str);
}
if (!cache_etag_file_start(vr, cfile)) {
cache_etag_file_free(cfile);
return HANDLER_GO_ON; /* no caching */
}
vrequest_add_filter_out(vr, cache_etag_filter_miss, cache_etag_filter_free, cfile);
*context = NULL;
return HANDLER_GO_ON;
}
static void cache_etag_free(server *srv, gpointer param) {
cache_etag_context *ctx = (cache_etag_context*) param;
UNUSED(srv);
g_string_free(ctx->path, TRUE);
g_slice_free(cache_etag_context, ctx);
}
static action* cache_etag_create(server *srv, plugin* p, value *val) {
cache_etag_context *ctx;
UNUSED(p);
if (val->type != VALUE_STRING) {
ERROR(srv, "%s", "cache.disk.etag expects a string as parameter");
return FALSE;
}
ctx = g_slice_new0(cache_etag_context);
ctx->path = value_extract_ptr(val);
return action_new_function(cache_etag_handle, cache_etag_cleanup, cache_etag_free, ctx);
}
static const plugin_option options[] = {
{ NULL, 0, NULL, NULL, NULL }
};
static const plugin_action actions[] = {
{ "cache.disk.etag", cache_etag_create },
{ NULL, NULL }
};
static const plugin_setup setups[] = {
{ NULL, NULL }
};
static void plugin_init(server *srv, plugin *p) {
UNUSED(srv);
p->options = options;
p->actions = actions;
p->setups = setups;
}
gboolean mod_cache_disk_etag_init(modules *mods, module *mod) {
MODULE_VERSION_CHECK(mods);
mod->config = plugin_register(mods->main, "mod_cache_disk_etag", plugin_init);
return mod->config != NULL;
}
gboolean mod_cache_disk_etag_free(modules *mods, module *mod) {
if (mod->config)
plugin_free(mods->main, mod->config);
return TRUE;
}

@ -8,10 +8,11 @@ static void filters_init(filters *fs) {
fs->out = chunkqueue_new();
}
static void filters_clean(filters *fs) {
static void filters_clean(vrequest *vr, filters *fs) {
guint i;
for (i = 0; i < fs->queue->len; i++) {
filter *f = (filter*) g_ptr_array_index(fs->queue, i);
if (f->handle_free && f->param) f->handle_free(vr, f);
if (i > 0) chunkqueue_free(fs->in);
g_slice_free(filter, f);
}
@ -20,11 +21,12 @@ static void filters_clean(filters *fs) {
chunkqueue_free(fs->out);
}
static void filters_reset(filters *fs) {
static void filters_reset(vrequest *vr, filters *fs) {
guint i;
fs->skip_ndx = 0;
for (i = 0; i < fs->queue->len; i++) {
filter *f = (filter*) g_ptr_array_index(fs->queue, i);
if (f->handle_free && f->param) f->handle_free(vr, f);
if (i > 0) chunkqueue_free(fs->in);
g_slice_free(filter, f);
}
@ -42,7 +44,7 @@ static gboolean filters_run(vrequest *vr, filters *fs) {
}
for (i = 0; i < fs->queue->len; i++) {
filter *f = (filter*) g_ptr_array_index(fs->queue, i);
switch (f->handle(vr, f, f->p)) {
switch (f->handle_data(vr, f)) {
case HANDLER_GO_ON:
break;
case HANDLER_COMEBACK:
@ -72,11 +74,12 @@ static gboolean filters_run(vrequest *vr, filters *fs) {
return TRUE;
}
static void filters_add(filters *fs, plugin *p, filter_handler handle) {
static void filters_add(filters *fs, filter_handler handle_data, filter_free handle_free, gpointer param) {
filter *f = g_slice_new0(filter);
f->out = fs->out;
f->p = p;
f->handle = handle;
f->param = param;
f->handle_data = handle_data;
f->handle_free = handle_free;
if (0 == fs->queue->len) {
f->in = fs->in;
} else {
@ -87,12 +90,12 @@ static void filters_add(filters *fs, plugin *p, filter_handler handle) {
g_ptr_array_add(fs->queue, f);
}
void vrequest_add_filter_in(vrequest *vr, plugin *p, filter_handler handle) {
filters_add(&vr->filters_in, p, handle);
void vrequest_add_filter_in(vrequest *vr, filter_handler handle_data, filter_free handle_free, gpointer param) {
filters_add(&vr->filters_in, handle_data, handle_free, param);
}
void vrequest_add_filter_out(vrequest *vr, plugin *p, filter_handler handle) {
filters_add(&vr->filters_out, p, handle);
void vrequest_add_filter_out(vrequest *vr, filter_handler handle_data, filter_free handle_free, gpointer param) {
filters_add(&vr->filters_out, handle_data, handle_free, param);
}
vrequest* vrequest_new(connection *con, vrequest_handler handle_response_headers, vrequest_handler handle_response_body, vrequest_handler handle_response_error, vrequest_handler handle_request_headers) {
@ -138,8 +141,8 @@ void vrequest_free(vrequest* vr) {
response_clear(&vr->response);
environment_clear(&vr->env);
filters_clean(&vr->filters_in);
filters_clean(&vr->filters_out);
filters_clean(vr, &vr->filters_in);
filters_clean(vr, &vr->filters_out);
if (vr->job_queue_link) {
g_queue_delete_link(&vr->con->wrk->job_queue, vr->job_queue_link);
@ -169,8 +172,8 @@ void vrequest_reset(vrequest *vr) {
response_reset(&vr->response);
environment_reset(&vr->env);
filters_reset(&vr->filters_in);
filters_reset(&vr->filters_out);
filters_reset(vr, &vr->filters_in);
filters_reset(vr, &vr->filters_out);
if (vr->job_queue_link) {
g_queue_delete_link(&vr->con->wrk->job_queue, vr->job_queue_link);

@ -114,6 +114,7 @@ def build(bld):
lighty_mod(bld, 'mod_accesslog', 'modules/mod_accesslog.c')
lighty_mod(bld, 'mod_balancer', 'modules/mod_balancer.c')
lighty_mod(bld, 'mod_cache_disk_etag', 'modules/mod_cache_disk_etag.c')
lighty_mod(bld, 'mod_dirlist', 'modules/mod_dirlist.c')
lighty_mod(bld, 'mod_fastcgi', 'modules/mod_fastcgi.c')
lighty_mod(bld, 'mod_fortune', 'modules/mod_fortune.c')

Loading…
Cancel
Save