add stat_cach_get_dir() to get directory listing; add stat_cache.ttl setup

personal/stbuehler/wip
Thomas Porzelt 14 years ago
parent 004dfc1bf6
commit 4d2cf580df
  1. 2
      include/lighttpd/server.h
  2. 39
      include/lighttpd/stat_cache.h
  3. 2
      include/lighttpd/typedefs.h
  4. 27
      src/plugin_core.c
  5. 179
      src/stat_cache.c
  6. 2
      src/worker.c

@ -75,6 +75,8 @@ struct server {
guint keep_alive_queue_timeout;
gdouble io_timeout;
gdouble stat_cache_ttl;
};

@ -39,28 +39,39 @@
#error Please include <lighttpd/base.h> instead of this file
#endif
struct stat_cache_entry {
struct stat_cache_entry_data {
GString *path;
GString *etag;
GString *content_type;
gboolean failed;
struct stat st;
ev_tstamp ts; /* timestamp the entry was created (not when the stat() was done) */
gint err;
gboolean failed;
};
struct stat_cache_entry {
enum {
STAT_CACHE_ENTRY_WAITING, /* waiting for stat thread to do the work, no info available */
STAT_CACHE_ENTRY_FINISHED, /* stat() done, info available */
STAT_CACHE_ENTRY_SINGLE, /* single file, this is the default or "normal" */
STAT_CACHE_ENTRY_DIR /* get a directory listing (with stat info) */
} type;
enum {
STAT_CACHE_ENTRY_WAITING, /* waiting for stat thread to do the work, no info available */
STAT_CACHE_ENTRY_FINISHED, /* stat() done, info available */
} state;
GPtrArray *vrequests; /* vrequests waiting for this info */
stat_cache_entry_data data;
GArray *dirlist; /* array of stat_cache_dirlist_entry, used together with STAT_CACHE_ENTRY_DIR */
ev_tstamp ts; /* timestamp the entry was created (not when the stat() was done) */
GPtrArray *vrequests; /* vrequests waiting for this info */
guint refcount;
waitqueue_elem queue_elem; /* queue element for the delete_queue */
waitqueue_elem queue_elem; /* queue element for the delete_queue */
gboolean in_cache;
};
struct stat_cache {
GHashTable *entries;
GAsyncQueue *job_queue_out; /* elements waiting for stat */
GAsyncQueue *job_queue_in; /* elements with finished stat */
GAsyncQueue *job_queue_out; /* elements waiting for stat */
GAsyncQueue *job_queue_in; /* elements with finished stat */
waitqueue delete_queue;
GThread *thread;
ev_async job_watcher;
@ -73,17 +84,15 @@ struct stat_cache {
void stat_cache_new(worker *wrk, gdouble ttl);
void stat_cache_free(stat_cache *sc);
void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents);
void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents);
gpointer stat_cache_thread(gpointer data);
void stat_cache_entry_free(stat_cache_entry *sce);
/*
gets a stat_cache_entry for a specified path
returns NULL in case of a cache MISS and you should return HANDLER_WAIT_FOR_EVENT
*/
LI_API stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path);
LI_API stat_cache_entry *stat_cache_get(vrequest *vr, GString *path);
/* like stat_cache_entry_get but gets stat info for a whole directory */
LI_API stat_cache_entry *stat_cache_get_dir(vrequest *vr, GString *path);
/* release a stat_cache_entry so it can be cleaned up */
LI_API void stat_cache_entry_release(vrequest *vr);

@ -204,6 +204,8 @@ typedef struct filters filters;
struct worker;
typedef struct worker worker;
struct stat_cache_entry_data;
typedef struct stat_cache_entry_data stat_cache_entry_data;
struct stat_cache_entry;
typedef struct stat_cache_entry stat_cache_entry;
struct stat_cache;

@ -197,15 +197,15 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
if (!vrequest_handle_direct(vr)) return HANDLER_GO_ON;
}
sce = stat_cache_entry_get(vr, vr->physical.path);
sce = stat_cache_get(vr, vr->physical.path);
if (!sce)
return HANDLER_WAIT_FOR_EVENT;
VR_DEBUG(vr, "serving static file: %s", vr->physical.path->str);
if (sce->failed) {
if (sce->data.failed) {
/* stat failed */
VR_DEBUG(vr, "stat() failed: %s (%d)", g_strerror(sce->err), sce->err);
VR_DEBUG(vr, "stat() failed: %s (%d)", g_strerror(sce->data.err), sce->data.err);
switch (errno) {
case ENOENT:
@ -237,7 +237,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
/* redirect to scheme + host + path + / + querystring if directory without trailing slash */
/* TODO: local addr if HTTP 1.0 without host header */
if (S_ISDIR(sce->st.st_mode) && vr->request.uri.orig_path->str[vr->request.uri.orig_path->len-1] != '/') {
if (S_ISDIR(sce->data.st.st_mode) && vr->request.uri.orig_path->str[vr->request.uri.orig_path->len-1] != '/') {
GString *host = vr->request.uri.authority->len ? vr->request.uri.authority : vr->con->local_addr_str;
GString *uri = g_string_sized_new(
8 /* https:// */ + host->len +
@ -259,7 +259,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
http_header_overwrite(vr->response.headers, CONST_STR_LEN("Location"), GSTR_LEN(uri));
g_string_free(uri, TRUE);
close(fd);
} else if (!S_ISREG(sce->st.st_mode)) {
} else if (!S_ISREG(sce->data.st.st_mode)) {
vr->response.http_status = 404;
close(fd);
} else {
@ -269,7 +269,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
http_header_overwrite(vr->response.headers, CONST_STR_LEN("Content-Type"), GSTR_LEN(mime_str));
else
http_header_overwrite(vr->response.headers, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("application/octet-stream"));
chunkqueue_append_file_fd(vr->out, NULL, 0, sce->st.st_size, fd);
chunkqueue_append_file_fd(vr->out, NULL, 0, sce->data.st.st_size, fd);
}
}
@ -579,6 +579,19 @@ static gboolean core_io_timeout(server *srv, plugin* p, value *val) {
return TRUE;
}
static gboolean core_stat_cache_ttl(server *srv, plugin* p, value *val) {
UNUSED(p);
if (!val || val->type != VALUE_NUMBER || val->data.number < 1) {
ERROR(srv, "%s", "stat_cache.ttl expects a positive number as parameter");
return FALSE;
}
srv->stat_cache_ttl = (gdouble)value_extract(val).number;
return TRUE;
}
/*
* OPTIONS
*/
@ -1081,6 +1094,8 @@ static const plugin_setup setups[] = {
{ "workers", core_workers },
{ "module_load", core_module_load },
{ "io_timeout", core_io_timeout },
{ "stat_cache.ttl", core_stat_cache_ttl },
{ NULL, NULL }
};

@ -1,10 +1,20 @@
#include <lighttpd/base.h>
static void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents);
static void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents);
static gpointer stat_cache_thread(gpointer data);
static void stat_cache_entry_free(stat_cache_entry *sce);
static stat_cache_entry *stat_cache_get_internal(vrequest *vr, GString *path, gboolean dir);
void stat_cache_new(worker *wrk, gdouble ttl) {
stat_cache *sc;
GError *err;
/* ttl default 10s */
if (ttl < 1)
ttl = 10.0;
sc = g_slice_new0(stat_cache);
sc->ttl = ttl;
sc->entries = g_hash_table_new_full((GHashFunc)g_string_hash, (GEqualFunc)g_string_equal, NULL, NULL);
@ -51,7 +61,7 @@ void stat_cache_free(stat_cache *sc) {
g_slice_free(stat_cache, sc);
}
void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
static void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
stat_cache *sc = (stat_cache*) w->data;
stat_cache_entry *sce;
waitqueue_elem *wqe;
@ -68,7 +78,7 @@ void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
} else {
/* no more vrequests using this entry, finally free it */
if (sce->in_cache)
g_hash_table_remove(sc->entries, sce->path);
g_hash_table_remove(sc->entries, sce->data.path);
stat_cache_entry_free(sce);
}
}
@ -76,7 +86,7 @@ void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
waitqueue_update(&sc->delete_queue);
}
void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
static void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
guint i;
stat_cache_entry *sce;
stat_cache *sc = ((worker*)w->data)->stat_cache;
@ -86,7 +96,7 @@ void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
UNUSED(revents);
while ((sce = g_async_queue_try_pop(sc->job_queue_in)) != NULL) {
if (sce->failed)
if (sce->data.failed)
sc->errors++;
for (i = 0; i < sce->vrequests->len; i++) {
@ -98,16 +108,95 @@ void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
}
}
void stat_cache_entry_free(stat_cache_entry *sce) {
static void stat_cache_entry_free(stat_cache_entry *sce) {
guint i;
assert(sce->vrequests->len == 0);
assert(sce->refcount == 0);
g_string_free(sce->path, TRUE);
g_string_free(sce->data.path, TRUE);
g_ptr_array_free(sce->vrequests, TRUE);
if (sce->type == STAT_CACHE_ENTRY_DIR) {
for (i = 0; i < sce->dirlist->len; i++) {
g_string_free(g_array_index(sce->dirlist, stat_cache_entry_data, i).path, TRUE);
}
g_array_free(sce->dirlist, TRUE);
}
g_slice_free(stat_cache_entry, sce);
}
stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
static gpointer stat_cache_thread(gpointer data) {
stat_cache *sc = data;
stat_cache_entry *sce;
while (TRUE) {
sce = g_async_queue_pop(sc->job_queue_out);
/* stat cache entry with path == NULL indicates server stop */
if (!sce->data.path)
break;
if (sce->type == STAT_CACHE_ENTRY_SINGLE) {
if (stat(sce->data.path->str, &sce->data.st) == -1) {
sce->data.failed = TRUE;
sce->data.err = errno;
} else {
sce->data.failed = FALSE;
}
} else {
/* dirlisting */
DIR *dirp;
gsize size;
struct dirent *entry;
struct dirent *result;
gint error;
stat_cache_entry_data sced;
dirp = opendir(sce->data.path->str);
if (dirp == NULL) {
sce->data.failed = TRUE;
sce->data.err = errno;
} else {
size = dirent_buf_size(dirp);
assert(size != (gsize)-1);
entry = g_slice_alloc(size);
while ((error = readdir_r(dirp, entry, &result)) != 0 && result != NULL) {
sced.path = g_string_sized_new(32);
g_string_assign(sced.path, result->d_name);
if (stat(result->d_name, &sced.st) == -1) {
sced.failed = TRUE;
sced.err = errno;
} else {
sced.failed = FALSE;
}
g_array_append_val(sce->dirlist, sced);
}
if (error) {
sce->data.failed = TRUE;
sce->data.err = error;
}
g_slice_free1(size, entry);
closedir(dirp);
}
}
g_atomic_int_set(&sce->state, STAT_CACHE_ENTRY_FINISHED);
g_async_queue_push(sc->job_queue_in, sce);
ev_async_send(sc->delete_queue.loop, &sc->job_watcher);
}
return NULL;
}
static stat_cache_entry *stat_cache_get_internal(vrequest *vr, GString *path, gboolean dir) {
stat_cache *sc;
stat_cache_entry *sce;
@ -127,20 +216,46 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
vr->stat_cache_entry = sce;
sce->refcount++;
}
VR_DEBUG(vr, "stat_cache: %"G_GUINT64_FORMAT" hits, %"G_GUINT64_FORMAT" misses, %u items in cache", sc->hits, sc->misses, g_hash_table_size(sc->entries));
return sce;
} else {
/* entry old */
if (sce->refcount == 0) {
/* no vrequests working on the entry, reuse it */
if (sce->type == STAT_CACHE_ENTRY_DIR) {
if (!dir) {
guint i;
for (i = 0; i < sce->dirlist->len; i++) {
g_string_free(g_array_index(sce->dirlist, stat_cache_entry_data, i).path, TRUE);
}
g_array_free(sce->dirlist, TRUE);
sce->type = STAT_CACHE_ENTRY_SINGLE;
} else {
g_array_set_size(sce->dirlist, 0);
}
} else {
/* single file */
if (dir) {
sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32);
sce->type = STAT_CACHE_ENTRY_DIR;
}
}
} else {
/* there are still vrequests using this entry, replace with a new one */
sce->in_cache = FALSE;
sce = g_slice_new0(stat_cache_entry);
sce->path = g_string_new_len(GSTR_LEN(path));
sce->data.path = g_string_new_len(GSTR_LEN(path));
sce->vrequests = g_ptr_array_sized_new(8);
sce->in_cache = TRUE;
sce->queue_elem.data = sce;
g_hash_table_replace(sc->entries, sce->path, sce);
g_hash_table_replace(sc->entries, sce->data.path, sce);
if (dir) {
sce->type = STAT_CACHE_ENTRY_DIR;
sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32);
} else {
sce->type = STAT_CACHE_ENTRY_SINGLE;
}
}
sce->ts = CUR_TS(vr->con->wrk);
@ -164,7 +279,7 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
} else {
/* cache miss, allocate new entry */
sce = g_slice_new0(stat_cache_entry);
sce->path = g_string_new_len(GSTR_LEN(path));
sce->data.path = g_string_new_len(GSTR_LEN(path));
sce->vrequests = g_ptr_array_sized_new(8);
sce->ts = CUR_TS(vr->con->wrk);
sce->state = STAT_CACHE_ENTRY_WAITING;
@ -174,40 +289,30 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
g_ptr_array_add(sce->vrequests, vr);
sce->refcount = 1;
waitqueue_push(&sc->delete_queue, &sce->queue_elem);
g_hash_table_insert(sc->entries, sce->path, sce);
g_hash_table_insert(sc->entries, sce->data.path, sce);
g_async_queue_push(sc->job_queue_out, sce);
sc->misses++;
if (dir) {
sce->type = STAT_CACHE_ENTRY_DIR;
sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32);
} else {
sce->type = STAT_CACHE_ENTRY_SINGLE;
}
return NULL;
}
}
void stat_cache_entry_release(vrequest *vr) {
vr->stat_cache_entry->refcount--;
vr->stat_cache_entry = NULL;
stat_cache_entry *stat_cache_get(vrequest *vr, GString *path) {
return stat_cache_get_internal(vr, path, FALSE);
}
stat_cache_entry *stat_cache_get_dir(vrequest *vr, GString *path) {
return stat_cache_get_internal(vr, path, TRUE);
}
gpointer stat_cache_thread(gpointer data) {
stat_cache *sc = data;
stat_cache_entry *sce;
while (TRUE) {
sce = g_async_queue_pop(sc->job_queue_out);
/* stat cache entry with path == NULL indicates server stop */
if (!sce->path)
break;
if (stat(sce->path->str, &sce->st) == -1) {
sce->failed = TRUE;
sce->err = errno;
} else
sce->failed = FALSE;
g_atomic_int_set(&sce->state, STAT_CACHE_ENTRY_FINISHED);
g_async_queue_push(sc->job_queue_in, sce);
ev_async_send(sc->delete_queue.loop, &sc->job_watcher);
}
return NULL;
void stat_cache_entry_release(vrequest *vr) {
vr->stat_cache_entry->refcount--;
vr->stat_cache_entry = NULL;
}

@ -335,7 +335,7 @@ worker* worker_new(struct server *srv, struct ev_loop *loop) {
ev_timer_init(&wrk->job_queue_watcher, worker_job_queue_cb, 0, 0);
wrk->job_queue_watcher.data = wrk;
stat_cache_new(wrk, 10.0);
stat_cache_new(wrk, srv->stat_cache_ttl);
return wrk;
}

Loading…
Cancel
Save