add stat_cach_get_dir() to get directory listing; add stat_cache.ttl setup
Thomas Porzelt
parent
004dfc1bf6
commit
4d2cf580df
|
|
@ -75,6 +75,8 @@ struct server {
|
|||
guint keep_alive_queue_timeout;
|
||||
|
||||
gdouble io_timeout;
|
||||
|
||||
gdouble stat_cache_ttl;
|
||||
};
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -39,28 +39,39 @@
|
|||
#error Please include <lighttpd/base.h> instead of this file
|
||||
#endif
|
||||
|
||||
struct stat_cache_entry {
|
||||
struct stat_cache_entry_data {
|
||||
GString *path;
|
||||
GString *etag;
|
||||
GString *content_type;
|
||||
struct stat st;
|
||||
ev_tstamp ts; /* timestamp the entry was created (not when the stat() was done) */
|
||||
gint err;
|
||||
gboolean failed;
|
||||
struct stat st;
|
||||
gint err;
|
||||
};
|
||||
|
||||
struct stat_cache_entry {
|
||||
enum {
|
||||
STAT_CACHE_ENTRY_WAITING, /* waiting for stat thread to do the work, no info available */
|
||||
STAT_CACHE_ENTRY_FINISHED, /* stat() done, info available */
|
||||
STAT_CACHE_ENTRY_SINGLE, /* single file, this is the default or "normal" */
|
||||
STAT_CACHE_ENTRY_DIR /* get a directory listing (with stat info) */
|
||||
} type;
|
||||
|
||||
enum {
|
||||
STAT_CACHE_ENTRY_WAITING, /* waiting for stat thread to do the work, no info available */
|
||||
STAT_CACHE_ENTRY_FINISHED, /* stat() done, info available */
|
||||
} state;
|
||||
GPtrArray *vrequests; /* vrequests waiting for this info */
|
||||
|
||||
stat_cache_entry_data data;
|
||||
GArray *dirlist; /* array of stat_cache_dirlist_entry, used together with STAT_CACHE_ENTRY_DIR */
|
||||
ev_tstamp ts; /* timestamp the entry was created (not when the stat() was done) */
|
||||
GPtrArray *vrequests; /* vrequests waiting for this info */
|
||||
guint refcount;
|
||||
waitqueue_elem queue_elem; /* queue element for the delete_queue */
|
||||
waitqueue_elem queue_elem; /* queue element for the delete_queue */
|
||||
gboolean in_cache;
|
||||
};
|
||||
|
||||
struct stat_cache {
|
||||
GHashTable *entries;
|
||||
GAsyncQueue *job_queue_out; /* elements waiting for stat */
|
||||
GAsyncQueue *job_queue_in; /* elements with finished stat */
|
||||
GAsyncQueue *job_queue_out; /* elements waiting for stat */
|
||||
GAsyncQueue *job_queue_in; /* elements with finished stat */
|
||||
waitqueue delete_queue;
|
||||
GThread *thread;
|
||||
ev_async job_watcher;
|
||||
|
|
@ -73,17 +84,15 @@ struct stat_cache {
|
|||
|
||||
void stat_cache_new(worker *wrk, gdouble ttl);
|
||||
void stat_cache_free(stat_cache *sc);
|
||||
void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents);
|
||||
void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents);
|
||||
gpointer stat_cache_thread(gpointer data);
|
||||
|
||||
void stat_cache_entry_free(stat_cache_entry *sce);
|
||||
|
||||
/*
|
||||
gets a stat_cache_entry for a specified path
|
||||
returns NULL in case of a cache MISS and you should return HANDLER_WAIT_FOR_EVENT
|
||||
*/
|
||||
LI_API stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path);
|
||||
LI_API stat_cache_entry *stat_cache_get(vrequest *vr, GString *path);
|
||||
|
||||
/* like stat_cache_entry_get but gets stat info for a whole directory */
|
||||
LI_API stat_cache_entry *stat_cache_get_dir(vrequest *vr, GString *path);
|
||||
|
||||
/* release a stat_cache_entry so it can be cleaned up */
|
||||
LI_API void stat_cache_entry_release(vrequest *vr);
|
||||
|
|
|
|||
|
|
@ -204,6 +204,8 @@ typedef struct filters filters;
|
|||
struct worker;
|
||||
typedef struct worker worker;
|
||||
|
||||
struct stat_cache_entry_data;
|
||||
typedef struct stat_cache_entry_data stat_cache_entry_data;
|
||||
struct stat_cache_entry;
|
||||
typedef struct stat_cache_entry stat_cache_entry;
|
||||
struct stat_cache;
|
||||
|
|
|
|||
|
|
@ -197,15 +197,15 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
|
|||
if (!vrequest_handle_direct(vr)) return HANDLER_GO_ON;
|
||||
}
|
||||
|
||||
sce = stat_cache_entry_get(vr, vr->physical.path);
|
||||
sce = stat_cache_get(vr, vr->physical.path);
|
||||
if (!sce)
|
||||
return HANDLER_WAIT_FOR_EVENT;
|
||||
|
||||
VR_DEBUG(vr, "serving static file: %s", vr->physical.path->str);
|
||||
|
||||
if (sce->failed) {
|
||||
if (sce->data.failed) {
|
||||
/* stat failed */
|
||||
VR_DEBUG(vr, "stat() failed: %s (%d)", g_strerror(sce->err), sce->err);
|
||||
VR_DEBUG(vr, "stat() failed: %s (%d)", g_strerror(sce->data.err), sce->data.err);
|
||||
|
||||
switch (errno) {
|
||||
case ENOENT:
|
||||
|
|
@ -237,7 +237,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
|
|||
|
||||
/* redirect to scheme + host + path + / + querystring if directory without trailing slash */
|
||||
/* TODO: local addr if HTTP 1.0 without host header */
|
||||
if (S_ISDIR(sce->st.st_mode) && vr->request.uri.orig_path->str[vr->request.uri.orig_path->len-1] != '/') {
|
||||
if (S_ISDIR(sce->data.st.st_mode) && vr->request.uri.orig_path->str[vr->request.uri.orig_path->len-1] != '/') {
|
||||
GString *host = vr->request.uri.authority->len ? vr->request.uri.authority : vr->con->local_addr_str;
|
||||
GString *uri = g_string_sized_new(
|
||||
8 /* https:// */ + host->len +
|
||||
|
|
@ -259,7 +259,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
|
|||
http_header_overwrite(vr->response.headers, CONST_STR_LEN("Location"), GSTR_LEN(uri));
|
||||
g_string_free(uri, TRUE);
|
||||
close(fd);
|
||||
} else if (!S_ISREG(sce->st.st_mode)) {
|
||||
} else if (!S_ISREG(sce->data.st.st_mode)) {
|
||||
vr->response.http_status = 404;
|
||||
close(fd);
|
||||
} else {
|
||||
|
|
@ -269,7 +269,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
|
|||
http_header_overwrite(vr->response.headers, CONST_STR_LEN("Content-Type"), GSTR_LEN(mime_str));
|
||||
else
|
||||
http_header_overwrite(vr->response.headers, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("application/octet-stream"));
|
||||
chunkqueue_append_file_fd(vr->out, NULL, 0, sce->st.st_size, fd);
|
||||
chunkqueue_append_file_fd(vr->out, NULL, 0, sce->data.st.st_size, fd);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -579,6 +579,19 @@ static gboolean core_io_timeout(server *srv, plugin* p, value *val) {
|
|||
return TRUE;
|
||||
}
|
||||
|
||||
static gboolean core_stat_cache_ttl(server *srv, plugin* p, value *val) {
|
||||
UNUSED(p);
|
||||
|
||||
if (!val || val->type != VALUE_NUMBER || val->data.number < 1) {
|
||||
ERROR(srv, "%s", "stat_cache.ttl expects a positive number as parameter");
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
srv->stat_cache_ttl = (gdouble)value_extract(val).number;
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
/*
|
||||
* OPTIONS
|
||||
*/
|
||||
|
|
@ -1081,6 +1094,8 @@ static const plugin_setup setups[] = {
|
|||
{ "workers", core_workers },
|
||||
{ "module_load", core_module_load },
|
||||
{ "io_timeout", core_io_timeout },
|
||||
{ "stat_cache.ttl", core_stat_cache_ttl },
|
||||
|
||||
{ NULL, NULL }
|
||||
};
|
||||
|
||||
|
|
|
|||
179
src/stat_cache.c
179
src/stat_cache.c
|
|
@ -1,10 +1,20 @@
|
|||
#include <lighttpd/base.h>
|
||||
|
||||
|
||||
static void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents);
|
||||
static void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents);
|
||||
static gpointer stat_cache_thread(gpointer data);
|
||||
static void stat_cache_entry_free(stat_cache_entry *sce);
|
||||
static stat_cache_entry *stat_cache_get_internal(vrequest *vr, GString *path, gboolean dir);
|
||||
|
||||
void stat_cache_new(worker *wrk, gdouble ttl) {
|
||||
stat_cache *sc;
|
||||
GError *err;
|
||||
|
||||
/* ttl default 10s */
|
||||
if (ttl < 1)
|
||||
ttl = 10.0;
|
||||
|
||||
sc = g_slice_new0(stat_cache);
|
||||
sc->ttl = ttl;
|
||||
sc->entries = g_hash_table_new_full((GHashFunc)g_string_hash, (GEqualFunc)g_string_equal, NULL, NULL);
|
||||
|
|
@ -51,7 +61,7 @@ void stat_cache_free(stat_cache *sc) {
|
|||
g_slice_free(stat_cache, sc);
|
||||
}
|
||||
|
||||
void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
|
||||
static void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
|
||||
stat_cache *sc = (stat_cache*) w->data;
|
||||
stat_cache_entry *sce;
|
||||
waitqueue_elem *wqe;
|
||||
|
|
@ -68,7 +78,7 @@ void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
|
|||
} else {
|
||||
/* no more vrequests using this entry, finally free it */
|
||||
if (sce->in_cache)
|
||||
g_hash_table_remove(sc->entries, sce->path);
|
||||
g_hash_table_remove(sc->entries, sce->data.path);
|
||||
stat_cache_entry_free(sce);
|
||||
}
|
||||
}
|
||||
|
|
@ -76,7 +86,7 @@ void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
|
|||
waitqueue_update(&sc->delete_queue);
|
||||
}
|
||||
|
||||
void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
|
||||
static void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
|
||||
guint i;
|
||||
stat_cache_entry *sce;
|
||||
stat_cache *sc = ((worker*)w->data)->stat_cache;
|
||||
|
|
@ -86,7 +96,7 @@ void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
|
|||
UNUSED(revents);
|
||||
|
||||
while ((sce = g_async_queue_try_pop(sc->job_queue_in)) != NULL) {
|
||||
if (sce->failed)
|
||||
if (sce->data.failed)
|
||||
sc->errors++;
|
||||
|
||||
for (i = 0; i < sce->vrequests->len; i++) {
|
||||
|
|
@ -98,16 +108,95 @@ void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
|
|||
}
|
||||
}
|
||||
|
||||
void stat_cache_entry_free(stat_cache_entry *sce) {
|
||||
static void stat_cache_entry_free(stat_cache_entry *sce) {
|
||||
guint i;
|
||||
|
||||
assert(sce->vrequests->len == 0);
|
||||
assert(sce->refcount == 0);
|
||||
g_string_free(sce->path, TRUE);
|
||||
|
||||
g_string_free(sce->data.path, TRUE);
|
||||
g_ptr_array_free(sce->vrequests, TRUE);
|
||||
|
||||
if (sce->type == STAT_CACHE_ENTRY_DIR) {
|
||||
for (i = 0; i < sce->dirlist->len; i++) {
|
||||
g_string_free(g_array_index(sce->dirlist, stat_cache_entry_data, i).path, TRUE);
|
||||
}
|
||||
|
||||
g_array_free(sce->dirlist, TRUE);
|
||||
}
|
||||
|
||||
g_slice_free(stat_cache_entry, sce);
|
||||
}
|
||||
|
||||
|
||||
stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
|
||||
static gpointer stat_cache_thread(gpointer data) {
|
||||
stat_cache *sc = data;
|
||||
stat_cache_entry *sce;
|
||||
|
||||
while (TRUE) {
|
||||
sce = g_async_queue_pop(sc->job_queue_out);
|
||||
|
||||
/* stat cache entry with path == NULL indicates server stop */
|
||||
if (!sce->data.path)
|
||||
break;
|
||||
|
||||
if (sce->type == STAT_CACHE_ENTRY_SINGLE) {
|
||||
if (stat(sce->data.path->str, &sce->data.st) == -1) {
|
||||
sce->data.failed = TRUE;
|
||||
sce->data.err = errno;
|
||||
} else {
|
||||
sce->data.failed = FALSE;
|
||||
}
|
||||
} else {
|
||||
/* dirlisting */
|
||||
DIR *dirp;
|
||||
gsize size;
|
||||
struct dirent *entry;
|
||||
struct dirent *result;
|
||||
gint error;
|
||||
stat_cache_entry_data sced;
|
||||
|
||||
dirp = opendir(sce->data.path->str);
|
||||
if (dirp == NULL) {
|
||||
sce->data.failed = TRUE;
|
||||
sce->data.err = errno;
|
||||
} else {
|
||||
size = dirent_buf_size(dirp);
|
||||
assert(size != (gsize)-1);
|
||||
entry = g_slice_alloc(size);
|
||||
|
||||
|
||||
while ((error = readdir_r(dirp, entry, &result)) != 0 && result != NULL) {
|
||||
sced.path = g_string_sized_new(32);
|
||||
g_string_assign(sced.path, result->d_name);
|
||||
if (stat(result->d_name, &sced.st) == -1) {
|
||||
sced.failed = TRUE;
|
||||
sced.err = errno;
|
||||
} else {
|
||||
sced.failed = FALSE;
|
||||
}
|
||||
g_array_append_val(sce->dirlist, sced);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
sce->data.failed = TRUE;
|
||||
sce->data.err = error;
|
||||
}
|
||||
|
||||
g_slice_free1(size, entry);
|
||||
closedir(dirp);
|
||||
}
|
||||
}
|
||||
|
||||
g_atomic_int_set(&sce->state, STAT_CACHE_ENTRY_FINISHED);
|
||||
g_async_queue_push(sc->job_queue_in, sce);
|
||||
ev_async_send(sc->delete_queue.loop, &sc->job_watcher);
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static stat_cache_entry *stat_cache_get_internal(vrequest *vr, GString *path, gboolean dir) {
|
||||
stat_cache *sc;
|
||||
stat_cache_entry *sce;
|
||||
|
||||
|
|
@ -127,20 +216,46 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
|
|||
vr->stat_cache_entry = sce;
|
||||
sce->refcount++;
|
||||
}
|
||||
VR_DEBUG(vr, "stat_cache: %"G_GUINT64_FORMAT" hits, %"G_GUINT64_FORMAT" misses, %u items in cache", sc->hits, sc->misses, g_hash_table_size(sc->entries));
|
||||
return sce;
|
||||
} else {
|
||||
/* entry old */
|
||||
if (sce->refcount == 0) {
|
||||
/* no vrequests working on the entry, reuse it */
|
||||
if (sce->type == STAT_CACHE_ENTRY_DIR) {
|
||||
if (!dir) {
|
||||
guint i;
|
||||
for (i = 0; i < sce->dirlist->len; i++) {
|
||||
g_string_free(g_array_index(sce->dirlist, stat_cache_entry_data, i).path, TRUE);
|
||||
}
|
||||
|
||||
g_array_free(sce->dirlist, TRUE);
|
||||
sce->type = STAT_CACHE_ENTRY_SINGLE;
|
||||
} else {
|
||||
g_array_set_size(sce->dirlist, 0);
|
||||
}
|
||||
} else {
|
||||
/* single file */
|
||||
if (dir) {
|
||||
sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32);
|
||||
sce->type = STAT_CACHE_ENTRY_DIR;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
/* there are still vrequests using this entry, replace with a new one */
|
||||
sce->in_cache = FALSE;
|
||||
sce = g_slice_new0(stat_cache_entry);
|
||||
sce->path = g_string_new_len(GSTR_LEN(path));
|
||||
sce->data.path = g_string_new_len(GSTR_LEN(path));
|
||||
sce->vrequests = g_ptr_array_sized_new(8);
|
||||
sce->in_cache = TRUE;
|
||||
sce->queue_elem.data = sce;
|
||||
g_hash_table_replace(sc->entries, sce->path, sce);
|
||||
g_hash_table_replace(sc->entries, sce->data.path, sce);
|
||||
if (dir) {
|
||||
sce->type = STAT_CACHE_ENTRY_DIR;
|
||||
sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32);
|
||||
} else {
|
||||
sce->type = STAT_CACHE_ENTRY_SINGLE;
|
||||
}
|
||||
}
|
||||
|
||||
sce->ts = CUR_TS(vr->con->wrk);
|
||||
|
|
@ -164,7 +279,7 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
|
|||
} else {
|
||||
/* cache miss, allocate new entry */
|
||||
sce = g_slice_new0(stat_cache_entry);
|
||||
sce->path = g_string_new_len(GSTR_LEN(path));
|
||||
sce->data.path = g_string_new_len(GSTR_LEN(path));
|
||||
sce->vrequests = g_ptr_array_sized_new(8);
|
||||
sce->ts = CUR_TS(vr->con->wrk);
|
||||
sce->state = STAT_CACHE_ENTRY_WAITING;
|
||||
|
|
@ -174,40 +289,30 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
|
|||
g_ptr_array_add(sce->vrequests, vr);
|
||||
sce->refcount = 1;
|
||||
waitqueue_push(&sc->delete_queue, &sce->queue_elem);
|
||||
g_hash_table_insert(sc->entries, sce->path, sce);
|
||||
g_hash_table_insert(sc->entries, sce->data.path, sce);
|
||||
g_async_queue_push(sc->job_queue_out, sce);
|
||||
sc->misses++;
|
||||
|
||||
if (dir) {
|
||||
sce->type = STAT_CACHE_ENTRY_DIR;
|
||||
sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32);
|
||||
} else {
|
||||
sce->type = STAT_CACHE_ENTRY_SINGLE;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
stat_cache_entry *stat_cache_get(vrequest *vr, GString *path) {
|
||||
return stat_cache_get_internal(vr, path, FALSE);
|
||||
}
|
||||
|
||||
stat_cache_entry *stat_cache_get_dir(vrequest *vr, GString *path) {
|
||||
return stat_cache_get_internal(vr, path, TRUE);
|
||||
}
|
||||
|
||||
void stat_cache_entry_release(vrequest *vr) {
|
||||
vr->stat_cache_entry->refcount--;
|
||||
vr->stat_cache_entry = NULL;
|
||||
}
|
||||
|
||||
|
||||
gpointer stat_cache_thread(gpointer data) {
|
||||
stat_cache *sc = data;
|
||||
stat_cache_entry *sce;
|
||||
|
||||
while (TRUE) {
|
||||
sce = g_async_queue_pop(sc->job_queue_out);
|
||||
|
||||
/* stat cache entry with path == NULL indicates server stop */
|
||||
if (!sce->path)
|
||||
break;
|
||||
|
||||
if (stat(sce->path->str, &sce->st) == -1) {
|
||||
sce->failed = TRUE;
|
||||
sce->err = errno;
|
||||
} else
|
||||
sce->failed = FALSE;
|
||||
|
||||
g_atomic_int_set(&sce->state, STAT_CACHE_ENTRY_FINISHED);
|
||||
g_async_queue_push(sc->job_queue_in, sce);
|
||||
ev_async_send(sc->delete_queue.loop, &sc->job_watcher);
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -335,7 +335,7 @@ worker* worker_new(struct server *srv, struct ev_loop *loop) {
|
|||
ev_timer_init(&wrk->job_queue_watcher, worker_job_queue_cb, 0, 0);
|
||||
wrk->job_queue_watcher.data = wrk;
|
||||
|
||||
stat_cache_new(wrk, 10.0);
|
||||
stat_cache_new(wrk, srv->stat_cache_ttl);
|
||||
|
||||
return wrk;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue