Browse Source

mod_deflate (only cmake support for now)

personal/stbuehler/wip
Stefan Bühler 13 years ago
parent
commit
4ec620bfac
  1. 30
      src/CMakeLists.txt
  2. 10
      src/config.h.cmake
  3. 569
      src/modules/mod_deflate.c

30
src/CMakeLists.txt

@ -22,6 +22,8 @@ OPTION(WITH_LUA "with lua 5.1 for lua-configfile [default: on]" ON)
OPTION(WITH_OPENSSL "with openssl-support [default: off]")
OPTION(BUILD_STATIC "build a static lighttpd with all modules added")
OPTION(BUILD_EXTRA_WARNINGS "extra warnings")
OPTION(WITH_BZIP "with bzip2-support for mod_deflate")
OPTION(WITH_ZLIB "with deflate-support for mod_deflate")
IF(BUILD_STATIC)
SET(LIGHTTPD_STATIC 1)
@ -93,6 +95,26 @@ IF(WITH_OPENSSL)
ENDIF(HAVE_OPENSSL_SSL_H)
ENDIF(WITH_OPENSSL)
IF(WITH_BZIP)
CHECK_INCLUDE_FILES(bzlib.h HAVE_BZLIB_H)
CHECK_LIBRARY_EXISTS(bz2 BZ2_bzCompressInit "" HAVE_LIBBZ2)
IF(HAVE_BZLIB_H AND HAVE_LIBBZ2)
SET(BZIP_LDFLAGS "-lbz2")
SET(BZIP_CFLAGS "")
SET(HAVE_BZIP 1)
ENDIF(HAVE_BZLIB_H AND HAVE_LIBBZ2)
ENDIF(WITH_BZIP)
IF(WITH_ZLIB)
CHECK_INCLUDE_FILES(zlib.h HAVE_ZLIB_H)
CHECK_LIBRARY_EXISTS(z deflate "" HAVE_LIBZ)
IF(HAVE_ZLIB_H AND HAVE_LIBZ)
SET(ZLIB_LDFLAGS "-lz")
SET(ZLIB_CFLAGS "")
SET(HAVE_ZLIB 1)
ENDIF(HAVE_ZLIB_H AND HAVE_LIBZ)
ENDIF(WITH_ZLIB)
IF(NOT BUILD_STATIC)
CHECK_INCLUDE_FILES(dlfcn.h HAVE_DLFCN_H)
ENDIF(NOT BUILD_STATIC)
@ -239,6 +261,7 @@ SET(L_INSTALL_TARGETS ${L_INSTALL_TARGETS} lighttpd-angel)
IF(BUILD_EXTRA_WARNINGS)
SET(WARN_CFLAGS "-g -O2 -g2 -Wall -Wmissing-declarations -Wdeclaration-after-statement -Wcast-align -Wsign-compare -Wnested-externs -Wpointer-arith")
SET(WARN_LDFLAGS "-g -O2 -g2 -Wall -Wl,--as-needed")
SET(WARN_LDFLAGS "-g -O2 -g2 -Wall")
# -Werror -Wbad-function-cast -Wmissing-prototypes
ELSE(BUILD_EXTRA_WARNINGS)
SET(WARN_CFLAGS "")
@ -263,6 +286,13 @@ ADD_AND_INSTALL_LIBRARY(mod_rewrite "modules/mod_rewrite.c")
ADD_AND_INSTALL_LIBRARY(mod_status "modules/mod_status.c")
ADD_AND_INSTALL_LIBRARY(mod_vhost "modules/mod_vhost.c")
IF(HAVE_ZLIB OR HAVE_BZIP)
ADD_AND_INSTALL_LIBRARY(mod_deflate "modules/mod_deflate.c")
ADD_TARGET_PROPERTIES(mod_deflate LINK_FLAGS ${BZIP_LDFLAGS} ${ZLIB_LDFLAGS})
ADD_TARGET_PROPERTIES(mod_deflate COMPILE_FLAGS ${BZIP_CFLAGS} ${ZLIB_CFLAGS})
ENDIF(HAVE_ZLIB OR HAVE_BZIP)
IF(HAVE_LIBSSL AND HAVE_LIBCRYPTO)
ADD_AND_INSTALL_LIBRARY(mod_openssl "modules/mod_openssl.c")
TARGET_LINK_LIBRARIES(mod_openssl ssl)

10
src/config.h.cmake

@ -49,10 +49,6 @@
#cmakedefine HAVE_AIO_H
/* BZip */
#cmakedefine HAVE_BZLIB_H
#cmakedefine HAVE_LIBBZ2
/* FAM */
#cmakedefine HAVE_FAM_H
@ -93,9 +89,11 @@
#cmakedefine HAVE_UUID_H
#cmakedefine HAVE_LIBUUID
/* BZip */
#cmakedefine HAVE_BZIP
/* ZLIB */
#cmakedefine HAVE_ZLIB_H
#cmakedefine HAVE_LIBZ
#cmakedefine HAVE_ZLIB
/* GLIB */
#cmakedefine HAVE_GLIB_H

569
src/modules/mod_deflate.c

@ -0,0 +1,569 @@
/*
* mod_deflate - compress content on the fly
*
* Description:
* compress content on the fly
*
* Setups:
* none
* Options:
* none
* Actions:
* deflate
*
* Example config:
* deflate
*
* Author:
* Copyright (c) 2009 Stefan Bühler
*/
#include <lighttpd/base.h>
#include <lighttpd/plugin_core.h>
LI_API gboolean mod_deflate_init(liModules *mods, liModule *mod);
LI_API gboolean mod_deflate_free(liModules *mods, liModule *mod);
/* encoding names */
#define ENCODING_NAME_IDENTITY "identity"
#define ENCODING_NAME_GZIP "gzip"
#define ENCODING_NAME_DEFLATE "deflate"
#define ENCODING_NAME_COMPRESS "compress"
#define ENCODING_NAME_BZIP2 "bzip2"
typedef enum {
ENCODING_IDENTITY,
ENCODING_BZIP2,
ENCODING_GZIP,
ENCODING_DEFLATE,
ENCODING_COMPRESS
} encodings;
static const char* encoding_names[] = {
"identity",
"bzip2",
"gzip",
"deflate",
"compress",
NULL
};
static const guint encoding_available_mask = 0
#ifdef HAVE_BZIP
| (1 << ENCODING_BZIP2)
#endif
#ifdef HAVE_ZLIB
| (1 << ENCODING_GZIP) | (1 << ENCODING_DEFLATE)
#endif
;
typedef struct deflate_config deflate_config;
struct deflate_config {
int dummy;
};
/**********************************************************************************/
#ifdef HAVE_ZLIB
# include <zlib.h>
/* Copied gzip_header from apache 2.2's mod_deflate.c */
/* RFC 1952 Section 2.3 defines the gzip header:
*
* +---+---+---+---+---+---+---+---+---+---+
* |ID1|ID2|CM |FLG| MTIME |XFL|OS |
* +---+---+---+---+---+---+---+---+---+---+
*/
static const unsigned char gzip_header[] = {
0x1f, 0x8b, Z_DEFLATED, 0,
0, 0, 0, 0, /* mtime */
0, 0x03 /* Unix OS_CODE */
};
typedef struct deflate_context_zlib deflate_context_zlib;
struct deflate_context_zlib {
z_stream z;
GByteArray *buf;
gboolean is_gzip, gzip_header;
unsigned long crc;
};
static void deflate_context_zlib_free(deflate_context_zlib *ctx) {
z_stream *z;
if (!ctx) return;
z = &ctx->z;
deflateEnd(z);
g_byte_array_free(ctx->buf, TRUE);
g_slice_free(deflate_context_zlib, ctx);
}
static deflate_context_zlib* deflate_context_zlib_create(liVRequest *vr, gboolean is_gzip) {
deflate_context_zlib *ctx = g_slice_new0(deflate_context_zlib);
z_stream *z = &ctx->z;
guint compression_level = Z_DEFAULT_COMPRESSION;
guint window_size = -MAX_WBITS; /* supress zlib-header */
guint mem_level = 8;
z->zalloc = Z_NULL;
z->zfree = Z_NULL;
z->opaque = Z_NULL;
z->total_in = 0;
z->total_out = 0;
z->next_out = NULL;
z->avail_out = 0;
if (Z_OK != deflateInit2(z,
compression_level,
Z_DEFLATED,
window_size,
mem_level,
Z_DEFAULT_STRATEGY)) {
g_slice_free(deflate_context_zlib, ctx);
VR_ERROR(vr, "%s", "Couldn't init z_stream");
return NULL;
}
ctx->buf = g_byte_array_new();
g_byte_array_set_size(ctx->buf, 64*1024);
ctx->is_gzip = is_gzip;
z->next_out = ctx->buf->data;
z->avail_out = ctx->buf->len;
return ctx;
}
static void deflate_filter_zlib_free(liVRequest *vr, liFilter *f) {
deflate_context_zlib *ctx = (deflate_context_zlib*) f->param;
UNUSED(vr);
deflate_context_zlib_free(ctx);
}
static liHandlerResult deflate_filter_zlib(liVRequest *vr, liFilter *f) {
const off_t blocksize = 64*1024;
const off_t max_compress = 4 * blocksize;
deflate_context_zlib *ctx = (deflate_context_zlib*) f->param;
z_stream *z = &ctx->z;
off_t l = 0;
liHandlerResult res;
int rc;
UNUSED(vr);
if (f->out->is_closed) {
li_chunkqueue_skip_all(f->in);
f->in->is_closed = TRUE;
return LI_HANDLER_GO_ON;
}
if (ctx->is_gzip && !ctx->gzip_header) {
ctx->gzip_header = TRUE;
/* as the buffer is unused it really should be big enough */
if (z->avail_out < sizeof(gzip_header)) {
f->out->is_closed = TRUE;
return LI_HANDLER_ERROR;
}
/* copy gzip header into output buffer */
memcpy(z->next_out, gzip_header, sizeof(gzip_header));
/* initialize crc32 */
ctx->crc = crc32(0L, Z_NULL, 0);
z->next_out += sizeof(gzip_header);
z->avail_out -= sizeof(gzip_header);
}
while (l < max_compress) {
char *data;
off_t len;
liChunkIter ci;
if (0 == f->in->length) break;
ci = li_chunkqueue_iter(f->in);
if (LI_HANDLER_GO_ON != (res = li_chunkiter_read(vr, ci, 0, blocksize, &data, &len)))
return res;
if (ctx->is_gzip) {
ctx->crc = crc32(ctx->crc, (unsigned char*) data, len);
}
z->next_in = (unsigned char*) data;
z->avail_in = len;
do {
if (Z_OK != deflate(z, Z_NO_FLUSH)) {
f->out->is_closed = TRUE;
return LI_HANDLER_ERROR;
}
if(z->avail_out == 0 || z->avail_in > 0) {
li_chunkqueue_append_mem(f->out, ctx->buf->data, ctx->buf->len - z->avail_out);
z->next_out = ctx->buf->data;
z->avail_out = ctx->buf->len;
}
} while (z->avail_in > 0);
li_chunkqueue_skip(f->in, len);
l += len;
}
if (0 == f->in->length && f->in->is_closed) {
do {
rc = deflate(z, Z_FINISH);
if (rc != Z_OK && rc != Z_STREAM_END) {
f->out->is_closed = TRUE;
return LI_HANDLER_ERROR;
}
/* flush every time until done */
if (0 < ctx->buf->len - z->avail_out) {
li_chunkqueue_append_mem(f->out, ctx->buf->data, ctx->buf->len - z->avail_out);
z->next_out = ctx->buf->data;
z->avail_out = ctx->buf->len;
}
} while (rc != Z_STREAM_END);
if (ctx->is_gzip) {
/* write gzip footer */
unsigned char c[8];
c[0] = (ctx->crc >> 0) & 0xff;
c[1] = (ctx->crc >> 8) & 0xff;
c[2] = (ctx->crc >> 16) & 0xff;
c[3] = (ctx->crc >> 24) & 0xff;
c[4] = (z->total_in >> 0) & 0xff;
c[5] = (z->total_in >> 8) & 0xff;
c[6] = (z->total_in >> 16) & 0xff;
c[7] = (z->total_in >> 24) & 0xff;
/* append footer to write_queue */
li_chunkqueue_append_mem(f->out, c, 8);
}
f->out->is_closed = TRUE;
}
if (0 == f->in->length && !f->in->is_closed) { /* flush z_stream */
rc = deflate(z, Z_SYNC_FLUSH);
if (rc != Z_OK && rc != Z_STREAM_END) {
f->out->is_closed = TRUE;
return LI_HANDLER_ERROR;
}
}
/* flush output buffer if there is no more data pending */
if (0 == f->in->length && 0 < ctx->buf->len - z->avail_out) {
li_chunkqueue_append_mem(f->out, ctx->buf->data, ctx->buf->len - z->avail_out);
z->next_out = ctx->buf->data;
z->avail_out = ctx->buf->len;
}
return 0 == f->in->length ? LI_HANDLER_GO_ON : LI_HANDLER_COMEBACK;
}
#endif /* HAVE_ZLIB */
/**********************************************************************************/
#ifdef HAVE_BZIP
/* we don't need stdio interface */
# define BZ_NO_STDIO
# include <bzlib.h>
typedef struct deflate_context_bzip2 deflate_context_bzip2;
struct deflate_context_bzip2 {
bz_stream bz;
GByteArray *buf;
};
static void deflate_context_bzip2_free(deflate_context_bzip2 *ctx) {
bz_stream *bz;
if (!ctx) return;
bz = &ctx->bz;
BZ2_bzCompressEnd(bz);
g_byte_array_free(ctx->buf, TRUE);
g_slice_free(deflate_context_bzip2, ctx);
}
static deflate_context_bzip2* deflate_context_bzip2_create(liVRequest *vr) {
deflate_context_bzip2 *ctx = g_slice_new0(deflate_context_bzip2);
bz_stream *bz = &ctx->bz;
guint compression_level = 9;
bz->bzalloc = NULL;
bz->bzfree = NULL;
bz->opaque = NULL;
bz->total_in_lo32 = 0;
bz->total_in_hi32 = 0;
bz->total_out_lo32 = 0;
bz->total_out_hi32 = 0;
if (BZ_OK != BZ2_bzCompressInit(bz, compression_level /* blocksize */, 0 /* no output */, 30 /* workFactor: default */)) {
VR_ERROR(vr, "%s", "Couldn't init bz_stream");
g_slice_free(deflate_context_bzip2, ctx);
return NULL;
}
ctx->buf = g_byte_array_new();
g_byte_array_set_size(ctx->buf, 64*1024);
bz->next_out = (char*) ctx->buf->data;
bz->avail_out = ctx->buf->len;
return ctx;
}
static void deflate_filter_bzip2_free(liVRequest *vr, liFilter *f) {
deflate_context_bzip2 *ctx = (deflate_context_bzip2*) f->param;
UNUSED(vr);
deflate_context_bzip2_free(ctx);
}
static liHandlerResult deflate_filter_bzip2(liVRequest *vr, liFilter *f) {
const off_t blocksize = 64*1024;
const off_t max_compress = 4 * blocksize;
deflate_context_bzip2 *ctx = (deflate_context_bzip2*) f->param;
bz_stream *bz = &ctx->bz;
off_t l = 0;
liHandlerResult res;
int rc;
UNUSED(vr);
if (f->out->is_closed) {
f->in->is_closed = TRUE;
return LI_HANDLER_GO_ON;
}
while (l < max_compress) {
char *data;
off_t len;
liChunkIter ci;
if (0 == f->in->length) break;
ci = li_chunkqueue_iter(f->in);
if (LI_HANDLER_GO_ON != (res = li_chunkiter_read(vr, ci, 0, blocksize, &data, &len)))
return res;
bz->next_in = data;
bz->avail_in = len;
do {
rc = BZ2_bzCompress(bz, BZ_RUN);
if (rc != BZ_RUN_OK) {
f->out->is_closed = TRUE;
return LI_HANDLER_ERROR;
}
if(bz->avail_out == 0 || bz->avail_in > 0) {
li_chunkqueue_append_mem(f->out, ctx->buf->data, ctx->buf->len - bz->avail_out);
bz->next_out = (char*) ctx->buf->data;
bz->avail_out = ctx->buf->len;
}
} while (bz->avail_in > 0);
li_chunkqueue_skip(f->in, len);
l += len;
}
if (0 == f->in->length && f->in->is_closed) {
do {
rc = BZ2_bzCompress(bz, BZ_FINISH);
if (rc != BZ_RUN_OK && rc != BZ_STREAM_END) {
f->out->is_closed = TRUE;
return LI_HANDLER_ERROR;
}
/* flush every time until done */
if (0 < ctx->buf->len - bz->avail_out) {
li_chunkqueue_append_mem(f->out, ctx->buf->data, ctx->buf->len - bz->avail_out);
bz->next_out = (char*) ctx->buf->data;
bz->avail_out = ctx->buf->len;
}
} while (rc != BZ_STREAM_END);
f->out->is_closed = TRUE;
}
/* flush output buffer if there is no more data pending */
if (0 == f->in->length && 0 < ctx->buf->len - bz->avail_out) {
li_chunkqueue_append_mem(f->out, ctx->buf->data, ctx->buf->len - bz->avail_out);
bz->next_out = (char*) ctx->buf->data;
bz->avail_out = ctx->buf->len;
}
return 0 == f->in->length ? LI_HANDLER_GO_ON : LI_HANDLER_COMEBACK;
}
#endif /* HAVE_BZIP */
/**********************************************************************************/
static liHandlerResult deflate_handle(liVRequest *vr, gpointer param, gpointer *context) {
deflate_config *config = (deflate_config*) param;
GList *hh_encoding_entry, *hh_etag_entry;
liHttpHeader *hh_encoding, *hh_etag = NULL;
guint encoding_mask = 0, i;
UNUSED(config);
UNUSED(context);
if (vr->request.http_method == LI_HTTP_METHOD_HEAD) return LI_HANDLER_GO_ON;
VREQUEST_WAIT_FOR_RESPONSE_HEADERS(vr);
if (vr->response.http_status != 200) return LI_HANDLER_GO_ON;
/* response already encoded */
if (li_http_header_find_first(vr->response.headers, CONST_STR_LEN("content-encoding"))) return LI_HANDLER_GO_ON;
hh_encoding_entry = li_http_header_find_first(vr->request.headers, CONST_STR_LEN("accept-encoding"));
while (hh_encoding_entry) {
hh_encoding = (liHttpHeader*) hh_encoding_entry->data;
for (i = 1; encoding_names[i]; i++) {
if (strstr(HEADER_VALUE(hh_encoding), encoding_names[i])) {
encoding_mask |= 1 << i;
}
}
hh_encoding_entry = li_http_header_find_next(hh_encoding_entry, CONST_STR_LEN("accept-encoding"));
}
encoding_mask &= encoding_available_mask;
if (0 == encoding_mask)
return LI_HANDLER_GO_ON; /* no common encoding found */
/* find best encoding (first in list) */
for (i = 1; 0 == (encoding_mask & (1 << i)) ; i++) ;
hh_etag_entry = li_http_header_find_first(vr->response.headers, CONST_STR_LEN("etag"));
if (hh_etag_entry) {
if (li_http_header_find_next(hh_etag_entry, CONST_STR_LEN("etag"))) {
if (CORE_OPTION(LI_CORE_OPTION_DEBUG_REQUEST_HANDLING).boolean) {
VR_ERROR(vr, "%s", "duplicate etag header in response, will not deflate it");
}
return LI_HANDLER_GO_ON;
}
hh_etag = (liHttpHeader*) hh_etag_entry->data;
}
switch ((encodings) i) {
case ENCODING_IDENTITY:
return LI_HANDLER_GO_ON;
case ENCODING_BZIP2:
#ifdef HAVE_BZIP
{
deflate_context_bzip2 *ctx = deflate_context_bzip2_create(vr);
if (!ctx) return LI_HANDLER_GO_ON;
li_vrequest_add_filter_out(vr, deflate_filter_bzip2, deflate_filter_bzip2_free, ctx);
}
break;
#endif
return LI_HANDLER_GO_ON;
case ENCODING_GZIP:
#ifdef HAVE_ZLIB
{
deflate_context_zlib *ctx = deflate_context_zlib_create(vr, TRUE);
if (!ctx) return LI_HANDLER_GO_ON;
li_vrequest_add_filter_out(vr, deflate_filter_zlib, deflate_filter_zlib_free, ctx);
}
break;
#endif
return LI_HANDLER_GO_ON;
case ENCODING_DEFLATE:
#ifdef HAVE_ZLIB
{
deflate_context_zlib *ctx = deflate_context_zlib_create(vr, FALSE);
if (!ctx) return LI_HANDLER_GO_ON;
li_vrequest_add_filter_out(vr, deflate_filter_zlib, deflate_filter_zlib_free, ctx);
}
break;
#endif
return LI_HANDLER_GO_ON;
case ENCODING_COMPRESS:
return LI_HANDLER_GO_ON;
}
li_http_header_append(vr->response.headers, CONST_STR_LEN("Content-Encoding"), encoding_names[i], strlen(encoding_names[i]));
if (hh_etag) {
GString *s = vr->wrk->tmp_str;
g_string_truncate(s, 0);
g_string_append_len(s, HEADER_VALUE_LEN(hh_etag));
g_string_append_len(s, CONST_STR_LEN("-"));
g_string_append_len(s, encoding_names[i], strlen(encoding_names[i]));
li_etag_mutate(s, s);
g_string_truncate(hh_etag->data, hh_etag->keylen + 2);
g_string_append_len(hh_etag->data, GSTR_LEN(s));
}
return LI_HANDLER_GO_ON;
}
static void deflate_free(liServer *srv, gpointer param) {
deflate_config *conf = (deflate_config*) param;
UNUSED(srv);
g_slice_free(deflate_config, conf);
}
static liAction* deflate_create(liServer *srv, liPlugin* p, liValue *val) {
deflate_config *conf;
UNUSED(srv);
UNUSED(p);
UNUSED(val);
conf = g_slice_new0(deflate_config);
return li_action_new_function(deflate_handle, NULL, deflate_free, conf);
}
static const liPluginOption options[] = {
{ NULL, 0, NULL, NULL, NULL }
};
static const liPluginAction actions[] = {
{ "deflate", deflate_create },
{ NULL, NULL }
};
static const liPluginSetup setups[] = {
{ NULL, NULL }
};
static void plugin_init(liServer *srv, liPlugin *p) {
UNUSED(srv);
p->options = options;
p->actions = actions;
p->setups = setups;
}
gboolean mod_deflate_init(liModules *mods, liModule *mod) {
MODULE_VERSION_CHECK(mods);
mod->config = li_plugin_register(mods->main, "mod_deflate", plugin_init);
return mod->config != NULL;
}
gboolean mod_deflate_free(liModules *mods, liModule *mod) {
if (mod->config)
li_plugin_free(mods->main, mod->config);
return TRUE;
}
Loading…
Cancel
Save