lighttpd
/
lighttpd1.4
2
0
Fork 0
Code Issues Releases Wiki Activity

[mod_deflate] Brotli support 

configuration option: ./configure --with-brotli
This commit is contained in:
Glenn Strauss 2020-07-19 03:07:47 -04:00
parent be8047d82f
commit 8d5e237c60
8 changed files with 216 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
SConstruct
View File

@ -239,6 +239,7 @@ vars.AddVariables(
BoolVariable('build_fullstatic', 'enable fullstatic build', 'no'),
BoolVariable('with_bzip2', 'enable bzip2 compression', 'no'),
BoolVariable('with_brotli', 'enable brotli compression', 'no'),
PackageVariable('with_dbi', 'enable dbi support', 'no'),
BoolVariable('with_fam', 'enable FAM/gamin support', 'no'),
BoolVariable('with_gdbm', 'enable gdbm support', 'no'),
@ -323,6 +324,7 @@ if 1:
autoconf.env.Append(APPEND_LIBS = '')
autoconf.env.Append(
LIBBROTLI = '',
LIBBZ2 = '',
LIBCRYPT = '',
LIBCRYPTO = '',
@ -500,6 +502,11 @@ if 1:
LIBBZ2 = 'bz2',
)
if env['with_brotli']:
if not autoconf.CheckParseConfigForLib('LIBBROTLI', 'pkg-config brotlienc --cflags --libs'):
fail("Couldn't find libbrotlienc")
autoconf.env.Append(CPPFLAGS = [ '-DHAVE_BROTLI_ENCODE_H', '-DHAVE_BROTLI' ])
if env['with_dbi']:
if not autoconf.CheckLibWithHeader('dbi', 'dbi/dbi.h', 'C'):
fail("Couldn't find dbi")

31
configure.ac
View File

@ -978,6 +978,34 @@ if test "$WITH_BZIP2" != no; then
AC_SUBST([BZ_LIB])
fi
dnl brotli
AC_MSG_NOTICE([----------------------------------------])
AC_MSG_CHECKING([for brotli support])
AC_ARG_WITH([brotli],
[AC_HELP_STRING([--with-brotli],
[Enable brotli support for mod_deflate]
)],
[WITH_BROTLI=$withval],
[WITH_BROTLI=no]
)
AC_MSG_RESULT([$WITH_BROTLI])
if test "$WITH_BROTLI" != no; then
if test "$WITH_BROTLI" != yes; then
BROTLI_LIB="-L$WITH_BROTLI -lbrotlienc"
CPPFLAGS="$CPPFLAGS -I$WITH_BROTLI"
else
PKG_CHECK_MODULES([BROTLI], [libbrotlienc], [], [
AC_MSG_ERROR([brotli not found, install it or build without --with-brotli])
])
fi
AC_DEFINE([HAVE_BROTLI_ENCODE_H], [1], [brotli/encode.h])
AC_DEFINE([HAVE_BROTLI], [1], [libbrotlienc])
AC_SUBST([BROTLI_CFLAGS])
AC_SUBST([BROTLI_LIBS])
fi
dnl Check for fam/gamin
AC_MSG_NOTICE([----------------------------------------])
AC_MSG_CHECKING([for FAM])
@ -1666,6 +1694,9 @@ lighty_track_feature "compress-gzip compress-deflate" "" \
lighty_track_feature "compress-bzip2" "" \
'test "$WITH_BZIP2" != no'
lighty_track_feature "compress-brotli" "" \
'test "$WITH_BROTLI" != no'
lighty_track_feature "kerberos" "mod_authn_gssapi" \
'test "$WITH_KRB5" != no'

5
meson_options.txt
View File

@ -1,3 +1,8 @@
option('with_brotli',
type: 'boolean',
value: false,
description: 'with brotli-support for mod_deflate [default: off]',
)
option('with_bzip',
type: 'boolean',
value: false,

27
src/CMakeLists.txt
View File

@ -29,6 +29,7 @@ option(WITH_NETTLE "with Nettle-support [default: off]")
option(WITH_PCRE "with regex support [default: on]" ON)
option(WITH_WEBDAV_PROPS "with property-support for mod_webdav [default: off]")
option(WITH_WEBDAV_LOCKS "locks in webdav [default: off]")
option(WITH_BROTLI "with brotli-support for mod_deflate [default: off]")
option(WITH_BZIP "with bzip2-support for mod_compress [default: off]")
option(WITH_ZLIB "with deflate-support for mod_compress [default: on]" ON)
option(WITH_KRB5 "with Kerberos5-support for mod_auth [default: off]")
@ -601,6 +602,14 @@ else()
unset(HAVE_LIBBZ2)
endif()
if(WITH_BROTLI)
pkg_check_modules(LIBBROTLI REQUIRED libbrotlienc)
set(HAVE_BROTLI 1)
add_definitions(-DHAVE_BROTLI_ENCODE_H -DHAVE_BROTLI)
else()
unset(HAVE_BROTLI)
endif()
if(WITH_LDAP)
check_include_files(ldap.h HAVE_LDAP_H)
check_library_exists(ldap ldap_bind "" HAVE_LIBLDAP)
@ -1039,14 +1048,18 @@ if(WITH_SASL)
target_link_libraries(mod_authn_sasl ${L_MOD_AUTHN_SASL})
endif()
if(HAVE_ZLIB_H)
if(HAVE_BZLIB_H)
target_link_libraries(mod_compress ${ZLIB_LIBRARY} bz2)
target_link_libraries(mod_deflate ${ZLIB_LIBRARY} bz2)
else()
target_link_libraries(mod_compress ${ZLIB_LIBRARY})
target_link_libraries(mod_deflate ${ZLIB_LIBRARY})
if(HAVE_ZLIB_H OR HAVE_BZLIB_H OR HAVE_BROTLI)
if(HAVE_ZLIB_H)
set(L_MOD_DEFLATE ${L_MOD_DEFLATE} ${ZLIB_LIBRARY})
endif()
if(HAVE_BZLIB_H)
set(L_MOD_DEFLATE ${L_MOD_DEFLATE} bz2)
endif()
target_link_libraries(mod_compress ${L_MOD_DEFLATE})
if(HAVE_BROTLI)
set(L_MOD_DEFLATE ${L_MOD_DEFLATE} brotlienc)
endif()
target_link_libraries(mod_deflate ${L_MOD_DEFLATE})
endif()
if(HAVE_LIBFAM)

7
src/Makefile.am
View File

@ -334,8 +334,8 @@ mod_compress_la_LIBADD = $(Z_LIB) $(BZ_LIB) $(common_libadd)
lib_LTLIBRARIES += mod_deflate.la
mod_deflate_la_SOURCES = mod_deflate.c
mod_deflate_la_LDFLAGS = $(common_module_ldflags)
mod_deflate_la_LIBADD = $(Z_LIB) $(BZ_LIB) $(common_libadd)
mod_deflate_la_LDFLAGS = $(BROTLI_CFLAGS) $(common_module_ldflags)
mod_deflate_la_LIBADD = $(Z_LIB) $(BZ_LIB) $(BROTLI_LIBS) $(common_libadd)
lib_LTLIBRARIES += mod_auth.la
mod_auth_la_SOURCES = mod_auth.c
@ -520,7 +520,8 @@ lighttpd_LDADD = \
$(common_libadd) \
$(CRYPT_LIB) $(CRYPTO_LIB) \
$(XML_LIBS) $(SQLITE_LIBS) $(UUID_LIBS) $(ELFTC_LIB) \
$(PCRE_LIB) $(Z_LIB) $(BZ_LIB) $(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) \
$(PCRE_LIB) $(Z_LIB) $(BZ_LIB) $(BROTLI_LIBS) \
$(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) \
$(FAM_LIBS) $(LIBEV_LIBS) $(LIBUNWIND_LIBS)
lighttpd_LDFLAGS = -export-dynamic

2
src/SConscript
View File

@ -104,7 +104,7 @@ modules = {
'mod_authn_file' : { 'src' : [ 'mod_authn_file.c' ], 'lib' : [ env['LIBCRYPT'], env['LIBCRYPTO'] ] },
'mod_cgi' : { 'src' : [ 'mod_cgi.c' ] },
'mod_compress' : { 'src' : [ 'mod_compress.c' ], 'lib' : [ env['LIBZ'], env['LIBBZ2'] ] },
'mod_deflate' : { 'src' : [ 'mod_deflate.c' ], 'lib' : [ env['LIBZ'], env['LIBBZ2'] ] },
'mod_deflate' : { 'src' : [ 'mod_deflate.c' ], 'lib' : [ env['LIBZ'], env['LIBBZ2'], env['LIBBROTLI'] ] },
'mod_dirlisting' : { 'src' : [ 'mod_dirlisting.c' ], 'lib' : [ env['LIBPCRE'] ] },
'mod_evasive' : { 'src' : [ 'mod_evasive.c' ] },
'mod_evhost' : { 'src' : [ 'mod_evhost.c' ] },

145
src/mod_deflate.c
View File

@ -140,6 +140,11 @@
# include <bzlib.h>
#endif
#if defined HAVE_BROTLI_ENCODE_H && defined HAVE_BROTLI
# define USE_BROTLI
# include <brotli/encode.h>
#endif
#if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
#define USE_MMAP
@ -165,6 +170,7 @@ static void sigbus_handler(int sig) {
#define HTTP_ACCEPT_ENCODING_BZIP2 BV(4)
#define HTTP_ACCEPT_ENCODING_X_GZIP BV(5)
#define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6)
#define HTTP_ACCEPT_ENCODING_BR BV(7)
#define KByte * 1024
#define MByte * 1024 KByte
@ -197,6 +203,9 @@ typedef struct {
#endif
#ifdef USE_BZ2LIB
bz_stream bz;
#endif
#ifdef USE_BROTLI
BrotliEncoderState *br;
#endif
int dummy;
} u;
@ -205,6 +214,7 @@ typedef struct {
chunkqueue *in_queue;
buffer *output;
plugin_data *plugin_data;
request_st *r;
int compression_type;
log_error_st *errh;
} handler_ctx;
@ -288,7 +298,7 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
short allowed_encodings = 0;
if (encodings->used) {
for (uint32_t j = 0; j < encodings->used; ++j) {
#if defined(USE_ZLIB) || defined(USE_BZ2LIB)
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI)
data_string *ds = (data_string *)encodings->data[j];
#endif
#ifdef USE_ZLIB
@ -311,6 +321,10 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
if (NULL != strstr(ds->value.ptr, "x-bzip2"))
allowed_encodings |= HTTP_ACCEPT_ENCODING_X_BZIP2;
#endif
#ifdef USE_BROTLI
if (NULL != strstr(ds->value.ptr, "br"))
allowed_encodings |= HTTP_ACCEPT_ENCODING_BR;
#endif
}
}
else {
@ -324,6 +338,9 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
allowed_encodings |= HTTP_ACCEPT_ENCODING_BZIP2
| HTTP_ACCEPT_ENCODING_X_BZIP2;
#endif
#ifdef USE_BROTLI
allowed_encodings |= HTTP_ACCEPT_ENCODING_BR;
#endif
}
return allowed_encodings;
}
@ -653,6 +670,94 @@ static int stream_bzip2_end(handler_ctx *hctx) {
#endif
#ifdef USE_BROTLI
static int stream_br_init(handler_ctx *hctx) {
BrotliEncoderState * const br = hctx->u.br =
BrotliEncoderCreateInstance(NULL, NULL, NULL);
if (NULL == br) return -1;
/* future: consider allowing tunables by encoder algorithm,
* (i.e. not generic "compression_level") */
/*(note: we ignore any errors while tuning parameters here)*/
const plugin_data * const p = hctx->plugin_data;
if (p->conf.compression_level >= 0) /* 0 .. 11 are valid values */
BrotliEncoderSetParameter(br, BROTLI_PARAM_QUALITY,
p->conf.compression_level);
/* XXX: is this worth checking?
* BROTLI_MODE_GENERIC vs BROTLI_MODE_TEXT or BROTLI_MODE_FONT */
const buffer *vb =
http_header_response_get(hctx->r, HTTP_HEADER_CONTENT_TYPE,
CONST_STR_LEN("Content-Type"));
if (NULL != vb) {
if (0 == strncmp(vb->ptr, "text/", sizeof("text/")-1)
|| (0 == strncmp(vb->ptr, "application/", sizeof("application/")-1)
&& (0 == strncmp(vb->ptr+12,"javascript",sizeof("javascript")-1)
|| 0 == strncmp(vb->ptr+12,"ld+json", sizeof("ld+json")-1)
|| 0 == strncmp(vb->ptr+12,"json", sizeof("json")-1)
|| 0 == strncmp(vb->ptr+12,"xhtml+xml", sizeof("xhtml+xml")-1)
|| 0 == strncmp(vb->ptr+12,"xml", sizeof("xml")-1)))
|| 0 == strncmp(vb->ptr, "image/svg+xml", sizeof("image/svg+xml")-1))
BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_TEXT);
else if (0 == strncmp(vb->ptr, "font/", sizeof("font/")-1))
BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_FONT);
}
return 0;
}
static int stream_br_compress(request_st * const r, handler_ctx * const hctx, unsigned char * const start, off_t st_size) {
const uint8_t *in = (uint8_t *)start;
BrotliEncoderState * const br = hctx->u.br;
hctx->bytes_in += st_size;
while (st_size || BrotliEncoderHasMoreOutput(br)) {
size_t insz = ((off_t)((~(uint32_t)0) >> 1) > st_size)
? (size_t)st_size
: ((~(uint32_t)0) >> 1);
size_t outsz = 0;
BrotliEncoderCompressStream(br, BROTLI_OPERATION_PROCESS,
&insz, &in, &outsz, NULL, NULL);
const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz);
st_size -= (st_size - (off_t)insz);
if (outsz) {
hctx->bytes_out += (off_t)outsz;
/*stream_http_chunk_append_mem(r, hctx, outsz);*/
if (0 != http_chunk_append_mem(r, (char *)out, outsz))
return -1;
}
}
return 0;
}
static int stream_br_flush(request_st * const r, handler_ctx * const hctx, int end) {
const int brmode = end ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_FLUSH;
BrotliEncoderState * const br = hctx->u.br;
do {
size_t insz = 0;
size_t outsz = 0;
BrotliEncoderCompressStream(br, brmode,
&insz, NULL, &outsz, NULL, NULL);
const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz);
if (outsz) {
hctx->bytes_out += (off_t)outsz;
/*stream_http_chunk_append_mem(r, hctx, outsz);*/
if (0 != http_chunk_append_mem(r, (char *)out, outsz))
return -1;
}
} while (BrotliEncoderHasMoreOutput(br));
return 0;
}
static int stream_br_end(handler_ctx *hctx) {
BrotliEncoderState * const br = hctx->u.br;
BrotliEncoderDestroyInstance(br);
return 0;
}
#endif
static int mod_deflate_stream_init(handler_ctx *hctx) {
switch(hctx->compression_type) {
#ifdef USE_ZLIB
@ -663,6 +768,10 @@ static int mod_deflate_stream_init(handler_ctx *hctx) {
#ifdef USE_BZ2LIB
case HTTP_ACCEPT_ENCODING_BZIP2:
return stream_bzip2_init(hctx);
#endif
#ifdef USE_BROTLI
case HTTP_ACCEPT_ENCODING_BR:
return stream_br_init(hctx);
#endif
default:
return -1;
@ -680,6 +789,10 @@ static int mod_deflate_compress(request_st * const r, handler_ctx * const hctx,
#ifdef USE_BZ2LIB
case HTTP_ACCEPT_ENCODING_BZIP2:
return stream_bzip2_compress(r, hctx, start, st_size);
#endif
#ifdef USE_BROTLI
case HTTP_ACCEPT_ENCODING_BR:
return stream_br_compress(r, hctx, start, st_size);
#endif
default:
UNUSED(r);
@ -699,6 +812,10 @@ static int mod_deflate_stream_flush(request_st * const r, handler_ctx * const hc
#ifdef USE_BZ2LIB
case HTTP_ACCEPT_ENCODING_BZIP2:
return stream_bzip2_flush(r, hctx, end);
#endif
#ifdef USE_BROTLI
case HTTP_ACCEPT_ENCODING_BR:
return stream_br_flush(r, hctx, end);
#endif
default:
UNUSED(r);
@ -729,6 +846,10 @@ static int mod_deflate_stream_end(handler_ctx *hctx) {
#ifdef USE_BZ2LIB
case HTTP_ACCEPT_ENCODING_BZIP2:
return stream_bzip2_end(hctx);
#endif
#ifdef USE_BROTLI
case HTTP_ACCEPT_ENCODING_BR:
return stream_br_end(hctx);
#endif
default:
return -1;
@ -975,7 +1096,7 @@ static handler_t deflate_compress_response(request_st * const r, handler_ctx * c
static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) {
/* get client side support encodings */
int accept_encoding = 0;
#if !defined(USE_ZLIB) && !defined(USE_BZ2LIB)
#if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI)
UNUSED(value);
#else
for (; *value; ++value) {
@ -985,6 +1106,12 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const
while (*value!=' ' && *value!=',' && *value!=';' && *value!='\0')
++value;
switch (value - v) {
case 2:
#ifdef USE_BROTLI
if (0 == memcmp(v, "br", 2))
accept_encoding |= HTTP_ACCEPT_ENCODING_BR;
#endif
break;
case 4:
#ifdef USE_ZLIB
if (0 == memcmp(v, "gzip", 4))
@ -1035,6 +1162,12 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const
accept_encoding &= p->conf.allowed_encodings;
/* select best matching encoding */
#ifdef USE_BROTLI
if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) {
*label = "br";
return HTTP_ACCEPT_ENCODING_BR;
} else
#endif
#ifdef USE_BZ2LIB
if (accept_encoding & HTTP_ACCEPT_ENCODING_BZIP2) {
*label = "bzip2";
@ -1210,6 +1343,7 @@ REQUEST_FUNC(mod_deflate_handle_response_start) {
hctx = handler_ctx_init();
hctx->plugin_data = p;
hctx->compression_type = compression_type;
hctx->r = r;
hctx->errh = r->conf.errh;
/* setup output buffer */
buffer_clear(&p->tmp_buf);
@ -1228,6 +1362,13 @@ REQUEST_FUNC(mod_deflate_handle_response_start) {
return HANDLER_GO_ON;
}
#ifdef USE_BROTLI
if (r->resp_body_finished
&& (hctx->compression_type & HTTP_ACCEPT_ENCODING_BR)
&& (len >> 1) < (off_t)((~(uint32_t)0u) >> 1))
BrotliEncoderSetParameter(hctx->u.br, BROTLI_PARAM_SIZE_HINT, (uint32_t)len);
#endif
if (r->resp_htags & HTTP_HEADER_CONTENT_LENGTH) {
http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, CONST_STR_LEN("Content-Length"));
}

5
src/server.c
View File

@ -435,6 +435,11 @@ static void show_features (void) {
#else
"\t- bzip2 support\n"
#endif
#if defined HAVE_BROTLI_ENCODE_H && defined HAVE_BROTLI
"\t+ brotli support\n"
#else
"\t- brotli support\n"
#endif
#if defined(HAVE_CRYPT) || defined(HAVE_CRYPT_R) || defined(HAVE_LIBCRYPT)
"\t+ crypt support\n"
#else