lighttpd
/
lighttpd1.4
2
0
Fork 0
Code Issues Releases Wiki Activity

[mod_deflate] support Accept-Encoding: zstd

This commit is contained in:
Glenn Strauss 2021-01-12 18:29:20 -05:00
parent ef28cce5e5
commit 9211fb3d86
8 changed files with 202 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
SConstruct
View File

@ -269,6 +269,7 @@ vars.AddVariables(
PackageVariable('with_xml', 'enable xml support (required for webdav props)', 'no'),
BoolVariable('with_xxhash', 'build with system-provided xxhash', 'no'),
BoolVariable('with_zlib', 'enable deflate/gzip compression', 'no'),
BoolVariable('with_zstd', 'enable zstd compression', 'no'),
BoolVariable('with_all', 'enable all with_* features', 'no'),
)
@ -357,6 +358,7 @@ if 1:
LIBXML2 = '',
LIBXXHASH = '',
LIBZ = '',
LIBZSTD = '',
)
autoconf.haveCHeaders([
@ -731,6 +733,14 @@ if 1:
LIBZ = 'z',
)
if env['with_zstd']:
if not autoconf.CheckLibWithHeader('zstd', 'zstd.h', 'C'):
fail("Couldn't find zstd")
autoconf.env.Append(
CPPFLAGS = [ '-DHAVE_ZSTD_H', '-DHAVE_ZSTD' ],
LIBZSTD = 'zstd',
)
env = autoconf.Finish()
if re.compile("cygwin|mingw|midipix").search(env['PLATFORM']):

31
configure.ac
View File

@ -962,6 +962,37 @@ if test "$WITH_ZLIB" != no; then
AC_SUBST([Z_LIB])
fi
dnl zstd
AC_MSG_NOTICE([----------------------------------------])
AC_MSG_CHECKING([for zstd support])
AC_ARG_WITH([zstd],
[AS_HELP_STRING([--with-zstd],
[Enable zstd support for mod_deflate]
)],
[WITH_ZSTD=$withval],
[WITH_ZSTD=no]
)
AC_MSG_RESULT([$WITH_ZSTD])
if test "$WITH_ZSTD" != no; then
if test "$WITH_ZSTD" != yes; then
ZSTD_LIB="-L$WITH_ZSTD -lzstd"
CPPFLAGS="$CPPFLAGS -I$WITH_ZSTD"
else
AC_CHECK_HEADERS([zstd.h], [],
[AC_MSG_ERROR([zstd headers not found, install them or build without --with-zstd])]
)
AC_CHECK_LIB([zstd], [ZSTD_versionNumber],
[ZSTD_LIB=-lzstd],
[AC_MSG_ERROR([zstd library not found, install it or build without --with-zstd])]
)
fi
AC_DEFINE([HAVE_ZSTD], [1], [libzstd])
AC_DEFINE([HAVE_ZSTD_H], [1])
AC_SUBST([ZSTD_LIB])
fi
dnl bzip2
AC_MSG_NOTICE([----------------------------------------])
AC_MSG_CHECKING([for bzip2 support])

14
src/CMakeLists.txt
View File

@ -32,6 +32,7 @@ option(WITH_WEBDAV_LOCKS "locks in webdav [default: off]")
option(WITH_BROTLI "with brotli-support for mod_deflate [default: off]")
option(WITH_BZIP "with bzip2-support for mod_deflate [default: off]")
option(WITH_ZLIB "with deflate-support for mod_deflate [default: on]" ON)
option(WITH_ZSTD "with zstd-support for mod_deflate [default: off]")
option(WITH_KRB5 "with Kerberos5-support for mod_auth [default: off]")
option(WITH_LDAP "with LDAP-support for mod_auth mod_vhostdb_ldap [default: off]")
option(WITH_PAM "with PAM-support for mod_auth [default: off]")
@ -607,6 +608,14 @@ else()
unset(ZLIB_LIBRARY)
endif()
if(WITH_ZSTD)
check_include_files(zstd.h HAVE_ZSTD_H)
check_library_exists(zstd ZSTD_versionNumber "" HAVE_ZSTD)
else()
unset(HAVE_ZSTD_H)
unset(HAVE_ZSTD)
endif()
if(WITH_BZIP)
check_include_files(bzlib.h HAVE_BZLIB_H)
check_library_exists(bz2 BZ2_bzCompress "" HAVE_LIBBZ2)
@ -1063,10 +1072,13 @@ if(WITH_SASL)
target_link_libraries(mod_authn_sasl ${L_MOD_AUTHN_SASL})
endif()
if(HAVE_ZLIB_H OR HAVE_BZLIB_H OR HAVE_BROTLI)
if(HAVE_ZLIB_H OR HAVE_ZSTD_H OR HAVE_BZLIB_H OR HAVE_BROTLI)
if(HAVE_ZLIB_H)
set(L_MOD_DEFLATE ${L_MOD_DEFLATE} ${ZLIB_LIBRARY})
endif()
if(HAVE_ZSTD_H)
set(L_MOD_DEFLATE ${L_MOD_DEFLATE} zstd)
endif()
if(HAVE_BZLIB_H)
set(L_MOD_DEFLATE ${L_MOD_DEFLATE} bz2)
endif()

4
src/Makefile.am
View File

@ -326,7 +326,7 @@ mod_access_la_LIBADD = $(common_libadd)
lib_LTLIBRARIES += mod_deflate.la
mod_deflate_la_SOURCES = mod_deflate.c
mod_deflate_la_LDFLAGS = $(BROTLI_CFLAGS) $(common_module_ldflags)
mod_deflate_la_LIBADD = $(Z_LIB) $(BZ_LIB) $(BROTLI_LIBS) $(common_libadd)
mod_deflate_la_LIBADD = $(Z_LIB) $(ZSTD_LIB) $(BZ_LIB) $(BROTLI_LIBS) $(common_libadd)
lib_LTLIBRARIES += mod_auth.la
mod_auth_la_SOURCES = mod_auth.c
@ -525,7 +525,7 @@ lighttpd_LDADD = \
$(common_libadd) \
$(CRYPT_LIB) $(CRYPTO_LIB) $(XXHASH_LIBS) \
$(XML_LIBS) $(SQLITE_LIBS) $(UUID_LIBS) $(ELFTC_LIB) \
$(PCRE_LIB) $(Z_LIB) $(BZ_LIB) $(BROTLI_LIBS) \
$(PCRE_LIB) $(Z_LIB) $(ZSTD_LIB) $(BZ_LIB) $(BROTLI_LIBS) \
$(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) \
$(FAM_LIBS) $(LIBEV_LIBS) $(LIBUNWIND_LIBS)
lighttpd_LDFLAGS = -export-dynamic

2
src/SConscript
View File

@ -105,7 +105,7 @@ modules = {
'mod_auth' : { 'src' : [ 'mod_auth.c' ], 'lib' : [ env['LIBCRYPTO'] ] },
'mod_authn_file' : { 'src' : [ 'mod_authn_file.c' ], 'lib' : [ env['LIBCRYPT'], env['LIBCRYPTO'] ] },
'mod_cgi' : { 'src' : [ 'mod_cgi.c' ] },
'mod_deflate' : { 'src' : [ 'mod_deflate.c' ], 'lib' : [ env['LIBZ'], env['LIBBZ2'], env['LIBBROTLI'], 'm' ] },
'mod_deflate' : { 'src' : [ 'mod_deflate.c' ], 'lib' : [ env['LIBZ'], env['LIBZSTD'], env['LIBBZ2'], env['LIBBROTLI'], 'm' ] },
'mod_dirlisting' : { 'src' : [ 'mod_dirlisting.c' ], 'lib' : [ env['LIBPCRE'] ] },
'mod_evasive' : { 'src' : [ 'mod_evasive.c' ] },
'mod_evhost' : { 'src' : [ 'mod_evhost.c' ] },

17
src/meson.build
View File

@ -683,6 +683,21 @@ if get_option('with_zlib')
conf_data.set('HAVE_LIBZ', true)
endif
libzstd = []
if get_option('with_zstd')
libz = dependency('zstd', required: false)
if libzstd.found()
libzstd = [ libzstd ]
else
libzstd = [ compiler.find_library('zstd') ]
if not(compiler.has_function('ZSTD_versionNumber', args: defs, dependencies: libzstd, prefix: '#include <zstd.h>'))
error('Couldn\'t find zstd header / library')
endif
endif
conf_data.set('HAVE_ZSTD_H', true)
conf_data.set('HAVE_ZSTD', true)
endif
configure_file(
output : 'config.h',
configuration : conf_data,
@ -977,7 +992,7 @@ modules = [
[ 'mod_alias', [ 'mod_alias.c' ] ],
[ 'mod_auth', [ 'mod_auth.c' ], [ libcrypto ] ],
[ 'mod_authn_file', [ 'mod_authn_file.c' ], [ libcrypt, libcrypto ] ],
[ 'mod_deflate', [ 'mod_deflate.c' ], libbz2 + libz + libbrotli ],
[ 'mod_deflate', [ 'mod_deflate.c' ], libbz2 + libz + libzstd + libbrotli ],
[ 'mod_dirlisting', [ 'mod_dirlisting.c' ], libpcre ],
[ 'mod_evasive', [ 'mod_evasive.c' ] ],
[ 'mod_evhost', [ 'mod_evhost.c' ] ],

127
src/mod_deflate.c
View File

@ -144,6 +144,14 @@
# include <brotli/encode.h>
#endif
#if defined HAVE_ZSTD_H && defined HAVE_ZSTD
# define USE_ZSTD
/* FIXME: wrote initial implementation to support zstd and then realized
* bufferless streaming compression is an experimental (unstable) zstd API */
# define ZSTD_STATIC_LINKING_ONLY
# include <zstd.h>
#endif
#if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
#define USE_MMAP
@ -170,6 +178,7 @@ static void sigbus_handler(int sig) {
#define HTTP_ACCEPT_ENCODING_X_GZIP BV(5)
#define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6)
#define HTTP_ACCEPT_ENCODING_BR BV(7)
#define HTTP_ACCEPT_ENCODING_ZSTD BV(8)
typedef struct {
const array *mimetypes;
@ -202,6 +211,9 @@ typedef struct {
#endif
#ifdef USE_BROTLI
BrotliEncoderState *br;
#endif
#ifdef USE_ZSTD
ZSTD_CCtx *cctx;
#endif
int dummy;
} u;
@ -244,7 +256,12 @@ static void handler_ctx_free(handler_ctx *hctx) {
INIT_FUNC(mod_deflate_init) {
plugin_data * const p = calloc(1, sizeof(plugin_data));
#ifdef USE_ZSTD
buffer_string_prepare_copy(&p->tmp_buf, /* 131072 */
ZSTD_COMPRESSBOUND(ZSTD_BLOCKSIZE_MAX));
#else
buffer_string_prepare_copy(&p->tmp_buf, 65536);
#endif
return p;
}
@ -384,7 +401,8 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
short allowed_encodings = 0;
if (encodings->used) {
for (uint32_t j = 0; j < encodings->used; ++j) {
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI)
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \
|| defined(USE_ZSTD)
data_string *ds = (data_string *)encodings->data[j];
#endif
#ifdef USE_ZLIB
@ -411,6 +429,10 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
if (NULL != strstr(ds->value.ptr, "br"))
allowed_encodings |= HTTP_ACCEPT_ENCODING_BR;
#endif
#ifdef USE_ZSTD
if (NULL != strstr(ds->value.ptr, "zstd"))
allowed_encodings |= HTTP_ACCEPT_ENCODING_ZSTD;
#endif
}
}
else {
@ -427,6 +449,9 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
#ifdef USE_BROTLI
allowed_encodings |= HTTP_ACCEPT_ENCODING_BR;
#endif
#ifdef USE_ZSTD
allowed_encodings |= HTTP_ACCEPT_ENCODING_ZSTD;
#endif
}
return allowed_encodings;
}
@ -612,7 +637,8 @@ SETDEFAULTS_FUNC(mod_deflate_set_defaults) {
}
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI)
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \
|| defined(USE_ZSTD)
static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) {
ssize_t wr;
do {
@ -933,6 +959,71 @@ static int stream_br_end(handler_ctx *hctx) {
#endif
#ifdef USE_ZSTD
static int stream_zstd_init(handler_ctx *hctx) {
ZSTD_CCtx * const cctx = hctx->u.cctx = ZSTD_createCCtx();
if (NULL == cctx) return -1;
return 0;
}
static int stream_zstd_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) {
ZSTD_CCtx * const cctx = hctx->u.cctx;
/* Note: each chunkqueue chunk is sent in its own frame, which may be
* suboptimal. lighttpd might read FILE_CHUNK into a reused buffer, so
* we can not meet ZSTD_compressContinue() requirement that prior input
* is still accessible and unmodified (up to maximum distance size).
* Also, chunkqueue_mark_written() on MEM_CHUNK might result in something
* else reusing those chunk buffers
*
* future: migrate to use Zstd streaming API */
/* future: consider allowing tunables by encoder algorithm,
* (i.e. not generic "compression_level" across all compression algorithms)
* (ZSTD_CCtx_setParameter()) */
/*(note: we ignore any errors while tuning parameters here)*/
const plugin_data * const p = hctx->plugin_data;
int level = ZSTD_CLEVEL_DEFAULT;
if (p->conf.compression_level >= 0) /* -1 is lighttpd default for "unset" */
level = p->conf.compression_level;
ZSTD_compressBegin(cctx, level);
char * const out = hctx->output->ptr;
const size_t outsz = hctx->output->size;
hctx->bytes_in += st_size;
for (off_t off = 0; st_size; ) {
/* XXX: size check must match mod_deflate_init ZSTD_COMPRESSBOUND arg */
size_t len = (size_t)st_size;
const size_t rv = (len > ZSTD_BLOCKSIZE_MAX)
? ZSTD_compressContinue(cctx, out, outsz, start+off,
(len = ZSTD_BLOCKSIZE_MAX))
: ZSTD_compressEnd(cctx, out, outsz, start+off, len);
off += (off_t)len;
st_size -= (off_t)len;
if (ZSTD_isError(rv)) return -1;
hctx->bytes_out += (off_t)rv;
if (0 != stream_http_chunk_append_mem(hctx, out, rv))
return -1;
}
return 0;
}
static int stream_zstd_flush(handler_ctx * const hctx, int end) {
UNUSED(hctx);
UNUSED(end);
return 0;
}
static int stream_zstd_end(handler_ctx *hctx) {
ZSTD_CCtx * const cctx = hctx->u.cctx;
ZSTD_freeCCtx(cctx);
return 0;
}
#endif
static int mod_deflate_stream_init(handler_ctx *hctx) {
switch(hctx->compression_type) {
#ifdef USE_ZLIB
@ -947,6 +1038,10 @@ static int mod_deflate_stream_init(handler_ctx *hctx) {
#ifdef USE_BROTLI
case HTTP_ACCEPT_ENCODING_BR:
return stream_br_init(hctx);
#endif
#ifdef USE_ZSTD
case HTTP_ACCEPT_ENCODING_ZSTD:
return stream_zstd_init(hctx);
#endif
default:
return -1;
@ -968,6 +1063,10 @@ static int mod_deflate_compress(handler_ctx * const hctx, unsigned char * const
#ifdef USE_BROTLI
case HTTP_ACCEPT_ENCODING_BR:
return stream_br_compress(hctx, start, st_size);
#endif
#ifdef USE_ZSTD
case HTTP_ACCEPT_ENCODING_ZSTD:
return stream_zstd_compress(hctx, start, st_size);
#endif
default:
UNUSED(start);
@ -990,6 +1089,10 @@ static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) {
#ifdef USE_BROTLI
case HTTP_ACCEPT_ENCODING_BR:
return stream_br_flush(hctx, end);
#endif
#ifdef USE_ZSTD
case HTTP_ACCEPT_ENCODING_ZSTD:
return stream_zstd_flush(hctx, end);
#endif
default:
UNUSED(end);
@ -1023,6 +1126,10 @@ static int mod_deflate_stream_end(handler_ctx *hctx) {
#ifdef USE_BROTLI
case HTTP_ACCEPT_ENCODING_BR:
return stream_br_end(hctx);
#endif
#ifdef USE_ZSTD
case HTTP_ACCEPT_ENCODING_ZSTD:
return stream_zstd_end(hctx);
#endif
default:
return -1;
@ -1263,7 +1370,8 @@ static handler_t deflate_compress_response(request_st * const r, handler_ctx * c
static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) {
/* get client side support encodings */
int accept_encoding = 0;
#if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI)
#if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) \
&& !defined(USE_ZSTD)
UNUSED(value);
UNUSED(label);
#else
@ -1284,6 +1392,13 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const
#ifdef USE_ZLIB
if (0 == memcmp(v, "gzip", 4))
accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP;
#endif
#ifdef USE_ZSTD
#ifdef USE_ZLIB
else
#endif
if (0 == memcmp(v, "zstd", 4))
accept_encoding |= HTTP_ACCEPT_ENCODING_ZSTD;
#endif
break;
case 5:
@ -1330,6 +1445,12 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const
accept_encoding &= p->conf.allowed_encodings;
/* select best matching encoding */
#ifdef USE_ZSTD
if (accept_encoding & HTTP_ACCEPT_ENCODING_ZSTD) {
*label = "zstd";
return HTTP_ACCEPT_ENCODING_ZSTD;
} else
#endif
#ifdef USE_BROTLI
if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) {
*label = "br";

5
src/server.c
View File

@ -555,6 +555,11 @@ static void show_features (void) {
#else
"\t- zlib support\n"
#endif
#if defined HAVE_ZSTD_H && defined HAVE_ZSTD
"\t+ zstd support\n"
#else
"\t- zstd support\n"
#endif
#if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2
"\t+ bzip2 support\n"
#else