[mod_deflate] support Accept-Encoding: zstd
This commit is contained in:
parent
ef28cce5e5
commit
9211fb3d86
10
SConstruct
10
SConstruct
|
@ -269,6 +269,7 @@ vars.AddVariables(
|
|||
PackageVariable('with_xml', 'enable xml support (required for webdav props)', 'no'),
|
||||
BoolVariable('with_xxhash', 'build with system-provided xxhash', 'no'),
|
||||
BoolVariable('with_zlib', 'enable deflate/gzip compression', 'no'),
|
||||
BoolVariable('with_zstd', 'enable zstd compression', 'no'),
|
||||
|
||||
BoolVariable('with_all', 'enable all with_* features', 'no'),
|
||||
)
|
||||
|
@ -357,6 +358,7 @@ if 1:
|
|||
LIBXML2 = '',
|
||||
LIBXXHASH = '',
|
||||
LIBZ = '',
|
||||
LIBZSTD = '',
|
||||
)
|
||||
|
||||
autoconf.haveCHeaders([
|
||||
|
@ -731,6 +733,14 @@ if 1:
|
|||
LIBZ = 'z',
|
||||
)
|
||||
|
||||
if env['with_zstd']:
|
||||
if not autoconf.CheckLibWithHeader('zstd', 'zstd.h', 'C'):
|
||||
fail("Couldn't find zstd")
|
||||
autoconf.env.Append(
|
||||
CPPFLAGS = [ '-DHAVE_ZSTD_H', '-DHAVE_ZSTD' ],
|
||||
LIBZSTD = 'zstd',
|
||||
)
|
||||
|
||||
env = autoconf.Finish()
|
||||
|
||||
if re.compile("cygwin|mingw|midipix").search(env['PLATFORM']):
|
||||
|
|
31
configure.ac
31
configure.ac
|
@ -962,6 +962,37 @@ if test "$WITH_ZLIB" != no; then
|
|||
AC_SUBST([Z_LIB])
|
||||
fi
|
||||
|
||||
dnl zstd
|
||||
AC_MSG_NOTICE([----------------------------------------])
|
||||
AC_MSG_CHECKING([for zstd support])
|
||||
AC_ARG_WITH([zstd],
|
||||
[AS_HELP_STRING([--with-zstd],
|
||||
[Enable zstd support for mod_deflate]
|
||||
)],
|
||||
[WITH_ZSTD=$withval],
|
||||
[WITH_ZSTD=no]
|
||||
)
|
||||
AC_MSG_RESULT([$WITH_ZSTD])
|
||||
|
||||
if test "$WITH_ZSTD" != no; then
|
||||
if test "$WITH_ZSTD" != yes; then
|
||||
ZSTD_LIB="-L$WITH_ZSTD -lzstd"
|
||||
CPPFLAGS="$CPPFLAGS -I$WITH_ZSTD"
|
||||
else
|
||||
AC_CHECK_HEADERS([zstd.h], [],
|
||||
[AC_MSG_ERROR([zstd headers not found, install them or build without --with-zstd])]
|
||||
)
|
||||
AC_CHECK_LIB([zstd], [ZSTD_versionNumber],
|
||||
[ZSTD_LIB=-lzstd],
|
||||
[AC_MSG_ERROR([zstd library not found, install it or build without --with-zstd])]
|
||||
)
|
||||
fi
|
||||
|
||||
AC_DEFINE([HAVE_ZSTD], [1], [libzstd])
|
||||
AC_DEFINE([HAVE_ZSTD_H], [1])
|
||||
AC_SUBST([ZSTD_LIB])
|
||||
fi
|
||||
|
||||
dnl bzip2
|
||||
AC_MSG_NOTICE([----------------------------------------])
|
||||
AC_MSG_CHECKING([for bzip2 support])
|
||||
|
|
|
@ -32,6 +32,7 @@ option(WITH_WEBDAV_LOCKS "locks in webdav [default: off]")
|
|||
option(WITH_BROTLI "with brotli-support for mod_deflate [default: off]")
|
||||
option(WITH_BZIP "with bzip2-support for mod_deflate [default: off]")
|
||||
option(WITH_ZLIB "with deflate-support for mod_deflate [default: on]" ON)
|
||||
option(WITH_ZSTD "with zstd-support for mod_deflate [default: off]")
|
||||
option(WITH_KRB5 "with Kerberos5-support for mod_auth [default: off]")
|
||||
option(WITH_LDAP "with LDAP-support for mod_auth mod_vhostdb_ldap [default: off]")
|
||||
option(WITH_PAM "with PAM-support for mod_auth [default: off]")
|
||||
|
@ -607,6 +608,14 @@ else()
|
|||
unset(ZLIB_LIBRARY)
|
||||
endif()
|
||||
|
||||
if(WITH_ZSTD)
|
||||
check_include_files(zstd.h HAVE_ZSTD_H)
|
||||
check_library_exists(zstd ZSTD_versionNumber "" HAVE_ZSTD)
|
||||
else()
|
||||
unset(HAVE_ZSTD_H)
|
||||
unset(HAVE_ZSTD)
|
||||
endif()
|
||||
|
||||
if(WITH_BZIP)
|
||||
check_include_files(bzlib.h HAVE_BZLIB_H)
|
||||
check_library_exists(bz2 BZ2_bzCompress "" HAVE_LIBBZ2)
|
||||
|
@ -1063,10 +1072,13 @@ if(WITH_SASL)
|
|||
target_link_libraries(mod_authn_sasl ${L_MOD_AUTHN_SASL})
|
||||
endif()
|
||||
|
||||
if(HAVE_ZLIB_H OR HAVE_BZLIB_H OR HAVE_BROTLI)
|
||||
if(HAVE_ZLIB_H OR HAVE_ZSTD_H OR HAVE_BZLIB_H OR HAVE_BROTLI)
|
||||
if(HAVE_ZLIB_H)
|
||||
set(L_MOD_DEFLATE ${L_MOD_DEFLATE} ${ZLIB_LIBRARY})
|
||||
endif()
|
||||
if(HAVE_ZSTD_H)
|
||||
set(L_MOD_DEFLATE ${L_MOD_DEFLATE} zstd)
|
||||
endif()
|
||||
if(HAVE_BZLIB_H)
|
||||
set(L_MOD_DEFLATE ${L_MOD_DEFLATE} bz2)
|
||||
endif()
|
||||
|
|
|
@ -326,7 +326,7 @@ mod_access_la_LIBADD = $(common_libadd)
|
|||
lib_LTLIBRARIES += mod_deflate.la
|
||||
mod_deflate_la_SOURCES = mod_deflate.c
|
||||
mod_deflate_la_LDFLAGS = $(BROTLI_CFLAGS) $(common_module_ldflags)
|
||||
mod_deflate_la_LIBADD = $(Z_LIB) $(BZ_LIB) $(BROTLI_LIBS) $(common_libadd)
|
||||
mod_deflate_la_LIBADD = $(Z_LIB) $(ZSTD_LIB) $(BZ_LIB) $(BROTLI_LIBS) $(common_libadd)
|
||||
|
||||
lib_LTLIBRARIES += mod_auth.la
|
||||
mod_auth_la_SOURCES = mod_auth.c
|
||||
|
@ -525,7 +525,7 @@ lighttpd_LDADD = \
|
|||
$(common_libadd) \
|
||||
$(CRYPT_LIB) $(CRYPTO_LIB) $(XXHASH_LIBS) \
|
||||
$(XML_LIBS) $(SQLITE_LIBS) $(UUID_LIBS) $(ELFTC_LIB) \
|
||||
$(PCRE_LIB) $(Z_LIB) $(BZ_LIB) $(BROTLI_LIBS) \
|
||||
$(PCRE_LIB) $(Z_LIB) $(ZSTD_LIB) $(BZ_LIB) $(BROTLI_LIBS) \
|
||||
$(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) \
|
||||
$(FAM_LIBS) $(LIBEV_LIBS) $(LIBUNWIND_LIBS)
|
||||
lighttpd_LDFLAGS = -export-dynamic
|
||||
|
|
|
@ -105,7 +105,7 @@ modules = {
|
|||
'mod_auth' : { 'src' : [ 'mod_auth.c' ], 'lib' : [ env['LIBCRYPTO'] ] },
|
||||
'mod_authn_file' : { 'src' : [ 'mod_authn_file.c' ], 'lib' : [ env['LIBCRYPT'], env['LIBCRYPTO'] ] },
|
||||
'mod_cgi' : { 'src' : [ 'mod_cgi.c' ] },
|
||||
'mod_deflate' : { 'src' : [ 'mod_deflate.c' ], 'lib' : [ env['LIBZ'], env['LIBBZ2'], env['LIBBROTLI'], 'm' ] },
|
||||
'mod_deflate' : { 'src' : [ 'mod_deflate.c' ], 'lib' : [ env['LIBZ'], env['LIBZSTD'], env['LIBBZ2'], env['LIBBROTLI'], 'm' ] },
|
||||
'mod_dirlisting' : { 'src' : [ 'mod_dirlisting.c' ], 'lib' : [ env['LIBPCRE'] ] },
|
||||
'mod_evasive' : { 'src' : [ 'mod_evasive.c' ] },
|
||||
'mod_evhost' : { 'src' : [ 'mod_evhost.c' ] },
|
||||
|
|
|
@ -683,6 +683,21 @@ if get_option('with_zlib')
|
|||
conf_data.set('HAVE_LIBZ', true)
|
||||
endif
|
||||
|
||||
libzstd = []
|
||||
if get_option('with_zstd')
|
||||
libz = dependency('zstd', required: false)
|
||||
if libzstd.found()
|
||||
libzstd = [ libzstd ]
|
||||
else
|
||||
libzstd = [ compiler.find_library('zstd') ]
|
||||
if not(compiler.has_function('ZSTD_versionNumber', args: defs, dependencies: libzstd, prefix: '#include <zstd.h>'))
|
||||
error('Couldn\'t find zstd header / library')
|
||||
endif
|
||||
endif
|
||||
conf_data.set('HAVE_ZSTD_H', true)
|
||||
conf_data.set('HAVE_ZSTD', true)
|
||||
endif
|
||||
|
||||
configure_file(
|
||||
output : 'config.h',
|
||||
configuration : conf_data,
|
||||
|
@ -977,7 +992,7 @@ modules = [
|
|||
[ 'mod_alias', [ 'mod_alias.c' ] ],
|
||||
[ 'mod_auth', [ 'mod_auth.c' ], [ libcrypto ] ],
|
||||
[ 'mod_authn_file', [ 'mod_authn_file.c' ], [ libcrypt, libcrypto ] ],
|
||||
[ 'mod_deflate', [ 'mod_deflate.c' ], libbz2 + libz + libbrotli ],
|
||||
[ 'mod_deflate', [ 'mod_deflate.c' ], libbz2 + libz + libzstd + libbrotli ],
|
||||
[ 'mod_dirlisting', [ 'mod_dirlisting.c' ], libpcre ],
|
||||
[ 'mod_evasive', [ 'mod_evasive.c' ] ],
|
||||
[ 'mod_evhost', [ 'mod_evhost.c' ] ],
|
||||
|
|
|
@ -144,6 +144,14 @@
|
|||
# include <brotli/encode.h>
|
||||
#endif
|
||||
|
||||
#if defined HAVE_ZSTD_H && defined HAVE_ZSTD
|
||||
# define USE_ZSTD
|
||||
/* FIXME: wrote initial implementation to support zstd and then realized
|
||||
* bufferless streaming compression is an experimental (unstable) zstd API */
|
||||
# define ZSTD_STATIC_LINKING_ONLY
|
||||
# include <zstd.h>
|
||||
#endif
|
||||
|
||||
#if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
|
||||
#define USE_MMAP
|
||||
|
||||
|
@ -170,6 +178,7 @@ static void sigbus_handler(int sig) {
|
|||
#define HTTP_ACCEPT_ENCODING_X_GZIP BV(5)
|
||||
#define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6)
|
||||
#define HTTP_ACCEPT_ENCODING_BR BV(7)
|
||||
#define HTTP_ACCEPT_ENCODING_ZSTD BV(8)
|
||||
|
||||
typedef struct {
|
||||
const array *mimetypes;
|
||||
|
@ -202,6 +211,9 @@ typedef struct {
|
|||
#endif
|
||||
#ifdef USE_BROTLI
|
||||
BrotliEncoderState *br;
|
||||
#endif
|
||||
#ifdef USE_ZSTD
|
||||
ZSTD_CCtx *cctx;
|
||||
#endif
|
||||
int dummy;
|
||||
} u;
|
||||
|
@ -244,7 +256,12 @@ static void handler_ctx_free(handler_ctx *hctx) {
|
|||
|
||||
INIT_FUNC(mod_deflate_init) {
|
||||
plugin_data * const p = calloc(1, sizeof(plugin_data));
|
||||
#ifdef USE_ZSTD
|
||||
buffer_string_prepare_copy(&p->tmp_buf, /* 131072 */
|
||||
ZSTD_COMPRESSBOUND(ZSTD_BLOCKSIZE_MAX));
|
||||
#else
|
||||
buffer_string_prepare_copy(&p->tmp_buf, 65536);
|
||||
#endif
|
||||
return p;
|
||||
}
|
||||
|
||||
|
@ -384,7 +401,8 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
|
|||
short allowed_encodings = 0;
|
||||
if (encodings->used) {
|
||||
for (uint32_t j = 0; j < encodings->used; ++j) {
|
||||
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI)
|
||||
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \
|
||||
|| defined(USE_ZSTD)
|
||||
data_string *ds = (data_string *)encodings->data[j];
|
||||
#endif
|
||||
#ifdef USE_ZLIB
|
||||
|
@ -411,6 +429,10 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
|
|||
if (NULL != strstr(ds->value.ptr, "br"))
|
||||
allowed_encodings |= HTTP_ACCEPT_ENCODING_BR;
|
||||
#endif
|
||||
#ifdef USE_ZSTD
|
||||
if (NULL != strstr(ds->value.ptr, "zstd"))
|
||||
allowed_encodings |= HTTP_ACCEPT_ENCODING_ZSTD;
|
||||
#endif
|
||||
}
|
||||
}
|
||||
else {
|
||||
|
@ -427,6 +449,9 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
|
|||
#ifdef USE_BROTLI
|
||||
allowed_encodings |= HTTP_ACCEPT_ENCODING_BR;
|
||||
#endif
|
||||
#ifdef USE_ZSTD
|
||||
allowed_encodings |= HTTP_ACCEPT_ENCODING_ZSTD;
|
||||
#endif
|
||||
}
|
||||
return allowed_encodings;
|
||||
}
|
||||
|
@ -612,7 +637,8 @@ SETDEFAULTS_FUNC(mod_deflate_set_defaults) {
|
|||
}
|
||||
|
||||
|
||||
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI)
|
||||
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \
|
||||
|| defined(USE_ZSTD)
|
||||
static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) {
|
||||
ssize_t wr;
|
||||
do {
|
||||
|
@ -933,6 +959,71 @@ static int stream_br_end(handler_ctx *hctx) {
|
|||
#endif
|
||||
|
||||
|
||||
#ifdef USE_ZSTD
|
||||
|
||||
static int stream_zstd_init(handler_ctx *hctx) {
|
||||
ZSTD_CCtx * const cctx = hctx->u.cctx = ZSTD_createCCtx();
|
||||
if (NULL == cctx) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int stream_zstd_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) {
|
||||
ZSTD_CCtx * const cctx = hctx->u.cctx;
|
||||
|
||||
/* Note: each chunkqueue chunk is sent in its own frame, which may be
|
||||
* suboptimal. lighttpd might read FILE_CHUNK into a reused buffer, so
|
||||
* we can not meet ZSTD_compressContinue() requirement that prior input
|
||||
* is still accessible and unmodified (up to maximum distance size).
|
||||
* Also, chunkqueue_mark_written() on MEM_CHUNK might result in something
|
||||
* else reusing those chunk buffers
|
||||
*
|
||||
* future: migrate to use Zstd streaming API */
|
||||
|
||||
/* future: consider allowing tunables by encoder algorithm,
|
||||
* (i.e. not generic "compression_level" across all compression algorithms)
|
||||
* (ZSTD_CCtx_setParameter()) */
|
||||
/*(note: we ignore any errors while tuning parameters here)*/
|
||||
const plugin_data * const p = hctx->plugin_data;
|
||||
int level = ZSTD_CLEVEL_DEFAULT;
|
||||
if (p->conf.compression_level >= 0) /* -1 is lighttpd default for "unset" */
|
||||
level = p->conf.compression_level;
|
||||
ZSTD_compressBegin(cctx, level);
|
||||
|
||||
char * const out = hctx->output->ptr;
|
||||
const size_t outsz = hctx->output->size;
|
||||
hctx->bytes_in += st_size;
|
||||
for (off_t off = 0; st_size; ) {
|
||||
/* XXX: size check must match mod_deflate_init ZSTD_COMPRESSBOUND arg */
|
||||
size_t len = (size_t)st_size;
|
||||
const size_t rv = (len > ZSTD_BLOCKSIZE_MAX)
|
||||
? ZSTD_compressContinue(cctx, out, outsz, start+off,
|
||||
(len = ZSTD_BLOCKSIZE_MAX))
|
||||
: ZSTD_compressEnd(cctx, out, outsz, start+off, len);
|
||||
off += (off_t)len;
|
||||
st_size -= (off_t)len;
|
||||
if (ZSTD_isError(rv)) return -1;
|
||||
hctx->bytes_out += (off_t)rv;
|
||||
if (0 != stream_http_chunk_append_mem(hctx, out, rv))
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int stream_zstd_flush(handler_ctx * const hctx, int end) {
|
||||
UNUSED(hctx);
|
||||
UNUSED(end);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int stream_zstd_end(handler_ctx *hctx) {
|
||||
ZSTD_CCtx * const cctx = hctx->u.cctx;
|
||||
ZSTD_freeCCtx(cctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
static int mod_deflate_stream_init(handler_ctx *hctx) {
|
||||
switch(hctx->compression_type) {
|
||||
#ifdef USE_ZLIB
|
||||
|
@ -947,6 +1038,10 @@ static int mod_deflate_stream_init(handler_ctx *hctx) {
|
|||
#ifdef USE_BROTLI
|
||||
case HTTP_ACCEPT_ENCODING_BR:
|
||||
return stream_br_init(hctx);
|
||||
#endif
|
||||
#ifdef USE_ZSTD
|
||||
case HTTP_ACCEPT_ENCODING_ZSTD:
|
||||
return stream_zstd_init(hctx);
|
||||
#endif
|
||||
default:
|
||||
return -1;
|
||||
|
@ -968,6 +1063,10 @@ static int mod_deflate_compress(handler_ctx * const hctx, unsigned char * const
|
|||
#ifdef USE_BROTLI
|
||||
case HTTP_ACCEPT_ENCODING_BR:
|
||||
return stream_br_compress(hctx, start, st_size);
|
||||
#endif
|
||||
#ifdef USE_ZSTD
|
||||
case HTTP_ACCEPT_ENCODING_ZSTD:
|
||||
return stream_zstd_compress(hctx, start, st_size);
|
||||
#endif
|
||||
default:
|
||||
UNUSED(start);
|
||||
|
@ -990,6 +1089,10 @@ static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) {
|
|||
#ifdef USE_BROTLI
|
||||
case HTTP_ACCEPT_ENCODING_BR:
|
||||
return stream_br_flush(hctx, end);
|
||||
#endif
|
||||
#ifdef USE_ZSTD
|
||||
case HTTP_ACCEPT_ENCODING_ZSTD:
|
||||
return stream_zstd_flush(hctx, end);
|
||||
#endif
|
||||
default:
|
||||
UNUSED(end);
|
||||
|
@ -1023,6 +1126,10 @@ static int mod_deflate_stream_end(handler_ctx *hctx) {
|
|||
#ifdef USE_BROTLI
|
||||
case HTTP_ACCEPT_ENCODING_BR:
|
||||
return stream_br_end(hctx);
|
||||
#endif
|
||||
#ifdef USE_ZSTD
|
||||
case HTTP_ACCEPT_ENCODING_ZSTD:
|
||||
return stream_zstd_end(hctx);
|
||||
#endif
|
||||
default:
|
||||
return -1;
|
||||
|
@ -1263,7 +1370,8 @@ static handler_t deflate_compress_response(request_st * const r, handler_ctx * c
|
|||
static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) {
|
||||
/* get client side support encodings */
|
||||
int accept_encoding = 0;
|
||||
#if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI)
|
||||
#if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) \
|
||||
&& !defined(USE_ZSTD)
|
||||
UNUSED(value);
|
||||
UNUSED(label);
|
||||
#else
|
||||
|
@ -1284,6 +1392,13 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const
|
|||
#ifdef USE_ZLIB
|
||||
if (0 == memcmp(v, "gzip", 4))
|
||||
accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP;
|
||||
#endif
|
||||
#ifdef USE_ZSTD
|
||||
#ifdef USE_ZLIB
|
||||
else
|
||||
#endif
|
||||
if (0 == memcmp(v, "zstd", 4))
|
||||
accept_encoding |= HTTP_ACCEPT_ENCODING_ZSTD;
|
||||
#endif
|
||||
break;
|
||||
case 5:
|
||||
|
@ -1330,6 +1445,12 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const
|
|||
accept_encoding &= p->conf.allowed_encodings;
|
||||
|
||||
/* select best matching encoding */
|
||||
#ifdef USE_ZSTD
|
||||
if (accept_encoding & HTTP_ACCEPT_ENCODING_ZSTD) {
|
||||
*label = "zstd";
|
||||
return HTTP_ACCEPT_ENCODING_ZSTD;
|
||||
} else
|
||||
#endif
|
||||
#ifdef USE_BROTLI
|
||||
if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) {
|
||||
*label = "br";
|
||||
|
|
|
@ -555,6 +555,11 @@ static void show_features (void) {
|
|||
#else
|
||||
"\t- zlib support\n"
|
||||
#endif
|
||||
#if defined HAVE_ZSTD_H && defined HAVE_ZSTD
|
||||
"\t+ zstd support\n"
|
||||
#else
|
||||
"\t- zstd support\n"
|
||||
#endif
|
||||
#if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2
|
||||
"\t+ bzip2 support\n"
|
||||
#else
|
||||
|
|
Loading…
Reference in New Issue