[scons] various improvements

- don't generate files in src/
- move all build stuff to sconsbuild/
- have different output directories for static/ and fullstatic/,
  so we can use that directory for the test suite
- each build type (dynamic, static, fullstatic) has its own check target
- read CFLAGS, LDFLAGS and LIBS from environment
  (LIBS are appended after all other dependencies)

From: Stefan Bühler <stbuehler@web.de>

git-svn-id: svn://svn.lighttpd.net/lighttpd/branches/lighttpd-1.4.x@3030 152afb58-edef-0310-8abb-c4023f1b3aa9

.gitignore

@@ -13,3 +13,6 @@ depcomp
 install-sh
 ltmain.sh
 missing
+.sconf_temp
+.sconsign.dblite
+config.log

SConstruct

@@ -70,37 +70,38 @@ int main() {
 	return result
 
 
-BuildDir('build', 'src', duplicate = 0)
+VariantDir('sconsbuild/build', 'src', duplicate = 0)
+VariantDir('sconsbuild/tests', 'tests', duplicate = 0)
 
-opts = Options('config.py')
-opts.AddOptions(
+vars = Variables() #('config.py')
+vars.AddVariables(
 	('prefix', 'prefix', '/usr/local'),
 	('bindir', 'binary directory', '${prefix}/bin'),
 	('sbindir', 'binary directory', '${prefix}/sbin'),
 	('libdir', 'library directory', '${prefix}/lib'),
-	PackageOption('with_mysql', 'enable mysql support', 'no'),
-	PackageOption('with_xml', 'enable xml support', 'no'),
-	PackageOption('with_pcre', 'enable pcre support', 'yes'),
-	PathOption('CC', 'path to the c-compiler', None),
-	BoolOption('build_dynamic', 'enable dynamic build', 'yes'),
-	BoolOption('build_static', 'enable static build', 'no'),
-	BoolOption('build_fullstatic', 'enable fullstatic build', 'no'),
-	BoolOption('with_sqlite3', 'enable sqlite3 support', 'no'),
-	BoolOption('with_memcache', 'enable memcache support', 'no'),
-	BoolOption('with_fam', 'enable FAM/gamin support', 'no'),
-	BoolOption('with_openssl', 'enable memcache support', 'no'),
-	BoolOption('with_gzip', 'enable gzip compression', 'no'),
-	BoolOption('with_bzip2', 'enable bzip2 compression', 'no'),
-	BoolOption('with_lua', 'enable lua support for mod_cml', 'no'),
-	BoolOption('with_ldap', 'enable ldap auth support', 'no'))
+	PackageVariable('with_mysql', 'enable mysql support', 'no'),
+	PackageVariable('with_xml', 'enable xml support', 'no'),
+	PackageVariable('with_pcre', 'enable pcre support', 'yes'),
+	PathVariable('CC', 'path to the c-compiler', None),
+	BoolVariable('build_dynamic', 'enable dynamic build', 'yes'),
+	BoolVariable('build_static', 'enable static build', 'no'),
+	BoolVariable('build_fullstatic', 'enable fullstatic build', 'no'),
+	BoolVariable('with_sqlite3', 'enable sqlite3 support', 'no'),
+	BoolVariable('with_memcache', 'enable memcache support', 'no'),
+	BoolVariable('with_fam', 'enable FAM/gamin support', 'no'),
+	BoolVariable('with_openssl', 'enable memcache support', 'no'),
+	BoolVariable('with_gzip', 'enable gzip compression', 'no'),
+	BoolVariable('with_bzip2', 'enable bzip2 compression', 'no'),
+	BoolVariable('with_lua', 'enable lua support for mod_cml', 'no'),
+	BoolVariable('with_ldap', 'enable ldap auth support', 'no'))
 
 env = Environment(
-	env = os.environ,
-	options = opts,
-	CPPPATH = Split('#build')
+	ENV = os.environ,
+	variables = vars,
+	CPPPATH = Split('#sconsbuild/build')
 )
 
-env.Help(opts.GenerateHelpText(env))
+env.Help(vars.GenerateHelpText(env))
 
 if env.subst('${CC}') is not '':
 	env['CC'] = env.subst('${CC}')
@@ -114,6 +115,19 @@ if env['CC'] == 'gcc':
 # cache configure checks
 if 1:
 	autoconf = Configure(env, custom_tests = {'CheckStructMember': checkStructMember })
+
+	if 'CFLAGS' in os.environ:
+		autoconf.env.Append(CCFLAGS = os.environ['CFLAGS'])
+		print(">> Appending custom build flags : " + os.environ['CFLAGS'])
+
+	if 'LDFLAGS' in os.environ:
+		autoconf.env.Append(LINKFLAGS = os.environ['LDFLAGS'])
+		print(">> Appending custom link flags : " + os.environ['LDFLAGS'])
+
+	if 'LIBS' in os.environ:
+		autoconf.env.Append(APPEND_LIBS = os.environ['LIBS'])
+		print(">> Appending custom libraries : " + os.environ['LIBS'])
+
 	autoconf.headerfile = "foo.h"
 	checkCHeaders(autoconf, string.split("""
 			arpa/inet.h
@@ -124,7 +138,7 @@ if 1:
 			string.h
 			sys/socket.h
 			sys/types.h sys/socket.h
-		 	sys/time.h
+			sys/time.h
 			unistd.h
 			sys/sendfile.h
 			sys/uio.h
@@ -164,7 +178,7 @@ if 1:
 
 	autoconf.env.Append( LIBSQLITE3 = '', LIBXML2 = '', LIBMYSQL = '', LIBZ = '',
 		LIBBZ2 = '', LIBCRYPT = '', LIBMEMCACHE = '', LIBFCGI = '', LIBPCRE = '',
-		LIBLDAP = '', LIBLBER = '', LIBLUA = '', LIBLUALIB = '', LIBDL = '')
+		LIBLDAP = '', LIBLBER = '', LIBLUA = '', LIBDL = '')
 
 	if env['with_fam']:
 		if autoconf.CheckLibWithHeader('fam', 'fam.h', 'C'):
@@ -225,14 +239,28 @@ if 1:
 
 	env = autoconf.Finish()
 
-	if env['with_lua']:
-		oldlibs = env['LIBS']
-		env.ParseConfig("pkg-config 'lua >= 5.0' --cflags --libs")
-		lualibs = env['LIBS'][len(oldlibs):]
-		env.Append(LIBLUA = lualibs)
+def TryLua(env, name):
+	result = False
+	oldlibs = env['LIBS']
+	try:
+		print("Searching for lua: " + name + " >= 5.0")
+		env.ParseConfig("pkg-config '" + name + " >= 5.0' --cflags --libs")
+		env.Append(LIBLUA = env['LIBS'][len(oldlibs):])
 		env.Append(CPPFLAGS = [ '-DHAVE_LUA_H' ])
-		env['LIBS'] = oldlibs
+		result = True
+	except:
+		pass
+	env['LIBS'] = oldlibs
+	return result
 
+if env['with_lua']:
+	found_lua = False
+	for lua_name in ['lua5.1', 'lua5.0', 'lua']:
+		if TryLua(env, lua_name):
+			found_lua = True
+			break
+	if not found_lua:
+		raise RuntimeError("Couldn't find any lua implementation")
 
 if env['with_pcre']:
 	pcre_config = checkProgram(env, 'pcre', 'pcre-config')
@@ -265,12 +293,12 @@ else:
 versions = string.split(version, '.')
 version_id = int(versions[0]) << 16 | int(versions[1]) << 8 | int(versions[2])
 env.Append(CPPFLAGS = [
-		'-DLIGHTTPD_VERSION_ID=' + str(version_id),
+		'-DLIGHTTPD_VERSION_ID=' + hex(version_id),
 		'-DPACKAGE_NAME=\\"' + package + '\\"',
 		'-DPACKAGE_VERSION=\\"' + version + '\\"',
 		'-DLIBRARY_DIR="\\"${libdir}\\""',
 		'-D_FILE_OFFSET_BITS=64', '-D_LARGEFILE_SOURCE', '-D_LARGE_FILES'
 		] )
 
-SConscript( 'src/SConscript', 'env', build_dir = 'build', duplicate = 0)
-SConscript( 'tests/SConscript', 'env' )
+SConscript('src/SConscript', exports = 'env', variant_dir = 'sconsbuild/build', duplicate = 0)
+SConscript('tests/SConscript', exports = 'env', variant_dir = 'sconsbuild/tests')

src/SConscript

@@ -1,42 +1,47 @@
 import os
 import re
 import types
+import itertools
 
 Import('env')
 
+def GatherLibs(env, *libs):
+	return env['LIBS'] + [libs] + [env['APPEND_LIBS']]
+
 common_src = Split("buffer.c log.c \
-      keyvalue.c chunk.c  \
-      http_chunk.c stream.c fdevent.c \
-      stat_cache.c plugin.c joblist.c etag.c array.c \
-      data_string.c data_count.c data_array.c \
-      data_integer.c md5.c data_fastcgi.c \
-      fdevent_select.c fdevent_libev.c \
-      fdevent_poll.c fdevent_linux_sysepoll.c \
-      fdevent_solaris_devpoll.c fdevent_solaris_port.c \
-      fdevent_freebsd_kqueue.c \
-      data_config.c \
-      inet_ntop_cache.c crc32.c \
-      connections-glue.c \
-      configfile-glue.c \
-      http-header-glue.c \
-      splaytree.c network_writev.c \
-      network_write_mmap.c network_write_no_mmap.c \
-      network_write.c network_linux_sendfile.c \
-      network_freebsd_sendfile.c  \
-      network_solaris_sendfilev.c network_openssl.c \
-      status_counter.c \
+	keyvalue.c chunk.c  \
+	http_chunk.c stream.c fdevent.c \
+	stat_cache.c plugin.c joblist.c etag.c array.c \
+	data_string.c data_count.c data_array.c \
+	data_integer.c md5.c data_fastcgi.c \
+	fdevent_select.c fdevent_libev.c \
+	fdevent_poll.c fdevent_linux_sysepoll.c \
+	fdevent_solaris_devpoll.c fdevent_solaris_port.c \
+	fdevent_freebsd_kqueue.c \
+	data_config.c \
+	inet_ntop_cache.c crc32.c \
+	connections-glue.c \
+	configfile-glue.c \
+	http-header-glue.c \
+	splaytree.c network_writev.c \
+	network_write_mmap.c network_write_no_mmap.c \
+	network_write.c network_linux_sendfile.c \
+	network_freebsd_sendfile.c  \
+	network_solaris_sendfilev.c network_openssl.c \
+	status_counter.c \
 ")
 
 src = Split("server.c response.c connections.c network.c \
-      configfile.c configparser.c request.c proc_open.c")
+	configfile.c configparser.c request.c proc_open.c")
 
-lemon = env.Program('lemon', 'lemon.c')
+lemon = env.Program('lemon', 'lemon.c', LIBS = GatherLibs(env))
 
-configparser = env.Command(['configparser.c', 'configparser.h'], 'configparser.y', '(cd build; ../' + lemon[0].path + ' -q ../$SOURCE ../src/lempar.c; cd ..)')
-env.Depends(configparser, lemon)
+def Lemon(env, input):
+	parser = env.Command([input.replace('.y', '.c'),input.replace('.y', '.h')], input, '(cd sconsbuild/build; ../../' + lemon[0].path + ' -q ../../$SOURCE ../../src/lempar.c)')
+	env.Depends(parser, lemon)
 
-mod_ssi_exprparser = env.Command(['mod_ssi_exprparser.c', 'mod_ssi_exprparser.h'], 'mod_ssi_exprparser.y', '(cd build; ../' + lemon[0].path + ' -q ../$SOURCE ../src/lempar.c; cd ..)')
-env.Depends(mod_ssi_exprparser, lemon)
+configparser = Lemon(env, 'configparser.y')
+mod_ssi_exprparser = Lemon(env, 'mod_ssi_exprparser.y')
 
 ## the modules and how they are built
 modules = {
@@ -68,18 +73,23 @@ modules = {
 		'lib' : [ env['LIBCRYPT'], env['LIBLDAP'], env['LIBLBER'] ] },
 	'mod_webdav' : { 'src' : [ 'mod_webdav.c' ], 'lib' : [ env['LIBXML2'], env['LIBSQLITE3'], env['LIBUUID'] ] },
 	'mod_mysql_vhost' : { 'src' : [ 'mod_mysql_vhost.c' ], 'lib' : [ env['LIBMYSQL'] ] },
-	'mod_trigger_b4_dl' : { 'src' : [ 'mod_trigger_b4_dl.c' ], 'lib' : [ env['LIBPCRE'] ] },
-	'mod_cml' : {
-		'src' : [ 'mod_cml_lua.c', 'mod_cml.c', 'mod_cml_funcs.c' ],
-		'lib' : [ env['LIBPCRE'], env['LIBMEMCACHE'], env['LIBLUA'], env['LIBLUALIB'] ] },
 #	'mod_uploadprogress' : { 'src' : [ 'mod_uploadprogress.c' ] },
 	'mod_evasive' : { 'src' : [ 'mod_evasive.c' ] },
 	'mod_ssi' : { 'src' : [ 'mod_ssi_exprparser.c', 'mod_ssi_expr.c', 'mod_ssi.c' ], 'lib' : [ env['LIBPCRE'] ] },
 	'mod_flv_streaming' : { 'src' : [ 'mod_flv_streaming.c' ] },
-	'mod_magnet' : { 'src' : [ 'mod_magnet.c', 'mod_magnet_cache.c' ], 'lib' : [ env['LIBLUA'] ] },
+	'mod_cml': {
+		'src' : [ 'mod_cml_lua.c', 'mod_cml.c', 'mod_cml_funcs.c' ],
+		'lib' : [ env['LIBPCRE'], env['LIBMEMCACHE'], env['LIBLUA'] ] 
+	},
 }
 
-staticenv = env.Copy(CPPFLAGS=[ env['CPPFLAGS'], '-DLIGHTTPD_STATIC', '-DOPENSSL_NO_KRB5'])
+if env['with_memcache']:
+	modules[mod_trigger_b4_dl] = { 'src' : [ 'mod_trigger_b4_dl.c' ], 'lib' : [ env['LIBPCRE'], env['LIBMEMCACHE'] ] }
+
+if env['with_lua']:
+	modules['mod_magnet'] = { 'src' : [ 'mod_magnet.c', 'mod_magnet_cache.c' ], 'lib' : [ env['LIBLUA'] ] }
+
+staticenv = env.Clone(CPPFLAGS=[ env['CPPFLAGS'], '-DLIGHTTPD_STATIC', '-DOPENSSL_NO_KRB5'])
 
 ## all the core-sources + the modules
 staticsrc = src + common_src
@@ -92,16 +102,27 @@ for module in modules.keys():
 	if modules[module].has_key('lib'):
 		staticlib += modules[module]['lib']
 
-open('plugin-static.h', 'w+').write(staticinit)
+def WriteStaticPluginHeader(target, source, env):
+	open(target[0].abspath, 'w+').write(env['STATICINIT'])
+env['STATICINIT'] = staticinit
+staticheader = env.AlwaysBuild(env.Command('plugin-static.h', [], WriteStaticPluginHeader))
 
 ## turn all src-files into objects
 staticobj = []
+static_plugin_obj = None
 for cfile in staticsrc:
-	staticobj += [ staticenv.Object('static-' + cfile.replace('.c', ''), cfile) ]
+	if cfile == 'plugin.c':
+		static_plugin_obj = [ staticenv.Object('static-' + cfile.replace('.c', ''), cfile) ]
+		staticobj += static_plugin_obj
+	else:
+		staticobj += [ staticenv.Object('static-' + cfile.replace('.c', ''), cfile) ]
+env.Depends(static_plugin_obj, 'plugin-static.h')
+print(str(static_plugin_obj))
 
-staticbin = staticenv.Program('lighttpd-semi-static',
+## includes all modules, but links dynamically against other libs
+staticbin = staticenv.Program('../static/build/lighttpd',
 	staticobj,
-	LIBS = staticlib
+	LIBS = GatherLibs(env, staticlib)
 	)
 
 ## you might have to adjust the list of libs and the order for your setup
@@ -128,11 +149,11 @@ for libs in staticlib:
 			fullstaticlib += [ aword ]
 		f.close
 
-
-fullstaticbin = staticenv.Program('lighttpd-static',
+## includes all modules, linked statically
+fullstaticbin = staticenv.Program('../fullstatic/build/lighttpd',
 	staticobj,
-	LIBS = fullstaticlib,
-	LINKFLAGS= ['-static']
+	LIBS = GatherLibs(env, fullstaticlib),
+	LINKFLAGS= [staticenv['LINKFLAGS'], '-static']
 	)
 
 Alias('static', staticbin)
@@ -152,7 +173,7 @@ else:
 	else:
 		bin_linkflags += [ '-Wl,--export-dynamic' ]
 
-instbin = env.Program(bin_targets, src, LINKFLAGS = bin_linkflags, LIBS= [ env['LIBS'], common_lib, env['LIBDL'] ])
+instbin = env.Program(bin_targets, src, LINKFLAGS = bin_linkflags, LIBS = GatherLibs(env, env['LIBS'], common_lib, env['LIBDL']))
 env.Depends(instbin, configparser)
 
 if env['COMMON_LIB'] == 'bin':
@@ -164,7 +185,8 @@ for module in modules.keys():
 	libs = [ common_lib ]
 	if modules[module].has_key('lib'):
 		libs +=  modules[module]['lib']
-	instlib += env.SharedLibrary(module, modules[module]['src'], LIBS= [ libs ])
+	instlib += env.SharedLibrary(module, modules[module]['src'], LIBS= GatherLibs(env, libs))
+env.Alias('modules', instlib)
 
 inst = []
 
@@ -190,4 +212,3 @@ env.Alias('install', inst)
 
 pkgdir = '.'
 tarname = env['package'] + '-' + env['version']
-

tests/SConscript

@@ -1,46 +1,87 @@
 Import('env')
 
 tests = Split('prepare.sh \
-        run-tests.pl \
-        cleanup.sh')
+	run-tests.pl \
+	cleanup.sh')
 
 extra_dist = Split('fastcgi-10.conf \
-      fastcgi-auth.conf \
-      fastcgi-responder.conf \
-      fastcgi-13.conf \
-      bug-06.conf \
-      bug-12.conf \
-      core-var-include.t \
-      var-include.conf \
-      var-include-sub.conf \
-      condition.conf \
-      core-condition.t \
-      core-request.t \
-      core-response.t \
-      core-keepalive.t \
-      core.t \
-      mod-access.t \
-      mod-auth.t \
-      mod-cgi.t \
-      mod-compress.t \
-      mod-compress.conf \
-      mod-fastcgi.t \
-      mod-redirect.t \
-      mod-userdir.t \
-      mod-rewrite.t \
-      request.t \
-      mod-ssi.t \
-      LightyTest.pm \
-      mod-setenv.t')
-
-t  = env.Command('foo1', 'prepare.sh', '(cd ./tests/; ./prepare.sh; cd ..)')
-t += env.Command('foo2', 'run-tests.pl', '( cd ./tests/; SHELL=/bin/sh ./run-tests.pl; cd ..)')
-t += env.Command('foo3', 'cleanup.sh', '(cd ./tests/; ./cleanup.sh; cd ..)')
+	fastcgi-auth.conf \
+	fastcgi-responder.conf \
+	fastcgi-13.conf \
+	bug-06.conf \
+	bug-12.conf \
+	core-var-include.t \
+	var-include.conf \
+	var-include-sub.conf \
+	condition.conf \
+	core-condition.t \
+	core-request.t \
+	core-response.t \
+	core-keepalive.t \
+	core.t \
+	mod-access.t \
+	mod-auth.t \
+	mod-cgi.t \
+	mod-compress.t \
+	mod-compress.conf \
+	mod-fastcgi.t \
+	mod-redirect.t \
+	mod-userdir.t \
+	mod-rewrite.t \
+	request.t \
+	mod-ssi.t \
+	LightyTest.pm \
+	mod-setenv.t')
+
+fcgi_auth = None
+fcgi_responder = None
 
 if env['LIBFCGI']:
-	fcgis = []
-	fcgis += env.Program("fcgi-auth", "fcgi-auth.c", LIBS=env['LIBFCGI'])
-	fcgis += env.Program("fcgi-responder", "fcgi-responder.c", LIBS=env['LIBFCGI'])
-	env.Depends(t, fcgis)
+	fcgi_auth = env.Program("fcgi-auth", "fcgi-auth.c", LIBS=[env['LIBFCGI'], env['APPEND_LIBS']])
+	fcgi_responder = env.Program("fcgi-responder", "fcgi-responder.c", LIBS=[env['LIBFCGI'], env['APPEND_LIBS']])
+
+def CopyTestBinary(env, binary):
+	return env.Command(target = env['ENV']['top_builddir'] + '/tests/' + binary, source = binary, action = Copy("$TARGET", "$SOURCE"))
+
+def BuildTestEnv(env, build_type):
+	builddir = build_type
+	dependencies = [build_type]
+	if build_type == 'dynamic':
+		builddir = '.'
+		dependencies += ['modules']
+
+	testenv = env.Clone()
+	testenv['ENV']['srcdir']='tests'
+	testenv['ENV']['top_builddir']='sconsbuild/' + builddir
+	prepare = testenv.AlwaysBuild(testenv.Command(build_type + '/prepare', 'prepare.sh', 'tests/prepare.sh'))
+	runtests = testenv.AlwaysBuild(testenv.Command(build_type + '/run-tests', 'run-tests.pl', 'tests/run-tests.pl'))
+	cleanup = testenv.AlwaysBuild(testenv.Command(build_type + '/cleanup', 'cleanup.sh', 'tests/cleanup.sh'))
+	testenv.Depends(runtests, prepare)
+	testenv.Depends(cleanup, runtests)
+	SideEffect('dummy-file-prevent-running-tests-in-parallel', runtests)
+
+	testenv.Depends(runtests, dependencies)
+
+	if env['LIBFCGI']:
+		fcgis = [CopyTestBinary(testenv, 'fcgi-auth'), CopyTestBinary(testenv, 'fcgi-responder')]
+		testenv.Depends(runtests, fcgis)
+
+	return [prepare, runtests, cleanup]
+
+check_dynamic = env.Alias('check_dynamic', BuildTestEnv(env, 'dynamic'))
+env.Depends(check_dynamic, 'modules')
+check_static = env.Alias('check_static', BuildTestEnv(env, 'static'))
+check_fullstatic = env.Alias('check_fullstatic', BuildTestEnv(env, 'fullstatic'))
+
+checks = []
+
+if env['build_dynamic']:
+	checks += check_dynamic
+
+if env['build_static']:
+	checks += check_static
+
+if env['build_fullstatic']:
+	checks += check_fullstatic
 
-env.Alias('check', t )
+env.Alias('check', checks)

tests/cleanup.sh

@@ -4,10 +4,14 @@ if test x$srcdir = x; then
 	srcdir=.
 fi
 
-tmpdir=$top_builddir/tests/tmp/
+if test x$top_builddir = x; then
+	top_builddir=..
+fi
+
+tmpdir="$top_builddir/tests/tmp/"
 
 # remove test-framework
-rm -rf $tmpdir
+rm -rf "$tmpdir"
 
 printf "%-40s" "cleaning up"