2
0
Fork 0

[tests] Create testing framework

This commit is contained in:
Stefan Bühler 2010-10-02 22:45:29 +02:00
parent b432e35eeb
commit d32de32b85
10 changed files with 800 additions and 0 deletions

View File

@ -10,3 +10,5 @@ SET(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
ENABLE_TESTING()
ADD_SUBDIRECTORY(src build)
add_subdirectory(tests)

View File

@ -347,5 +347,6 @@ AC_CONFIG_FILES([Makefile \
src/modules/Makefile \
src/unittests/Makefile \
src/lighttpd2.pc \
tests/Makefile \
])
AC_OUTPUT

2
tests/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
*.pyc
tmp*

4
tests/CMakeLists.txt Normal file
View File

@ -0,0 +1,4 @@
cmake_policy(VERSION 2.6.4)
add_test(NAME http COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/runtests.py --angel $<TARGET_FILE:lighttpd2> --worker $<TARGET_FILE:lighttpd2-worker> --plugindir $<TARGET_FILE_DIR:lighttpd2>)

347
tests/base.py Normal file
View File

@ -0,0 +1,347 @@
# -*- coding: utf-8 -*-
import os
import imp
import sys
import traceback
from service import *
__all__ = [ "Env", "Tests", "TestBase" ]
class Dict(object):
pass
Env = Dict()
def fix_test_name(name):
if None == name: return '/'
if (name[:1] != '/'): name = '/' + name
if (name[-1:] != '/'): name = name + '/'
return name
def load_test_file(name):
path = os.path.join(Env.sourcedir, name)
file = open(path)
(modname, ext) = os.path.splitext(name)
module = imp.load_module(modname, file, path, (ext, 'r', imp.PY_SOURCE))
file.close()
return module
def vhostname(testname):
return testname[1:-1].replace('/', '.')
# basic interface
class TestBase(object):
config = "defaultaction;"
name = None
vhost = None
runnable = True
def __init__(self):
self._test_cleanup_files = []
self._test_cleanup_dirs = []
self._test_failed = False # "not run" is "successful"
# internal methods, do not override
def _register(self, tests):
self.tests = tests
if not self.vhost: self.vhost = vhostname(self.name)
self.vhostdir = os.path.join(Env.dir, 'www', 'vhosts', self.vhost)
tests.add_test(self)
def _prepare(self):
self.Prepare()
if None != self.config:
errorlog = self.PrepareFile("log/error.log-%s" % self.vhost, "")
accesslog = self.PrepareFile("log/access.log-%s" % self.vhost, "")
config = """
var.vhosts = var.vhosts + [ "%s" : ${
log = [ "*": "file:%s" ];
accesslog = "%s";
%s
}
];
""" % (self.vhost, errorlog, accesslog, self.config)
self.tests.append_config(config)
def _run(self):
failed = False
print >> Env.log, "[Start] Running test %s" % (self.name)
try:
if not self.Run():
failed = True
print >> sys.stderr, "Test %s failed" % (self.name)
except Exception as e:
failed = True
print >> sys.stderr, "Test %s failed:" % (self.name)
print >> sys.stderr, traceback.format_exc(10)
print >> Env.log, "[Done] Running test %s [result=%s]" % (self.name, failed and "Failed" or "Succeeded")
self._test_failed = failed
return not failed
def _cleanup(self):
if Env.no_cleanup or (not Env.force_cleanup and self._test_failed):
return
self.Cleanup()
for f in self._test_cleanup_files:
self._cleanupFile(f)
for d in self._test_cleanup_dirs:
self._cleanupDir(d)
def _cleanupFile(self, fname):
self.tests.CleanupFile(fname)
def _cleanupDir(self, dirname):
self.tests.CleanupDir(dirname)
# public
def PrepareVHostFile(self, fname, content):
"""remembers which files have been prepared and while remove them on cleanup; returns absolute pathname"""
fname = 'www/vhosts/' + self.vhost + '/' + fname
return self.tests.PrepareFile(fname, content)
def PrepareFile(self, fname, content):
"""remembers which files have been prepared and while remove them on cleanup; returns absolute pathname"""
self._test_cleanup_files.append(fname)
return self.tests.PrepareFile(fname, content)
def PrepareDir(self, dirname):
"""remembers which directories have been prepared and while remove them on cleanup; returns absolute pathname"""
self._test_cleanup_dirs.append(fname)
return self.tests.PrepareDir(dirname)
# implement these yourself
def Prepare(self):
pass
def Run(self):
raise BaseException("Test '%s' not implemented yet" % self.name)
def Cleanup(self):
pass
class Tests(object):
def __init__(self):
self.tests_filter = []
if 0 == len(Env.tests):
self.tests_filter.append("")
else:
for t in Env.tests:
self.tests_filter.append(fix_test_name(t))
self.services = []
self.run = [] # tests we want to run
self.tests = [] # all tests (we always prepare/cleanup all tests)
self.tests_dict = { }
self.config = None
self.prepared_dirs = { }
self.prepared_files = { }
self.failed = False
self.add_service(Lighttpd())
def add_test(self, test):
name = test.name
if self.tests_dict.has_key(name):
raise BaseException("Test '%s' already defined" % name)
self.tests_dict[name] = test
for f in self.tests_filter:
if name.startswith(f):
if test.runnable:
self.run.append(test)
break
self.tests.append(test)
def add_service(self, service):
service.tests = self
self.services.append(service)
def append_config(self, config):
if None == self.config:
raise BaseException("Not prepared for adding config")
self.config += config
def LoadTests(self):
files = os.listdir(Env.sourcedir)
files = filter(lambda x: x[-3:] == '.py', files)
files = filter(lambda x: x[:2] == 't-', files)
files.sort()
mods = []
for f in files:
mods.append(load_test_file(f))
for m in mods:
t = m.Test()
t.name = fix_test_name(t.name)
if '/' == t.name:
(n, _) = os.path.splitext(os.path.basename(m.__file__))
t.name = fix_test_name(n[2:])
t._register(self)
def Prepare(self):
print >> Env.log, "[Start] Preparing tests"
errorlog = self.PrepareFile("log/error.log", "")
accesslog = self.PrepareFile("log/access.log", "")
self.config = """
setup {{
workers 2;
module_load (
"mod_accesslog",
"mod_dirlist",
"mod_lua",
"mod_vhost"
);
listen "127.0.0.1:{Env.port}";
log = [ "*": "stderr" ];
lua.plugin "{Env.luadir}/core.lua";
lua.plugin "{Env.luadir}/secdownload.lua";
accesslog.format = "%h %V %u %t \\"%r\\" %>s %b \\"%{{Referer}}i\\" \\"%{{User-Agent}}i\\"";
accesslog = "{accesslog}";
}}
log = [ "*": "file:{errorlog}" ];
defaultaction {{
docroot "{Env.defaultwww}";
}}
var.vhosts = [ "default": ${{
defaultaction;
}} ];
""".format(Env = Env, errorlog = errorlog, accesslog = accesslog)
for t in self.tests:
print >> Env.log, "[Start] Preparing test '%s'" % (t.name)
t._prepare()
self.config += """
vhost.map var.vhosts;
"""
Env.lighttpdconf = self.PrepareFile("conf/lighttpd.conf", self.config)
Env.angelconf = self.PrepareFile("conf/angel.conf", """
instance {{
binary "{Env.worker}";
config "{Env.lighttpdconf}";
modules "{Env.plugindir}";
}}
allow-listen {{ ip "127.0.0.1:{Env.port}"; }}
""".format(Env = Env))
print >> Env.log, "[Done] Preparing tests"
print >> Env.log, "[Start] Preparing services"
for s in self.services:
try:
s._prepare()
except:
self.failed = True
raise
print >> Env.log, "[Done] Preparing services"
def Run(self):
print >> Env.log, "[Start] Running tests"
failed = False
for t in self.run:
if not t._run(): failed = True
self.failed = failed
print >> Env.log, "[Done] Running tests [result=%s]" % (failed and "Failed" or "Succeeded")
return not failed
def Cleanup(self):
# print >> sys.stderr, "cleanup_files: %s, cleanup_dirs: %s" % (self.prepared_files, self.prepared_dirs)
if not Env.no_cleanup and not self.failed:
print >> Env.log, "[Start] Cleanup services"
for s in self.services:
s._cleanup()
print >> Env.log, "[Done] Cleanup services"
else:
print >> Env.log, "[Start] Stopping services"
for s in self.services:
s._stop()
print >> Env.log, "[Done] Stopping services"
print >> Env.log, "[Start] Cleanup tests"
for t in self.tests:
t._cleanup()
if not Env.no_cleanup and not self.failed:
self.CleanupFile("log/access.log")
self.CleanupFile("log/error.log")
self.CleanupFile("conf/lighttpd.conf")
self.CleanupFile("conf/angel.conf")
print >> Env.log, "[Done] Cleanup tests"
## helpers for prepare/cleanup
def _preparefile(self, fname, content):
if self.prepared_files.has_key(fname):
raise BaseException("File '%s' already exists!" % fname)
else:
f = open(os.path.join(Env.dir, fname), "w")
f.write(content)
f.close()
self.prepared_files[fname] = 1
def _cleanupfile(self, fname):
if self.prepared_files.has_key(fname):
os.remove(os.path.join(Env.dir, fname))
return True
else:
return False
def _preparedir(self, dirname):
if self.prepared_dirs.has_key(dirname):
self.prepared_dirs[dirname] += 1
else:
os.mkdir(os.path.join(Env.dir, dirname))
self.prepared_dirs[dirname] = 1
def _cleanupdir(self, dirname):
self.prepared_dirs[dirname] -= 1
if 0 == self.prepared_dirs[dirname]:
os.rmdir(os.path.join(Env.dir, dirname))
def PrepareFile(self, fname, content):
path = filter(lambda x: x != '', fname.split('/'))
for i in range(1, len(path)):
self._preparedir(os.path.join(*path[0:i]))
self._preparefile(os.path.join(*path), content)
return os.path.join(Env.dir, *path)
def PrepareDir(self, dirname):
path = filter(lambda x: x != '', fname.split('/'))
for i in range(1, len(path)+1):
self._preparedir(os.path.join(*path[0:i]))
return os.path.join(Env.dir, *path)
def CleanupDir(self, dirname):
path = filter(lambda x: x != '', fname.split('/'))
for i in reversed(range(1, len(path)+1)):
self._cleanupdir(os.path.join(*path[0:i]))
def CleanupFile(self, fname):
path = filter(lambda x: x != '', fname.split('/'))
if not self._cleanupfile(os.path.join(*path)):
return False
for i in reversed(range(1, len(path))):
self._cleanupdir(os.path.join(*path[0:i]))
class Lighttpd(Service):
name = "lighttpd"
def Prepare(self):
self.portfree(Env.port)
self.fork(Env.angel, '-m', Env.plugindir, '-c', Env.angelconf)
self.waitconnect(Env.port)

83
tests/logfile.py Normal file
View File

@ -0,0 +1,83 @@
# -*- coding: utf-8 -*-
import time
__all__ = [ 'LogFile' ]
ATTRS = [ 'closed', 'encoding', 'errors', 'mode', 'name', 'newlines', 'softspace' ]
class LogFile(object):
def __init__(self, file, **clones):
self.file = file
self.clones = clones
self.newline = True
def __enter__(self, *args, **kwargs): return self.file.__enter__(*args, **kwargs)
def __exit__(self, *args, **kwargs): return self.file.__exit__(*args, **kwargs)
def __iter__(self, *args, **kwargs): return self.file.__iter__(*args, **kwargs)
def __repr__(self, *args, **kwargs): return self.file.__repr__(*args, **kwargs)
def __delattr__(self, name):
if name in ATTRS:
return delattr(self.file, name)
else:
return super(LogFile, self).__delattr__(name, value)
def __getattr__(self, name):
if name in ATTRS:
return getattr(self.file, name)
else:
return super(LogFile, self).__getattr__(name, value)
def __getattribute__(self, name):
if name in ATTRS:
return self.file.__getattribute__(name)
else:
return object.__getattribute__(self, name)
def __setattr__(self, name, value):
if name in ATTRS:
return setattr(self.file, name, value)
else:
return super(LogFile, self).__setattr__(name, value)
def close(self, *args, **kwargs): return self.file.close(*args, **kwargs)
def fileno(self, *args, **kwargs):
pass
def flush(self, *args, **kwargs): return self.file.flush(*args, **kwargs)
def isatty(self, *args, **kwargs): return False
def next(self, *args, **kwargs): return self.file.next(*args, **kwargs)
def read(self, *args, **kwargs): return self.file.read(*args, **kwargs)
def readinto(self, *args, **kwargs): return self.file.readinto(*args, **kwargs)
def readline(self, *args, **kwargs): return self.file.readline(*args, **kwargs)
def readlines(self, *args, **kwargs): return self.file.readlines(*args, **kwargs)
def seek(self, *args, **kwargs):
pass
def tell(self, *args, **kwargs): return self.file.tell(*args, **kwargs)
def truncate(self, *args, **kwargs):
pass
def __write(self, str):
self.file.write(str)
for (p, f) in self.clones.items():
f.write(p + str)
def _write(self, str):
if "" == str: return
if self.newline:
# "%f" needs python 2.6
# ts = time.strftime("%Y/%m/%d %H:%M:%S.%f %Z: ")
ts = time.strftime("%Y/%m/%d %H:%M:%S %Z")
self.file.write(ts + ": " + str)
for (p, f) in self.clones.items():
f.write(ts + " " + p + ": " + str)
else:
self.file.write(str)
for (p, f) in self.clones.items():
f.write(str)
self.newline = ('\n' == str[-1])
def write(self, str):
lines = str.split('\n')
for l in lines[:-1]:
self._write(l + '\n')
self._write(lines[-1])
def writelines(self, *args):
return self.write(''.join(args))
def xreadlines(self, *args, **kwargs): return self.file.xreadlines(*args, **kwargs)

117
tests/requests.py Normal file
View File

@ -0,0 +1,117 @@
# -*- coding: utf-8 -*-
import pycurl
import StringIO
import sys
from base import *
class CurlRequestException(Exception):
def __init__(self, value): self.value = value
def __str__(self): return repr(self.value)
class CurlRequest(TestBase):
URL = None
SCHEME = "http"
PORT = 0 # offset to Env.port
EXPECT_RESPONSE_BODY = None
EXPECT_RESPONSE_CODE = None
EXPECT_RESPONSE_HEADERS = []
def __init__(self):
super(CurlRequest, self).__init__()
self.resp_header_list = []
self.resp_headers = { }
self.resp_first_line = None
def _recv_header(self, header):
header = header.rstrip()
if None == self.resp_first_line:
self.resp_first_line = header
return
if header == "":
return
try:
(key, value) = header.split(":", 1)
except:
print >> sys.stderr, "Couldn't parse header '%s'" % header
raise
key = key.strip()
value = value.strip()
self.resp_header_list.append((key, value))
if self.resp_headers.has_key(key):
self.resp_headers[key] += ", " + value
else:
self.resp_headers[key] = value
def Run(self):
if None == self.URL:
raise BasicException("You have to set URL in your CurlRequest instance")
c = pycurl.Curl()
c.setopt(pycurl.URL, self.SCHEME + ("://127.0.0.1:%i" % (Env.port + self.PORT)) + self.URL)
c.setopt(pycurl.HTTPHEADER, ["Host: " + self.vhost])
b = StringIO.StringIO()
c.setopt(pycurl.WRITEFUNCTION, b.write)
c.setopt(pycurl.HEADERFUNCTION, self._recv_header)
self.curl = c
self.buffer = b
self.PrepareRequest()
c.perform()
try:
if not self._checkResponse():
raise CurlRequestException("Response check failed")
except:
if not Env.debugRequests:
self.dump()
raise
return True
def PrepareRequest(self):
pass
def dump(self):
c = self.curl
print >> sys.stdout, "Curl request: URL = %s://%s:%i%s" % (self.SCHEME, self.vhost, Env.port + self.PORT, self.URL)
print >> sys.stdout, "Curl response code: %i " % (c.getinfo(pycurl.RESPONSE_CODE))
print >> sys.stdout, "Curl response headers:"
for (k, v) in self.resp_header_list:
print >> sys.stdout, " %s: %s" % (k, v)
print >> sys.stdout, "Curl response body:"
print >> sys.stdout, self.buffer.getvalue()
def _checkResponse(self):
c = self.curl
if Env.debugRequests:
self.dump()
if not self.CheckResponse():
return False
if None != self.EXPECT_RESPONSE_CODE:
code = c.getinfo(pycurl.RESPONSE_CODE)
if code != self.EXPECT_RESPONSE_CODE:
raise CurlRequestException("Unexpected response code %i (wanted %i)" % (code, self.EXPECT_RESPONSE_CODE))
if None != self.EXPECT_RESPONSE_BODY:
body = self.buffer.getvalue()
if body != self.EXPECT_RESPONSE_BODY:
raise CurlRequestException("Unexpected response body")
for (k, v) in self.EXPECT_RESPONSE_HEADERS:
if not self.resp_headers.has_key(k):
raise CurlRequestException("Didn't get wanted response header '%s'" % (k))
v1 = self.resp_headers[k]
if v1 != v:
raise CurlRequestException("Unexpected response header '%s' = '%s' (wanted '%s')" % (k, v, v1))
return True
def CheckResponse(self):
return True

98
tests/runtests.py Executable file
View File

@ -0,0 +1,98 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
from logfile import LogFile
from base import Env, Tests
from optparse import OptionParser
from tempfile import mkdtemp
def find_port(port):
if port >= 1024 and port < (65536-8):
return port
from random import Random
r = Random(os.getpid())
return r.randint(1024, 65536-8)
class ArgumentError(Exception):
def __init__(self, value): self.value = value
def __str__(self): return repr(self.value)
parser = OptionParser()
parser.add_option("--angel", help = "Path to angel binary (required)")
parser.add_option("--worker", help = "Path to worker binary (required)")
parser.add_option("--plugindir", help = "Path to plugin directory (required)")
parser.add_option("-k", "--no-cleanup", help = "Keep temporary files, no cleanup", action = "store_true", default = False)
parser.add_option("-p", "--port", help = "Use [port,port+7] as tcp ports on 127.0.0.1 (default: 8088; use 0 for random port)", default = 8088, type = "int")
parser.add_option("-t", "--test", help = "Run specific test", action = "append", dest = "tests", default = [])
parser.add_option("-c", "--force-cleanup", help = "Keep no temporary files (overwrites -k)", action = "store_true", default = False)
parser.add_option("-s", "--strace", help = "Strace services", action = "store_true", default = False)
parser.add_option("--debug-requests", help = "Dump requests", action = "store_true", default = False)
(options, args) = parser.parse_args()
if not options.angel or not options.worker or not options.plugindir:
raise ArgumentError("Missing required arguments")
if options.force_cleanup: options.no_cleanup = False
Env.angel = os.path.abspath(options.angel)
Env.worker = os.path.abspath(options.worker)
Env.plugindir = os.path.abspath(options.plugindir)
Env.no_cleanup = options.no_cleanup
Env.force_cleanup = options.force_cleanup
Env.port = find_port(options.port)
Env.tests = options.tests
Env.sourcedir = os.path.abspath(os.path.dirname(__file__))
Env.luadir = os.path.join(os.path.dirname(Env.sourcedir), "doc")
Env.debugRequests = options.debug_requests
Env.strace = options.strace
Env.dir = mkdtemp(dir = os.getcwd())
Env.defaultwww = os.path.join(Env.dir, "www", "default")
Env.log = open(os.path.join(Env.dir, "tests.log"), "w")
sys.stderr = LogFile(sys.stderr, **{ "[stderr]": Env.log })
sys.stdout = LogFile(sys.stdout, **{ "[stdout]": Env.log })
Env.log = LogFile(Env.log)
failed = False
try:
# run tests
tests = Tests()
tests.LoadTests()
failed = True
try:
tests.Prepare()
except:
raise
else:
if tests.Run():
failed = False
finally:
tests.Cleanup()
if not Env.no_cleanup and not failed:
os.remove(os.path.join(Env.dir, "tests.log"))
finally:
try:
if Env.force_cleanup:
import shutil
shutil.rmtree(Env.dir)
elif not Env.no_cleanup and not failed:
os.rmdir(Env.dir)
except OSError:
print >> sys.stderr, "Couldn't delete temporary directory '%s', probably not empty (perhaps due to some errors)" % Env.dir
Env.log.close()
if failed:
sys.exit(1)
else:
sys.exit(0)

125
tests/service.py Normal file
View File

@ -0,0 +1,125 @@
# -*- coding: utf-8 -*-
import os
import atexit
import subprocess
import socket
import select
import signal
import base
__all__ = [ "Service", "ServiceException" ]
class ServiceException(Exception):
def __init__(self, value): self.value = value
def __str__(self): return repr(self.value)
def devnull():
try:
f = open("/dev/null", "r")
return f
except:
return None
straceargs = [ 'strace', '-tt', '-f', '-s', '4096', '-o' ]
def preexec():
os.setsid()
class Service(object):
name = None
def __init__(self):
self.proc = None
self.tests = None
self.failed = False
def fork(self, *args):
if None == self.name:
raise ServiceException("Service needs a name!")
log = self.tests.PrepareFile("log/service-%s.log" % self.name, "")
logfile = open(log, "w")
inp = devnull()
if base.Env.strace:
slog = self.tests.PrepareFile("log/strace-%s.log" % self.name, "")
args = straceargs + [ slog ] + list(args)
print >> base.Env.log, "Spawning '%s': %s" % (self.name, ' '.join(args))
proc = subprocess.Popen(args, stdin = inp, stdout = logfile, stderr = logfile, close_fds = True, preexec_fn = preexec)
if None != inp: inp.close()
logfile.close()
self.proc = proc
atexit.register(self.kill)
def kill(self):
proc = self.proc
if None == proc: return
self.proc = None
if None == proc.poll():
print >> base.Env.log, "Terminating service '%s'" % (self.name)
try:
os.killpg(proc.pid, signal.SIGINT)
proc.terminate()
except:
pass
print >> base.Env.log, "Waiting for service '%s'" % (self.name)
proc.wait()
def portfree(self, port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(("127.0.0.1", port))
except:
pass
else:
raise ServiceException("Cannot start service '%s', port 127.0.0.1:%i already in use" % (self.name, port))
finally:
s.close()
def waitconnect(self, port):
timeout = 5*10
while True:
if None != self.proc.poll():
raise ServiceException("Service %s died before we could establish a connection" % (self.name))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(("127.0.0.1", port))
except:
pass
else:
return True
finally:
s.close()
select.select([], [], [], 0.1)
timeout -= 1
if 0 > timeout:
raise ServiceException("Timeout: cannot establish a connection to service %s on port %i" % (self.name, port))
def _prepare(self):
self.failed = True
self.Prepare()
self.failed = False
def _cleanup(self):
self.kill()
if not base.Env.force_cleanup and self.failed:
return
self.tests.CleanupFile("log/service-%s.log" % self.name)
self.tests.CleanupFile("log/strace-%s.log" % self.name)
self.Cleanup()
def _stop(self):
self.kill()
self.Stop()
def Prepare(self):
raise BaseException("Not implemented yet")
def Cleanup(self):
pass
def Stop(self):
pass

21
tests/t-basic-gets.py Normal file
View File

@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
import sys
from base import *
from requests import *
TEST_TXT="""Hi!
0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
"""
class Test(CurlRequest):
URL = "/test.txt"
EXPECT_RESPONSE_BODY = TEST_TXT
EXPECT_RESPONSE_CODE = 200
def Prepare(self):
self.PrepareFile("www/default/test.txt", TEST_TXT)