Commit 72d88d15 authored by Jelmer Vernooij's avatar Jelmer Vernooij Committed by Gerald (Jerry) Carter

r21707: Finally merge my (long-living) perlselftest branch.

This changes the main selftest code to be in perl rather than in shell script.

The selftest script is now no longer a black box but a regular executable that takes
--help.

This adds the following features:

 * "make test TESTS=foo" will run only the tests that match the regex "foo"
 * ability to deal with expected failures. the suite will not warn about tests
   that fail and are known to fail, but will warn about other failing tests and
   tests that are succeeding tests but incorrectly marked as failing.
 * ability to print a summary with all failures at the end of the run

It also opens up the way to the following features, which I hope to implement later:
 * "environments", for example having a complete domains with DCs and domain members
 in a testenvironment
 * only set up smbd if necessary (not when running LOCAL tests, for example)
 * different mktestsetup scripts per target. except for the mktestsetup script, we can
   use the same infrastructure for samba 3 or windows.
(This used to be commit 38f86788)
parent d18afd6a
......@@ -157,7 +157,7 @@ YAPP=$self->{config}->{YAPP}
GCOV=$self->{config}->{GCOV}
DEFAULT_TEST_TARGET=$self->{config}->{DEFAULT_TEST_TARGET}
DEFAULT_TEST_OPTIONS=$self->{config}->{DEFAULT_TEST_OPTIONS}
__EOD__
);
......
AC_ARG_ENABLE(socket-wrapper,
[ --enable-socket-wrapper Turn on socket wrapper library (default=no)])
DEFAULT_TEST_TARGET=test-noswrap
DEFAULT_TEST_OPTIONS=
HAVE_SOCKET_WRAPPER=no
if eval "test x$developer = xyes"; then
......@@ -10,9 +10,9 @@ fi
if eval "test x$enable_socket_wrapper = xyes"; then
AC_DEFINE(SOCKET_WRAPPER,1,[Use socket wrapper library])
DEFAULT_TEST_TARGET=test-swrap
DEFAULT_TEST_OPTIONS=--socket-wrapper
HAVE_SOCKET_WRAPPER=yes
fi
AC_SUBST(DEFAULT_TEST_TARGET)
AC_SUBST(DEFAULT_TEST_OPTIONS)
AC_SUBST(HAVE_SOCKET_WRAPPER)
......@@ -282,49 +282,56 @@ realdistclean: distclean removebackup
-rm -f $(MANPAGES)
check:: test
test: $(DEFAULT_TEST_TARGET)
SELFTEST = builddir=$(builddir) srcdir=$(srcdir) \
$(srcdir)/script/tests/selftest.sh ${selftest_prefix}
SELFTEST = $(srcdir)/script/tests/selftest.pl --prefix=${selftest_prefix} --builddir=$(builddir) --srcdir=$(srcdir) --expected-failures=samba4-knownfail
test: all libraries
$(SELFTEST) $(DEFAULT_TEST_OPTIONS) $(TESTS) --immediate
testone: all libraries
$(SELFTEST) $(DEFAULT_TEST_OPTIONS) $(TESTS) --one
test-swrap: all libraries
$(SELFTEST) all SOCKET_WRAPPER
$(SELFTEST) --socket-wrapper --immediate $(TESTS)
test-noswrap: all libraries
$(SELFTEST) all
$(SELFTEST) --immediate $(TESTS)
quicktestone: all
$(SELFTEST) --quick --socket-wrapper --one $(TESTS)
quicktest: all
$(SELFTEST) quick SOCKET_WRAPPER
$(SELFTEST) --quick --socket-wrapper --immediate $(TESTS)
testenv: all libraries
$(SELFTEST) xterm SOCKET_WRAPPER
$(srcdir)/script/tests/testenv.pl
valgrindtest: valgrindtest-quick
valgrindtest-quick: all
SMBD_VALGRIND="xterm -n smbd -e valgrind -q --db-attach=yes --num-callers=30" \
VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
$(SELFTEST) quick SOCKET_WRAPPER
$(SELFTEST) --quick --immediate --socket-wrapper
valgrindtest-all: all libraries
SMBD_VALGRIND="xterm -n smbd -e valgrind -q --db-attach=yes --num-callers=30" \
VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
$(SELFTEST) all SOCKET_WRAPPER
$(SELFTEST) --immediate --socket-wrapper
valgrindtest-env: all libraries
SMBD_VALGRIND="xterm -n smbd -e valgrind -q --db-attach=yes --num-callers=30" \
VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
$(SELFTEST) xterm SOCKET_WRAPPER
$(srcdir)/script/tests/testenv.pl
gdbtest: gdbtest-quick
gdbtest-quick: all
SMBD_VALGRIND="xterm -n smbd -e gdb --args " \
$(SELFTEST) quick SOCKET_WRAPPER
$(SELFTEST) --immediate --quick --socket-wrapper
gdbtest-all: all libraries
SMBD_VALGRIND="xterm -n smbd -e gdb --args " \
$(SELFTEST) all SOCKET_WRAPPER
$(SELFTEST) --immediate --socket-wrapper
wintest: all
$(SELFTEST) win
......
LOCAL-REGISTRY-*
LOCAL-RESOLVE-async
LOCAL-ICONV-next_codepoint()
LOCAL-REGISTRY/(nt4|ldb|dir) # Not implemented yet
LOCAL-RESOLVE/async
LOCAL-ICONV/next_codepoint()
BASE-DELAYWRITE/finfo update on close
RAW-OPLOCK/OPLOCK
#!/usr/bin/perl
# Bootstrap Samba and run a number of tests against it.
# Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
# Published under the GNU GPL, v3 or later.
package Samba4;
use Exporter;
@ISA = qw(Exporter);
@EXPORT_OK = qw(slapd_start slapd_stop smbd_check_or_start provision);
use strict;
use FindBin qw($RealBin);
use POSIX;
sub slapd_start($$)
{
my ($conf, $uri) = @_;
if (defined($ENV{FEDORA_DS_PREFIX})) {
system("$ENV{FEDORA_DS_PREFIX}/lib/fedora-ds/ds_newinst.pl $ENV{FEDORA_DS_INF}") or die("Unable to provision fedora ds ldapd");
} else {
my $oldpath = $ENV{PATH};
$ENV{PATH} = "/usr/local/sbin:/usr/sbin:/sbin:$ENV{PATH}";
# running slapd in the background means it stays in the same process group, so it can be
# killed by timelimit
system("slapd -d0 -f $conf -h $uri &");
$ENV{PATH} = $oldpath;
}
return $? >> 8;
}
sub slapd_stop()
{
if (defined($ENV{FEDORA_DS_PREFIX})) {
system("$ENV{LDAPDIR}/slapd-samba4/stop-slapd");
} else {
open(IN, "<$ENV{PIDDIR}/slapd.pid") or
die("unable to open slapd pid file");
kill 9, <IN>;
close(IN);
}
}
sub smbd_check_or_start($$$$$$)
{
my ($bindir, $test_fifo, $test_log, $socket_wrapper_dir, $max_time, $conffile) = @_;
return 0 if ( -p $test_fifo );
warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless
defined($socket_wrapper_dir) or $< == 0;
if (defined($socket_wrapper_dir)) {
if ( -d $socket_wrapper_dir ) {
unlink <$socket_wrapper_dir/*>;
} else {
mkdir($socket_wrapper_dir);
}
}
unlink($test_fifo);
POSIX::mkfifo($test_fifo, 0700);
unlink($test_log);
my $valgrind = "";
if (defined($ENV{SMBD_VALGRIND})) {
$valgrind = $ENV{SMBD_VALGRIND};
}
print "STARTING SMBD...";
my $pid = fork();
if ($pid == 0) {
open STDIN, $test_fifo;
open STDOUT, ">$test_log";
open STDERR, '>&STDOUT';
my $optarg = "";
if (defined($max_time)) {
$optarg = "--maximum-runtime=$max_time ";
}
my $ret = system("$valgrind $bindir/smbd $optarg -s $conffile -M single -i --leak-report-full");
if ($? == -1) {
print "Unable to start smbd: $ret: $!\n";
exit 1;
}
unlink($test_fifo);
unlink(<$socket_wrapper_dir/*>) if (defined($socket_wrapper_dir) and -d $socket_wrapper_dir);
my $exit = $? >> 8;
if ( $ret == 0 ) {
print "smbd exits with status $exit\n";
} elsif ( $ret & 127 ) {
print "smbd got signal ".($ret & 127)." and exits with $exit!\n";
} else {
$ret = $? >> 8;
print "smbd failed with status $exit!\n";
}
exit $exit;
}
print "DONE\n";
return $pid;
}
sub wait_for_start()
{
# give time for nbt server to register its names
print "delaying for nbt name registration\n";
sleep(4);
# This will return quickly when things are up, but be slow if we
# need to wait for (eg) SSL init
system("bin/nmblookup $ENV{CONFIGURATION} $ENV{SERVER}");
system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{SERVER}");
system("bin/nmblookup $ENV{CONFIGURATION} $ENV{SERVER}");
system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{NETBIOSNAME}");
system("bin/nmblookup $ENV{CONFIGURATION} $ENV{NETBIOSNAME}");
system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{NETBIOSNAME}");
}
sub provision($)
{
my ($prefix) = @_;
my %ret = ();
print "PROVISIONING...";
open(IN, "$RealBin/mktestsetup.sh $prefix|") or die("Unable to setup");
while (<IN>) {
die ("Error parsing `$_'") unless (/^([A-Z0-9a-z_]+)=(.*)$/);
$ret{$1} = $2;
}
close(IN);
return \%ret;
}
sub provision_ldap($$)
{
my ($bindir, $setupdir) = @_;
system("$bindir/smbscript $setupdir/provision $ENV{PROVISION_OPTIONS} \"$ENV{PROVISION_ACI}\" --ldap-backend=$ENV{LDAPI}") or
die("LDAP PROVISIONING failed: $bindir/smbscript $setupdir/provision $ENV{PROVISION_OPTIONS} \"$ENV{PROVISION_ACI}\" --ldap-backend=$ENV{LDAPI}");
}
1;
#!/usr/bin/perl
# Bootstrap Samba and run a number of tests against it.
# Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
# Published under the GNU GPL, v3 or later.
package SocketWrapper;
use Exporter;
@ISA = qw(Exporter);
@EXPORT_OK = qw(setup_dir setup_pcap set_default_iface);
use strict;
use FindBin qw($RealBin);
sub setup_dir($)
{
my ($dir) = @_;
$ENV{SOCKET_WRAPPER_DIR} = $dir;
return $dir;
}
sub setup_pcap($)
{
my ($pcap_file) = @_;
}
sub set_default_iface($)
{
my ($i) = @_;
$ENV{SOCKET_WRAPPER_DEFAULT_IFACE} = $i;
}
1;
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
......@@ -2,6 +2,107 @@
# Bootstrap Samba and run a number of tests against it.
# Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
# Published under the GNU GPL, v3 or later.
=pod
=head1 NAME
selftest - Samba test runner
=head1 SYNOPSIS
selftest --help
selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
=head1 DESCRIPTION
A simple test runner. TESTS is a regular expression with tests to run.
=head1 OPTIONS
=over 4
=item I<--help>
Show list of available options.
=item I<--srcdir=DIR>
Source directory.
=item I<--builddir=DIR>
Build directory.
=item I<--prefix=DIR>
Change directory to run tests in. Default is 'st'.
=item I<--immediate>
Show errors as soon as they happen rather than at the end of the test run.
=item I<--target samba4|samba3|win>
Specify test target against which to run. Default is 'samba4'.
=item I<--quick>
Run only a limited number of tests. Intended to run in about 30 seconds on
moderately recent systems.
=item I<--socket-wrapper>
Use socket wrapper library for communication with server. Only works
when the server is running locally.
Will prevent TCP and UDP ports being opened on the local host but
(transparently) redirects these calls to use unix domain sockets.
=item I<--expected-failures>
Specify a file containing a list of tests that are expected to fail. Failures for
these tests will be counted as successes, successes will be counted as failures.
The format for the file is, one entry per line:
TESTSUITE-NAME/TEST-NAME
=item I<--one>
Abort as soon as one test fails.
=back
=head1 ENVIRONMENT
=over 4
=item I<SMBD_VALGRIND>
=item I<TORTURE_MAXTIME>
=item I<VALGRIND>
=item I<TEST_LDAP>
=item I<TLS_ENABLED>
=item I<srcdir>
=back
=head1 LICENSE
selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
=head1 AUTHOR
Jelmer Vernooij
=cut
use strict;
use warnings;
......@@ -10,86 +111,163 @@ use File::Spec;
use Getopt::Long;
use POSIX;
use Cwd;
use lib "$RealBin";
use Samba4;
use SocketWrapper;
sub slapd_start($$) {
my ($conf, $uri) = @_;
my $oldpath = $ENV{PATH};
$ENV{PATH} = "/usr/local/sbin:/usr/sbin:/sbin:$ENV{PATH}";
# running slapd in the background means it stays in the same process group, so it can be
# killed by timelimit
system("slapd -d0 -f $conf -h $uri &");
$ENV{PATH} = $oldpath;
return $? >> 8;
}
my $opt_help = 0;
my $opt_target = "samba4";
my $opt_quick = 0;
my $opt_socket_wrapper = 0;
my $opt_socket_wrapper_pcap = undef;
my $opt_one = 0;
my $opt_immediate = 0;
my $opt_expected_failures = undef;
my $opt_verbose = 0;
sub smbd_check_or_start($$$$$$)
my $srcdir = ".";
my $builddir = ".";
my $prefix = "st";
my $suitesfailed = [];
my $start = time();
my @expected_failures = ();
my $statistics = {
SUITES_FAIL => 0,
SUITES_OK => 0,
TESTS_UNEXPECTED_OK => 0,
TESTS_EXPECTED_OK => 0,
TESTS_UNEXPECTED_FAIL => 0,
TESTS_EXPECTED_FAIL => 0,
TESTS_ERROR => 0
};
sub expecting_failure($)
{
my ($bindir, $test_fifo, $test_log, $socket_wrapper_dir, $max_time, $conffile) = @_;
return 0 if ( -p $test_fifo );
my $fullname = shift;
if (defined($socket_wrapper_dir)) {
if ( -d $socket_wrapper_dir ) {
unlink <$socket_wrapper_dir/*>;
} else {
mkdir($socket_wrapper_dir);
}
foreach (@expected_failures) {
return 1 if $fullname =~ /^$_$/;
}
unlink($test_fifo);
system("mkfifo $test_fifo");
unlink($test_log);
my $valgrind = "";
if (defined($ENV{SMBD_VALGRIND})) {
$valgrind = $ENV{SMBD_VALGRIND};
}
print "STARTING SMBD...";
my $pid = fork();
if ($pid == 0) {
my $ret = system("$valgrind $bindir/smbd --maximum-runtime=$max_time -s $conffile -M single -i --leak-report-full < $test_fifo > $test_log");
open LOG, ">>$test_log";
if ($? == -1) {
print LOG "Unable to start smbd: $ret: $!\n";
print "Unable to start smbd: $ret: $!\n";
exit 1;
}
unlink($test_fifo);
unlink(<$socket_wrapper_dir/*>) if (defined($socket_wrapper_dir) and -d $socket_wrapper_dir);
my $exit = $? >> 8;
if ( $ret == 0 ) {
print "smbd exits with status $exit\n";
print LOG "smbd exits with status $exit\n";
} elsif ( $ret & 127 ) {
print "smbd got signal ".($ret & 127)." and exits with $exit!\n";
print LOG "smbd got signal".($ret & 127). " and exits with $exit!\n";
} else {
$ret = $? >> 8;
print "smbd failed with status $exit!\n";
print LOG "smbd failed with status $exit!\n";
return 0;
}
sub run_test_buildfarm($$$$)
{
my ($name, $cmd, $i, $suitestotal) = @_;
print "--==--==--==--==--==--==--==--==--==--==--\n";
print "Running test $name (level 0 stdout)\n";
print "--==--==--==--==--==--==--==--==--==--==--\n";
system("date");
my $expected_ret = 1;
my $open_tests = {};
open(RESULT, "$cmd|");
while (<RESULT>) {
print;
if (/^test: (.+)\n/) {
$open_tests->{$1} = 1;
} elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
my $result = $1;
if ($1 eq "success") {
delete $open_tests->{$2};
if (expecting_failure("$name/$2")) {
$statistics->{TESTS_UNEXPECTED_OK}++;
} else {
$statistics->{TESTS_EXPECTED_OK}++;
}
} elsif ($1 eq "failure") {
delete $open_tests->{$2};
if (expecting_failure("$name/$2")) {
$statistics->{TESTS_EXPECTED_FAIL}++;
$expected_ret = 0;
} else {
$statistics->{TESTS_UNEXPECTED_FAIL}++;
}
} elsif ($1 eq "skip") {
delete $open_tests->{$2};
} elsif ($1 eq "error") {
$statistics->{TESTS_ERROR}++;
delete $open_tests->{$2};
}
}
close(LOG);
exit $exit;
}
print "DONE\n";
print "COMMAND: $cmd\n";
foreach (keys %$open_tests) {
print "$_ was started but never finished!\n";
$statistics->{TESTS_ERROR}++;
}
my $ret = close(RESULT);
return $pid;
print "==========================================\n";
if ($ret == $expected_ret) {
print "TEST PASSED: $name\n";
} else {
print "TEST FAILED: $name (status $ret)\n";
}
print "==========================================\n";
}
sub teststatus($$) {
my ($name, $failed) = @_;
print "TEST STATUS: $failed failures\n";
if ($failed > 0) {
print <<EOF
************************
*** TESTSUITE FAILED ***
************************
EOF
;
my $test_output = {};
sub run_test_plain($$$$)
{
my ($name, $cmd, $i, $totalsuites) = @_;
my $err = "";
if ($#$suitesfailed+1 > 0) { $err = ", ".($#$suitesfailed+1)." errors"; }
printf "[$i/$totalsuites in " . (time() - $start)."s$err] $name\n";
open(RESULT, "$cmd 2>&1|");
my $expected_ret = 1;
my $open_tests = {};
$test_output->{$name} = "";
while (<RESULT>) {
$test_output->{$name}.=$_;
print if ($opt_verbose);
if (/^test: (.+)\n/) {
$open_tests->{$1} = 1;
} elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
my $result = $1;
if ($1 eq "success") {
delete $open_tests->{$2};
if (expecting_failure("$name/$2")) {
$statistics->{TESTS_UNEXPECTED_OK}++;
} else {
$statistics->{TESTS_EXPECTED_OK}++;
}
} elsif ($1 eq "failure") {
delete $open_tests->{$2};
if (expecting_failure("$name/$2")) {
$statistics->{TESTS_EXPECTED_FAIL}++;
$expected_ret = 0;
} else {
$statistics->{TESTS_UNEXPECTED_FAIL}++;
}
} elsif ($1 eq "skip") {
delete $open_tests->{$2};
} elsif ($1 eq "error") {
$statistics->{TESTS_ERROR}++;
delete $open_tests->{$2};
}
}
}
$test_output->{$name}.="COMMAND: $cmd\n";
foreach (keys %$open_tests) {
$test_output->{$name}.="$_ was started but never finished!\n";
$statistics->{TESTS_ERROR}++;
}
my $ret = close(RESULT);
if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
print "$test_output->{$name}\n";
}
if ($ret != $expected_ret) {
push(@$suitesfailed, $name);
$statistics->{SUITES_FAIL}++;
exit(1) if ($opt_one);
} else {
$statistics->{SUITES_OK}++;
}
exit $failed;
}
sub ShowHelp()
......@@ -97,51 +275,60 @@ sub ShowHelp()
print "Samba test runner
Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
Usage: $Script PREFIX
Usage: $Script [OPTIONS] PREFIX
Generic options:
--help this help page
Paths:
--prefix=DIR prefix to run tests in [st]
--srcdir=DIR source directory [.]
--builddir=DIR output directory [.]
Target Specific:
--target=samba4|samba3|win Samba version to target
--socket-wrapper-pcap=FILE save traffic to pcap file
--socket-wrapper enable socket wrapper
--expected-failures=FILE specify list of tests that is guaranteed to fail
Behaviour:
--quick run quick overall test
--one abort when the first test fails
--immediate print test output for failed tests during run
--verbose be verbose
";
exit(0);
}
my $opt_help = 0;
my $opt_target = "samba4";
my $opt_quick = 0;
my $opt_socket_wrapper = 0;
my $opt_one = 0;
my $result = GetOptions (
'help|h|?' => \$opt_help,
'target' => \$opt_target,
'target=s' => \$opt_target,
'prefix=s' => \$prefix,
'socket-wrapper' => \$opt_socket_wrapper,
'socket-wrapper-pcap=s' => \$opt_socket_wrapper_pcap,
'quick' => \$opt_quick,
'one' => \$opt_one
'one' => \$opt_one,
'immediate' => \$opt_immediate,
'expected-failures=s' => \$opt_expected_failures,
'srcdir=s' => \$srcdir,
'builddir=s' => \$builddir,
'verbose' => \$opt_verbose
);
if (not $result) {
exit(1);
}
exit(1) if (not $result);
ShowHelp() if ($opt_help);
ShowHelp() if ($#ARGV < 0);
my $prefix = shift;
my $tests = shift;
my $torture_maxtime = $ENV{TORTURE_MAXTIME};
unless (defined($torture_maxtime)) {
$torture_maxtime = 1200;
}
# disable rpc validation when using valgrind - its way too slow
my $valgrind = $ENV{VALGRIND};
my $validate = undef;
unless (defined($valgrind)) {
$validate = "validate";
# quick hack to disable rpc validation when using valgrind - its way too slow
unless (defined($ENV{VALGRIND})) {
$ENV{VALIDATE} = "validate";
}
my $old_pwd = "$RealBin/../..";
......@@ -150,10 +337,6 @@ my $ldap = (defined($ENV{TEST_LDAP}) and ($ENV{TEST_LDAP} eq "yes"))?1:0;
$prefix =~ s+//+/+;
$ENV{PREFIX} = $prefix;
my $srcdir = "$RealBin/../..";
if (defined($ENV{srcdir})) {
$srcdir = $ENV{srcdir};
}
$ENV{SRCDIR} = $srcdir;
my $bindir = "$srcdir/bin";
......@@ -161,6 +344,8 @@ my $setupdir = "$srcdir/setup";
my $testsdir = "$srcdir/script/tests";
my $tls_enabled = not $opt_quick;
my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
$ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
$ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
......@@ -173,14 +358,12 @@ if (defined($ENV{LD_LIBRARY_PATH})) {
$ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
$ENV{PATH} = "$old_pwd