subunithelper.py 24.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
# Python module for parsing and generating the Subunit protocol
# (Samba-specific)
# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

18
from __future__ import print_function
19 20
__all__ = ['parse_results']

21
import datetime
22
import re
23
import sys
24
import os
25 26
from samba import subunit
from samba.subunit.run import TestProtocolClient
27
from samba.subunit import iso8601
28
import unittest
29

30 31 32 33 34
VALID_RESULTS = set(['success', 'successful', 'failure', 'fail', 'skip',
                     'knownfail', 'error', 'xfail', 'skip-testsuite',
                     'testsuite-failure', 'testsuite-xfail',
                     'testsuite-success', 'testsuite-error',
                     'uxsuccess', 'testsuite-uxsuccess'])
35

36
class TestsuiteEnabledTestResult(unittest.TestResult):
37 38 39 40 41

    def start_testsuite(self, name):
        raise NotImplementedError(self.start_testsuite)


42
def parse_results(msg_ops, statistics, fh):
43
    exitcode = 0
44
    open_tests = {}
45 46 47

    while fh:
        l = fh.readline()
48 49
        if l == "":
            break
50 51
        parts = l.split(None, 1)
        if not len(parts) == 2 or not l.startswith(parts[0]):
52
            msg_ops.output_msg(l)
53 54 55 56 57
            continue
        command = parts[0].rstrip(":")
        arg = parts[1]
        if command in ("test", "testing"):
            msg_ops.control_msg(l)
58 59 60
            name = arg.rstrip()
            test = subunit.RemotedTestCase(name)
            if name in open_tests:
61
                msg_ops.addError(open_tests.pop(name), subunit.RemoteError(u"Test already running"))
62 63
            msg_ops.startTest(test)
            open_tests[name] = test
64
        elif command == "time":
65
            msg_ops.control_msg(l)
66
            try:
67
                dt = iso8601.parse_date(arg.rstrip("\n"))
68
            except TypeError as e:
69
                print("Unable to parse time line: %s" % arg.rstrip("\n"))
70 71
            else:
                msg_ops.time(dt)
72
        elif command in VALID_RESULTS:
73
            msg_ops.control_msg(l)
74 75 76
            result = command
            grp = re.match("(.*?)( \[)?([ \t]*)( multipart)?\n", arg)
            (testname, hasreason) = (grp.group(1), grp.group(2))
77 78 79 80 81 82
            if hasreason:
                reason = ""
                # reason may be specified in next lines
                terminated = False
                while fh:
                    l = fh.readline()
83 84
                    if l == "":
                        break
85
                    msg_ops.control_msg(l)
86 87
                    if l[-2:] == "]\n":
                        reason += l[:-2]
88 89 90 91
                        terminated = True
                        break
                    else:
                        reason += l
92

93 94
                remote_error = subunit.RemoteError(reason.decode("utf-8"))

95 96
                if not terminated:
                    statistics['TESTS_ERROR']+=1
97
                    msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"reason (%s) interrupted" % result))
98
                    return 1
99 100
            else:
                reason = None
101
                remote_error = subunit.RemoteError(u"No reason specified")
102
            if result in ("success", "successful"):
103
                try:
104 105
                    test = open_tests.pop(testname)
                except KeyError:
106
                    statistics['TESTS_ERROR']+=1
107
                    exitcode = 1
108
                    msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
109 110
                else:
                    statistics['TESTS_EXPECTED_OK']+=1
111
                    msg_ops.addSuccess(test)
112
            elif result in ("xfail", "knownfail"):
113
                try:
114 115
                    test = open_tests.pop(testname)
                except KeyError:
116
                    statistics['TESTS_ERROR']+=1
117
                    exitcode = 1
118
                    msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
119 120
                else:
                    statistics['TESTS_EXPECTED_FAIL']+=1
121
                    msg_ops.addExpectedFailure(test, remote_error)
122 123 124 125 126 127 128 129 130
            elif result in ("uxsuccess", ):
                try:
                    test = open_tests.pop(testname)
                except KeyError:
                    statistics['TESTS_ERROR']+=1
                    exitcode = 1
                    msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
                else:
                    statistics['TESTS_UNEXPECTED_OK']+=1
131
                    msg_ops.addUnexpectedSuccess(test)
132
                    exitcode = 1
133
            elif result in ("failure", "fail"):
134
                try:
135 136
                    test = open_tests.pop(testname)
                except KeyError:
137
                    statistics['TESTS_ERROR']+=1
138
                    exitcode = 1
139
                    msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
140 141
                else:
                    statistics['TESTS_UNEXPECTED_FAIL']+=1
142
                    exitcode = 1
143
                    msg_ops.addFailure(test, remote_error)
144 145 146
            elif result == "skip":
                statistics['TESTS_SKIP']+=1
                # Allow tests to be skipped without prior announcement of test
147 148 149 150 151
                try:
                    test = open_tests.pop(testname)
                except KeyError:
                    test = subunit.RemotedTestCase(testname)
                msg_ops.addSkip(test, reason)
152 153
            elif result == "error":
                statistics['TESTS_ERROR']+=1
154
                exitcode = 1
155
                try:
156 157 158
                    test = open_tests.pop(testname)
                except KeyError:
                    test = subunit.RemotedTestCase(testname)
159
                msg_ops.addError(test, remote_error)
160 161 162 163 164 165
            elif result == "skip-testsuite":
                msg_ops.skip_testsuite(testname)
            elif result == "testsuite-success":
                msg_ops.end_testsuite(testname, "success", reason)
            elif result == "testsuite-failure":
                msg_ops.end_testsuite(testname, "failure", reason)
166
                exitcode = 1
167 168
            elif result == "testsuite-xfail":
                msg_ops.end_testsuite(testname, "xfail", reason)
169 170 171
            elif result == "testsuite-uxsuccess":
                msg_ops.end_testsuite(testname, "uxsuccess", reason)
                exitcode = 1
172 173
            elif result == "testsuite-error":
                msg_ops.end_testsuite(testname, "error", reason)
174
                exitcode = 1
175 176 177
            else:
                raise AssertionError("Recognized but unhandled result %r" %
                    result)
178 179 180 181
        elif command == "testsuite":
            msg_ops.start_testsuite(arg.strip())
        elif command == "progress":
            arg = arg.strip()
182 183 184 185 186 187 188 189
            if arg == "pop":
                msg_ops.progress(None, subunit.PROGRESS_POP)
            elif arg == "push":
                msg_ops.progress(None, subunit.PROGRESS_PUSH)
            elif arg[0] in '+-':
                msg_ops.progress(int(arg), subunit.PROGRESS_CUR)
            else:
                msg_ops.progress(int(arg), subunit.PROGRESS_SET)
190 191 192 193
        else:
            msg_ops.output_msg(l)

    while open_tests:
194
        test = subunit.RemotedTestCase(open_tests.popitem()[1])
195
        msg_ops.addError(test, subunit.RemoteError(u"was started but never finished!"))
196
        statistics['TESTS_ERROR']+=1
197
        exitcode = 1
198

199
    return exitcode
200 201


202 203 204
class SubunitOps(TestProtocolClient,TestsuiteEnabledTestResult):

    def progress(self, count, whence):
205 206 207 208 209 210 211 212
        if whence == subunit.PROGRESS_POP:
            self._stream.write("progress: pop\n")
        elif whence == subunit.PROGRESS_PUSH:
            self._stream.write("progress: push\n")
        elif whence == subunit.PROGRESS_SET:
            self._stream.write("progress: %d\n" % count)
        elif whence == subunit.PROGRESS_CUR:
            raise NotImplementedError
213

214 215
    # The following are Samba extensions:
    def start_testsuite(self, name):
216
        self._stream.write("testsuite: %s\n" % name)
217

218 219
    def skip_testsuite(self, name, reason=None):
        if reason:
220
            self._stream.write("skip-testsuite: %s [\n%s\n]\n" % (name, reason))
221
        else:
222
            self._stream.write("skip-testsuite: %s\n" % name)
223

224 225
    def end_testsuite(self, name, result, reason=None):
        if reason:
226
            self._stream.write("testsuite-%s: %s [\n%s\n]\n" % (result, name, reason))
227
        else:
228
            self._stream.write("testsuite-%s: %s\n" % (result, name))
229

230 231 232
    def output_msg(self, msg):
        self._stream.write(msg)

233

234
def read_test_regexes(*names):
235
    ret = {}
236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
    files = []
    for name in names:
        # if we are given a directory, we read all the files it contains
        # (except the ones that end with "~").
        if os.path.isdir(name):
            files.extend([os.path.join(name, x)
                          for x in os.listdir(name)
                          if x[-1] != '~'])
        else:
            files.append(name)

    for filename in files:
        f = open(filename, 'r')
        try:
            for l in f:
                l = l.strip()
                if l == "" or l[0] == "#":
                    continue
                if "#" in l:
                    (regex, reason) = l.split("#", 1)
                    ret[regex.strip()] = reason.strip()
                else:
                    ret[l] = None
        finally:
            f.close()
261
    return ret
262 263 264


def find_in_list(regexes, fullname):
265
    for regex, reason in regexes.items():
266 267 268 269 270 271 272
        if re.match(regex, fullname):
            if reason is None:
                return ""
            return reason
    return None


273 274 275 276 277 278 279
class ImmediateFail(Exception):
    """Raised to abort immediately."""

    def __init__(self):
        super(ImmediateFail, self).__init__("test failed and fail_immediately set")


280
class FilterOps(unittest.TestResult):
281 282 283 284

    def control_msg(self, msg):
        pass # We regenerate control messages, so ignore this

285 286
    def time(self, time):
        self._ops.time(time)
287

288 289 290
    def progress(self, delta, whence):
        self._ops.progress(delta, whence)

291 292 293 294 295 296
    def output_msg(self, msg):
        if self.output is None:
            sys.stdout.write(msg)
        else:
            self.output+=msg

297
    def startTest(self, test):
298
        self.seen_output = True
299
        test = self._add_prefix(test)
300 301 302
        if self.strip_ok_output:
           self.output = ""

303
        self._ops.startTest(test)
304

305
    def _add_prefix(self, test):
306
        return subunit.RemotedTestCase(self.prefix + test.id() + self.suffix)
307

308
    def addError(self, test, err=None):
309
        test = self._add_prefix(test)
310 311
        self.error_added+=1
        self.total_error+=1
312
        self._ops.addError(test, err)
313
        self.output = None
314
        if self.fail_immediately:
315
            raise ImmediateFail()
316

317
    def addSkip(self, test, reason=None):
318
        self.seen_output = True
319
        test = self._add_prefix(test)
320
        self._ops.addSkip(test, reason)
321
        self.output = None
322

323
    def addExpectedFailure(self, test, err=None):
324
        test = self._add_prefix(test)
325
        self._ops.addExpectedFailure(test, err)
326
        self.output = None
327

328
    def addUnexpectedSuccess(self, test):
329
        test = self._add_prefix(test)
330 331
        self.uxsuccess_added+=1
        self.total_uxsuccess+=1
332
        self._ops.addUnexpectedSuccess(test)
333 334
        if self.output:
            self._ops.output_msg(self.output)
335
        self.output = None
336 337
        if self.fail_immediately:
            raise ImmediateFail()
338

339
    def addFailure(self, test, err=None):
340
        test = self._add_prefix(test)
341
        xfail_reason = find_in_list(self.expected_failures, test.id())
342 343
        if xfail_reason is None:
            xfail_reason = find_in_list(self.flapping, test.id())
344
        if xfail_reason is not None:
345 346
            self.xfail_added+=1
            self.total_xfail+=1
347
            self._ops.addExpectedFailure(test, err)
348
        else:
349 350
            self.fail_added+=1
            self.total_fail+=1
351
            self._ops.addFailure(test, err)
352 353
            if self.output:
                self._ops.output_msg(self.output)
354 355
            if self.fail_immediately:
                raise ImmediateFail()
356 357
        self.output = None

358
    def addSuccess(self, test):
359
        test = self._add_prefix(test)
360 361 362 363
        xfail_reason = find_in_list(self.expected_failures, test.id())
        if xfail_reason is not None:
            self.uxsuccess_added += 1
            self.total_uxsuccess += 1
364
            self._ops.addUnexpectedSuccess(test)
365 366 367 368 369
            if self.output:
                self._ops.output_msg(self.output)
            if self.fail_immediately:
                raise ImmediateFail()
        else:
370
            self._ops.addSuccess(test)
371
        self.output = None
372 373 374 375 376 377 378 379 380

    def skip_testsuite(self, name, reason=None):
        self._ops.skip_testsuite(name, reason)

    def start_testsuite(self, name):
        self._ops.start_testsuite(name)
        self.error_added = 0
        self.fail_added = 0
        self.xfail_added = 0
381
        self.uxsuccess_added = 0
382 383 384 385 386 387

    def end_testsuite(self, name, result, reason=None):
        xfail = False

        if self.xfail_added > 0:
            xfail = True
388
        if self.fail_added > 0 or self.error_added > 0 or self.uxsuccess_added > 0:
389 390 391 392 393
            xfail = False

        if xfail and result in ("fail", "failure"):
            result = "xfail"

394 395 396 397 398 399
        if self.uxsuccess_added > 0 and result != "uxsuccess":
            result = "uxsuccess"
            if reason is None:
                reason = "Subunit/Filter Reason"
            reason += "\n uxsuccess[%d]" % self.uxsuccess_added

400 401 402 403 404 405 406 407 408 409 410 411 412
        if self.fail_added > 0 and result != "failure":
            result = "failure"
            if reason is None:
                reason = "Subunit/Filter Reason"
            reason += "\n failures[%d]" % self.fail_added

        if self.error_added > 0 and result != "error":
            result = "error"
            if reason is None:
                reason = "Subunit/Filter Reason"
            reason += "\n errors[%d]" % self.error_added

        self._ops.end_testsuite(name, result, reason)
413 414 415 416 417 418
        if result not in ("success", "xfail"):
            if self.output:
                self._ops.output_msg(self.output)
            if self.fail_immediately:
                raise ImmediateFail()
        self.output = None
419

420
    def __init__(self, out, prefix=None, suffix=None, expected_failures=None,
421 422
                 strip_ok_output=False, fail_immediately=False,
                 flapping=None):
423
        self._ops = out
424
        self.seen_output = False
425 426
        self.output = None
        self.prefix = prefix
427
        self.suffix = suffix
428 429 430 431
        if expected_failures is not None:
            self.expected_failures = expected_failures
        else:
            self.expected_failures = {}
432 433 434 435
        if flapping is not None:
            self.flapping = flapping
        else:
            self.flapping = {}
436 437
        self.strip_ok_output = strip_ok_output
        self.xfail_added = 0
438
        self.fail_added = 0
439
        self.uxsuccess_added = 0
440 441 442
        self.total_xfail = 0
        self.total_error = 0
        self.total_fail = 0
443
        self.total_uxsuccess = 0
444
        self.error_added = 0
445
        self.fail_immediately = fail_immediately
446 447


448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
class PerfFilterOps(unittest.TestResult):

    def progress(self, delta, whence):
        pass

    def output_msg(self, msg):
        pass

    def control_msg(self, msg):
        pass

    def skip_testsuite(self, name, reason=None):
        self._ops.skip_testsuite(name, reason)

    def start_testsuite(self, name):
        self.suite_has_time = False

    def end_testsuite(self, name, result, reason=None):
        pass

    def _add_prefix(self, test):
        return subunit.RemotedTestCase(self.prefix + test.id() + self.suffix)

    def time(self, time):
        self.latest_time = time
        #self._ops.output_msg("found time %s\n" % time)
        self.suite_has_time = True

    def get_time(self):
        if self.suite_has_time:
            return self.latest_time
        return datetime.datetime.utcnow()

    def startTest(self, test):
        self.seen_output = True
        test = self._add_prefix(test)
        self.starts[test.id()] = self.get_time()

    def addSuccess(self, test):
        test = self._add_prefix(test)
        tid = test.id()
        if tid not in self.starts:
            self._ops.addError(test, "%s succeeded without ever starting!" % tid)
        delta = self.get_time() - self.starts[tid]
        self._ops.output_msg("elapsed-time: %s: %f\n" % (tid, delta.total_seconds()))

    def addFailure(self, test, err=''):
        tid = test.id()
        delta = self.get_time() - self.starts[tid]
        self._ops.output_msg("failure: %s failed after %f seconds (%s)\n" %
                             (tid, delta.total_seconds(), err))

    def addError(self, test, err=''):
        tid = test.id()
        delta = self.get_time() - self.starts[tid]
        self._ops.output_msg("error: %s failed after %f seconds (%s)\n" %
                             (tid, delta.total_seconds(), err))

    def __init__(self, out, prefix='', suffix=''):
        self._ops = out
        self.prefix = prefix or ''
        self.suffix = suffix or ''
        self.starts = {}
        self.seen_output = False
        self.suite_has_time = False


515 516
class PlainFormatter(TestsuiteEnabledTestResult):

517
    def __init__(self, verbose, immediate, statistics,
518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
            totaltests=None):
        super(PlainFormatter, self).__init__()
        self.verbose = verbose
        self.immediate = immediate
        self.statistics = statistics
        self.start_time = None
        self.test_output = {}
        self.suitesfailed = []
        self.suites_ok = 0
        self.skips = {}
        self.index = 0
        self.name = None
        self._progress_level = 0
        self.totalsuites = totaltests
        self.last_time = None

534
    @staticmethod
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
    def _format_time(delta):
        minutes, seconds = divmod(delta.seconds, 60)
        hours, minutes = divmod(minutes, 60)
        ret = ""
        if hours:
            ret += "%dh" % hours
        if minutes:
            ret += "%dm" % minutes
        ret += "%ds" % seconds
        return ret

    def progress(self, offset, whence):
        if whence == subunit.PROGRESS_POP:
            self._progress_level -= 1
        elif whence == subunit.PROGRESS_PUSH:
            self._progress_level += 1
        elif whence == subunit.PROGRESS_SET:
            if self._progress_level == 0:
                self.totalsuites = offset
        elif whence == subunit.PROGRESS_CUR:
            raise NotImplementedError

    def time(self, dt):
        if self.start_time is None:
            self.start_time = dt
        self.last_time = dt

    def start_testsuite(self, name):
        self.index += 1
        self.name = name

        if not self.verbose:
            self.test_output[name] = ""

569 570 571 572 573 574 575
        total_tests = (self.statistics['TESTS_EXPECTED_OK'] +
                       self.statistics['TESTS_EXPECTED_FAIL'] +
                       self.statistics['TESTS_ERROR'] +
                       self.statistics['TESTS_UNEXPECTED_FAIL'] +
                       self.statistics['TESTS_UNEXPECTED_OK'])

        out = "[%d(%d)" % (self.index, total_tests)
576 577 578
        if self.totalsuites is not None:
            out += "/%d" % self.totalsuites
        if self.start_time is not None:
579
            out += " at " + self._format_time(self.last_time - self.start_time)
580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
        if self.suitesfailed:
            out += ", %d errors" % (len(self.suitesfailed),)
        out += "] %s" % name
        if self.immediate:
            sys.stdout.write(out + "\n")
        else:
            sys.stdout.write(out + ": ")

    def output_msg(self, output):
        if self.verbose:
            sys.stdout.write(output)
        elif self.name is not None:
            self.test_output[self.name] += output
        else:
            sys.stdout.write(output)

    def control_msg(self, output):
        pass

    def end_testsuite(self, name, result, reason):
        out = ""
        unexpected = False

        if not name in self.test_output:
604
            print("no output for name[%s]" % name)
605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630

        if result in ("success", "xfail"):
            self.suites_ok+=1
        else:
            self.output_msg("ERROR: Testsuite[%s]\n" % name)
            if reason is not None:
                self.output_msg("REASON: %s\n" % (reason,))
            self.suitesfailed.append(name)
            if self.immediate and not self.verbose and name in self.test_output:
                out += self.test_output[name]
            unexpected = True

        if not self.immediate:
            if not unexpected:
                out += " ok\n"
            else:
                out += " " + result.upper() + "\n"

        sys.stdout.write(out)

    def startTest(self, test):
        pass

    def addSuccess(self, test):
        self.end_test(test.id(), "success", False)

631 632
    def addError(self, test, err=None):
        self.end_test(test.id(), "error", True, err)
633

634 635
    def addFailure(self, test, err=None):
        self.end_test(test.id(), "failure", True, err)
636

637 638
    def addSkip(self, test, reason=None):
        self.end_test(test.id(), "skip", False, reason)
639

640 641
    def addExpectedFailure(self, test, err=None):
        self.end_test(test.id(), "xfail", False, err)
642

643 644
    def addUnexpectedSuccess(self, test):
        self.end_test(test.id(), "uxsuccess", True)
645

646
    def end_test(self, testname, result, unexpected, err=None):
647 648 649 650 651 652 653 654 655 656 657 658 659 660
        if not unexpected:
            self.test_output[self.name] = ""
            if not self.immediate:
                sys.stdout.write({
                    'failure': 'f',
                    'xfail': 'X',
                    'skip': 's',
                    'success': '.'}.get(result, "?(%s)" % result))
            return

        if not self.name in self.test_output:
            self.test_output[self.name] = ""

        self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
661
        if err is not None:
662
            self.test_output[self.name] += "REASON: %s\n" % str(err[1]).strip()
663 664

        if self.immediate and not self.verbose:
665
            sys.stdout.write(self.test_output[self.name])
666 667 668 669 670 671
            self.test_output[self.name] = ""

        if not self.immediate:
            sys.stdout.write({
               'error': 'E',
               'failure': 'F',
672
               'uxsuccess': 'U',
673 674
               'success': 'S'}.get(result, "?"))

675 676
    def write_summary(self, path):
        f = open(path, 'w+')
677 678 679 680 681 682 683 684 685 686 687 688 689

        if self.suitesfailed:
            f.write("= Failed tests =\n")

            for suite in self.suitesfailed:
                f.write("== %s ==\n" % suite)
                if suite in self.test_output:
                    f.write(self.test_output[suite]+"\n\n")

            f.write("\n")

        if not self.immediate and not self.verbose:
            for suite in self.suitesfailed:
690 691
                print("=" * 78)
                print("FAIL: %s" % suite)
692
                if suite in self.test_output:
693 694
                    print(self.test_output[suite])
                print("")
695 696 697 698 699 700 701 702 703 704 705

        f.write("= Skipped tests =\n")
        for reason in self.skips.keys():
            f.write(reason + "\n")
            for name in self.skips[reason]:
                f.write("\t%s\n" % name)
            f.write("\n")
        f.close()

        if (not self.suitesfailed and
            not self.statistics['TESTS_UNEXPECTED_FAIL'] and
706
            not self.statistics['TESTS_UNEXPECTED_OK'] and
707 708 709
            not self.statistics['TESTS_ERROR']):
            ok = (self.statistics['TESTS_EXPECTED_OK'] +
                  self.statistics['TESTS_EXPECTED_FAIL'])
710
            print("\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok))
711
        else:
712
            print("\nFAILED (%d failures, %d errors and %d unexpected successes in %d testsuites)" % (
713 714
                self.statistics['TESTS_UNEXPECTED_FAIL'],
                self.statistics['TESTS_ERROR'],
715
                self.statistics['TESTS_UNEXPECTED_OK'],
716
                len(self.suitesfailed)))
717 718 719 720 721

    def skip_testsuite(self, name, reason="UNKNOWN"):
        self.skips.setdefault(reason, []).append(name)
        if self.totalsuites:
            self.totalsuites-=1