From 2caed46ea50dc8a5093e69042719a3a70c9243fe Mon Sep 17 00:00:00 2001
From: Kunal Mehta <legoktm@member.fsf.org>
Date: Sun, 12 Aug 2018 14:00:46 -0700
Subject: [PATCH] Run test suite during autopkgtest

Introduce a wrapper script around run-tests.php: run-tests.sh. It runs
the tests, and then diffs the failed tests (if any) against the
arch-specific test failures for that group. diff-failures.php is a
wrapper around diff to normalize the run-tests.php output to not include
absolute paths.

Tests are broken down into groups mostly for ease of maintenance.
* basic: tests/
* Zend: Zend/tests/
* ext: ext/*/tests/ (that are included in -common)
* ext-bz2: ext/bz2/tests/ (example of extension in a separate package)

If this system works, then future patches can add the rest of extension
tests, and sapi tests.
---
 debian/tests.in/cli                   |  1 +
 debian/tests.in/control               | 12 +++++++
 debian/tests.in/run-tests.sh          |  6 ++++
 debian/tests/basic-amd64-failures.txt |  6 ++++
 debian/tests/cli                      |  1 +
 debian/tests/cli-amd64-failures.txt   |  1 +
 debian/tests/control                  | 12 +++++++
 debian/tests/diff-failures.php        | 20 +++++++++++
 debian/tests/ext-amd64-failures.txt   | 49 +++++++++++++++++++++++++++
 debian/tests/run-Zend-tests           |  2 ++
 debian/tests/run-basic-tests          |  2 ++
 debian/tests/run-ext-bz2-tests        |  2 ++
 debian/tests/run-ext-tests            | 12 +++++++
 debian/tests/run-tests.sh             |  6 ++++
 14 files changed, 132 insertions(+)
 create mode 100755 debian/tests.in/run-tests.sh
 create mode 100644 debian/tests/basic-amd64-failures.txt
 create mode 100644 debian/tests/cli-amd64-failures.txt
 create mode 100644 debian/tests/diff-failures.php
 create mode 100644 debian/tests/ext-amd64-failures.txt
 create mode 100644 debian/tests/run-Zend-tests
 create mode 100644 debian/tests/run-basic-tests
 create mode 100644 debian/tests/run-ext-bz2-tests
 create mode 100644 debian/tests/run-ext-tests
 create mode 100755 debian/tests/run-tests.sh

diff --git a/debian/tests.in/cli b/debian/tests.in/cli
index 1e38ab97a..fcb781204 100644
--- a/debian/tests.in/cli
+++ b/debian/tests.in/cli
@@ -3,6 +3,7 @@ set -e
 
 # Author: Robie Basak <robie.basak@ubuntu.com>
 
+GROUP=cli ./debian/tests/run-tests.sh sapi/cli/tests/
 cd "$ADTTMP"
 cat > hello.php <<EOT
 <?php echo "Hello, world!\n"; ?>
diff --git a/debian/tests.in/control b/debian/tests.in/control
index dfe9a457d..bde79776f 100644
--- a/debian/tests.in/control
+++ b/debian/tests.in/control
@@ -12,3 +12,15 @@ Restrictions: needs-root
 Tests: fpm
 Depends: apache2, php@PHP_VERSION@-fpm, wget
 Restrictions: needs-root
+
+Tests: run-basic-tests
+Depends: php@PHP_VERSION@-cli, php@PHP_VERSION@-cgi, php@PHP_VERSION@-mbstring
+
+Tests: run-Zend-tests
+Depends: php@PHP_VERSION@-cli
+
+Tests: run-ext-tests
+Depends: php@PHP_VERSION@-cli
+
+Tests: run-ext-bz2-tests
+Depends: php@PHP_VERSION@-cli, php@PHP_VERSION@-bz2
diff --git a/debian/tests.in/run-tests.sh b/debian/tests.in/run-tests.sh
new file mode 100755
index 000000000..5040397bc
--- /dev/null
+++ b/debian/tests.in/run-tests.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+# Save any failures to a file, and then diff it against our known failures
+# if we have any, to spot any new regressions (or improvements!).
+TEST_PHP_EXECUTABLE=/usr/bin/php@PHP_VERSION@ php run-tests.php $@ -w failures.txt
+export ARCH=$(dpkg --print-architecture)
+php debian/tests/diff-failures.php "./debian/tests/$GROUP-$ARCH-failures.txt" failures.txt
diff --git a/debian/tests/basic-amd64-failures.txt b/debian/tests/basic-amd64-failures.txt
new file mode 100644
index 000000000..ae75e6d03
--- /dev/null
+++ b/debian/tests/basic-amd64-failures.txt
@@ -0,0 +1,6 @@
+tests/output/stream_isatty_err.phpt
+tests/output/stream_isatty_in-err.phpt
+tests/output/stream_isatty_in-out.phpt
+tests/output/stream_isatty_out-err.phpt
+tests/output/stream_isatty_out.phpt
+tests/security/open_basedir_linkinfo.phpt
diff --git a/debian/tests/cli b/debian/tests/cli
index 344223cd4..3a1e26d7d 100644
--- a/debian/tests/cli
+++ b/debian/tests/cli
@@ -3,6 +3,7 @@ set -e
 
 # Author: Robie Basak <robie.basak@ubuntu.com>
 
+GROUP=cli ./debian/tests/run-tests.sh sapi/cli/tests/
 cd "$ADTTMP"
 cat > hello.php <<EOT
 <?php echo "Hello, world!\n"; ?>
diff --git a/debian/tests/cli-amd64-failures.txt b/debian/tests/cli-amd64-failures.txt
new file mode 100644
index 000000000..24db8dfa5
--- /dev/null
+++ b/debian/tests/cli-amd64-failures.txt
@@ -0,0 +1 @@
+sapi/cli/tests/017.phpt
diff --git a/debian/tests/control b/debian/tests/control
index 9ee24bbfa..41b19b88b 100644
--- a/debian/tests/control
+++ b/debian/tests/control
@@ -12,3 +12,15 @@ Restrictions: needs-root
 Tests: fpm
 Depends: apache2, php7.2-fpm, wget
 Restrictions: needs-root
+
+Tests: run-basic-tests
+Depends: php7.2-cli, php7.2-cgi, php7.2-mbstring
+
+Tests: run-Zend-tests
+Depends: php7.2-cli
+
+Tests: run-ext-tests
+Depends: php7.2-cli
+
+Tests: run-ext-bz2-tests
+Depends: php7.2-cli, php7.2-bz2
diff --git a/debian/tests/diff-failures.php b/debian/tests/diff-failures.php
new file mode 100644
index 000000000..f951c64ef
--- /dev/null
+++ b/debian/tests/diff-failures.php
@@ -0,0 +1,20 @@
+<?php
+// Usage:
+//  php run-tests.php -w actual-failures.txt
+//  php diff-failures.php expected-failures.txt actual-failures.txt
+// $argv[0] is the script, 1 is expected, 2 is actual
+
+$actual = str_replace(getcwd() . '/', '', file_get_contents($argv[2]));
+file_put_contents($argv[2], $actual);
+
+$output = [];
+$retVal = 0;
+if (!file_exists($argv[1])) {
+    // If expected doesn't exist, we expect no failures.
+    file_put_contents($argv[1], '');
+}
+exec("diff -u {$argv[1]} {$argv[2]}", $output, $retVal);
+echo implode("\n", $output) . "\n";
+
+exit($retVal);
+
diff --git a/debian/tests/ext-amd64-failures.txt b/debian/tests/ext-amd64-failures.txt
new file mode 100644
index 000000000..856beef48
--- /dev/null
+++ b/debian/tests/ext-amd64-failures.txt
@@ -0,0 +1,49 @@
+ext/date/tests/DateTime_add-fall-type2-type3.phpt
+ext/date/tests/DateTime_add-fall-type3-type2.phpt
+ext/date/tests/DateTime_add-fall-type3-type3.phpt
+ext/date/tests/DateTime_add-spring-type2-type3.phpt
+ext/date/tests/DateTime_add-spring-type3-type2.phpt
+ext/date/tests/DateTime_add-spring-type3-type3.phpt
+ext/date/tests/DateTime_diff-fall-type2-type3.phpt
+ext/date/tests/DateTime_diff-fall-type3-type2.phpt
+ext/date/tests/DateTime_diff-fall-type3-type3.phpt
+ext/date/tests/DateTime_diff-spring-type2-type3.phpt
+ext/date/tests/DateTime_diff-spring-type3-type2.phpt
+ext/date/tests/DateTime_diff-spring-type3-type3.phpt
+ext/date/tests/DateTime_sub-fall-type2-type3.phpt
+ext/date/tests/DateTime_sub-fall-type3-type2.phpt
+ext/date/tests/DateTime_sub-fall-type3-type3.phpt
+ext/date/tests/DateTime_sub-spring-type2-type3.phpt
+ext/date/tests/DateTime_sub-spring-type3-type2.phpt
+ext/date/tests/DateTime_sub-spring-type3-type3.phpt
+ext/date/tests/bug52480.phpt
+ext/date/tests/rfc-datetime_and_daylight_saving_time-type3-bd2.phpt
+ext/date/tests/rfc-datetime_and_daylight_saving_time-type3-fs.phpt
+ext/date/tests/timezone_version_get.phpt
+ext/date/tests/timezone_version_get_basic1.phpt
+ext/filter/tests/bug49184.phpt
+ext/filter/tests/bug67167.02.phpt
+ext/ftp/tests/004.phpt
+ext/ftp/tests/ftp_ssl_connect_error.phpt
+ext/openssl/tests/bug72165.phpt
+ext/openssl/tests/ecc.phpt
+ext/openssl/tests/openssl_error_string_basic.phpt
+ext/openssl/tests/stream_security_level.phpt
+ext/pcre/tests/bug37911.phpt
+ext/pcre/tests/grep2.phpt
+ext/pcre/tests/match_flags3.phpt
+ext/phar/tests/bug69958.phpt
+ext/session/tests/bug71162.phpt
+ext/session/tests/bug73529.phpt
+ext/session/tests/bug74514.phpt
+ext/session/tests/bug74936.phpt
+ext/session/tests/session_gc_basic.phpt
+ext/spl/tests/DirectoryIterator_getGroup_basic.phpt
+ext/standard/tests/checkdnsrr.phpt
+ext/standard/tests/directory/DirectoryClass_error_001-mb.phpt
+ext/standard/tests/dns_get_record.phpt
+ext/standard/tests/file/bug41655_1.phpt
+ext/standard/tests/file/glob_variation5.phpt
+ext/standard/tests/serialize/bug70219.phpt
+ext/standard/tests/streams/stream_context_tcp_nodelay_fopen.phpt
+ext/standard/tests/streams/stream_socket_sendto.phpt
diff --git a/debian/tests/run-Zend-tests b/debian/tests/run-Zend-tests
new file mode 100644
index 000000000..43aecadaf
--- /dev/null
+++ b/debian/tests/run-Zend-tests
@@ -0,0 +1,2 @@
+#!/bin/sh
+GROUP=Zend ./debian/tests/run-tests.sh Zend/tests/
diff --git a/debian/tests/run-basic-tests b/debian/tests/run-basic-tests
new file mode 100644
index 000000000..35237bcba
--- /dev/null
+++ b/debian/tests/run-basic-tests
@@ -0,0 +1,2 @@
+#!/bin/sh
+GROUP=basic ./debian/tests/run-tests.sh tests/
diff --git a/debian/tests/run-ext-bz2-tests b/debian/tests/run-ext-bz2-tests
new file mode 100644
index 000000000..94a595f28
--- /dev/null
+++ b/debian/tests/run-ext-bz2-tests
@@ -0,0 +1,2 @@
+#!/bin/sh
+GROUP=ext-bz2 ./debian/tests/run-tests.sh ext/bz2/tests/
diff --git a/debian/tests/run-ext-tests b/debian/tests/run-ext-tests
new file mode 100644
index 000000000..6e5da5517
--- /dev/null
+++ b/debian/tests/run-ext-tests
@@ -0,0 +1,12 @@
+#!/bin/sh
+GROUP=ext ./debian/tests/run-tests.sh \
+ ext/calendar/tests/ ext/ctype/tests/ ext/date/tests/ \
+ ext/exif/tests/ ext/fileinfo/tests/ ext/filter/tests/ \
+ ext/ftp/tests/ ext/gettext/tests/ ext/hash/tests/ \
+ ext/iconv/tests/ ext/libxml/tests/ ext/openssl/tests/ \
+ ext/pcntl/tests/ ext/pcre/tests/ ext/pdo/tests/ \
+ ext/phar/tests/ ext/posix/tests/ ext/reflection/tests/ \
+ ext/session/tests/ ext/shmop/tests/ ext/sockets/tests/ \
+ ext/sodium/tests/ ext/spl/tests/ ext/standard/tests/ \
+ ext/sysvmsg/tests/  ext/sysvshm/tests/ ext/tokenizer/tests/ \
+ ext/zlib/tests/
diff --git a/debian/tests/run-tests.sh b/debian/tests/run-tests.sh
new file mode 100755
index 000000000..c0ea89516
--- /dev/null
+++ b/debian/tests/run-tests.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+# Save any failures to a file, and then diff it against our known failures
+# if we have any, to spot any new regressions (or improvements!).
+TEST_PHP_EXECUTABLE=/usr/bin/php7.2 php run-tests.php $@ -w failures.txt
+export ARCH=$(dpkg --print-architecture)
+php debian/tests/diff-failures.php "./debian/tests/$GROUP-$ARCH-failures.txt" failures.txt
-- 
GitLab