diff --git a/debian/tests.in/cli b/debian/tests.in/cli
index 1e38ab97ab1a1b13cbb32721f09167c8878e17f3..fcb7812044cc31232b2f4416ff1a4f644cd6d17e 100644
--- a/debian/tests.in/cli
+++ b/debian/tests.in/cli
@@ -3,6 +3,7 @@ set -e
 
 # Author: Robie Basak <robie.basak@ubuntu.com>
 
+GROUP=cli ./debian/tests/run-tests.sh sapi/cli/tests/
 cd "$ADTTMP"
 cat > hello.php <<EOT
 <?php echo "Hello, world!\n"; ?>
diff --git a/debian/tests.in/control b/debian/tests.in/control
index dfe9a457db3d794191c3a55c76bafe81e0f98c05..bde79776fefc416f83b6beafe22a83438e9abe15 100644
--- a/debian/tests.in/control
+++ b/debian/tests.in/control
@@ -12,3 +12,15 @@ Restrictions: needs-root
 Tests: fpm
 Depends: apache2, php@PHP_VERSION@-fpm, wget
 Restrictions: needs-root
+
+Tests: run-basic-tests
+Depends: php@PHP_VERSION@-cli, php@PHP_VERSION@-cgi, php@PHP_VERSION@-mbstring
+
+Tests: run-Zend-tests
+Depends: php@PHP_VERSION@-cli
+
+Tests: run-ext-tests
+Depends: php@PHP_VERSION@-cli
+
+Tests: run-ext-bz2-tests
+Depends: php@PHP_VERSION@-cli, php@PHP_VERSION@-bz2
diff --git a/debian/tests.in/run-tests.sh b/debian/tests.in/run-tests.sh
new file mode 100755
index 0000000000000000000000000000000000000000..5040397bc4e323bd8d1b8796f836a100ecfcd43f
--- /dev/null
+++ b/debian/tests.in/run-tests.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+# Save any failures to a file, and then diff it against our known failures
+# if we have any, to spot any new regressions (or improvements!).
+TEST_PHP_EXECUTABLE=/usr/bin/php@PHP_VERSION@ php run-tests.php $@ -w failures.txt
+export ARCH=$(dpkg --print-architecture)
+php debian/tests/diff-failures.php "./debian/tests/$GROUP-$ARCH-failures.txt" failures.txt
diff --git a/debian/tests/basic-amd64-failures.txt b/debian/tests/basic-amd64-failures.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ae75e6d037488a18741b107e612b16cff302d55a
--- /dev/null
+++ b/debian/tests/basic-amd64-failures.txt
@@ -0,0 +1,6 @@
+tests/output/stream_isatty_err.phpt
+tests/output/stream_isatty_in-err.phpt
+tests/output/stream_isatty_in-out.phpt
+tests/output/stream_isatty_out-err.phpt
+tests/output/stream_isatty_out.phpt
+tests/security/open_basedir_linkinfo.phpt
diff --git a/debian/tests/cli b/debian/tests/cli
index 344223cd42013d5992e8e19b50f267443cb5f251..3a1e26d7dc3c4680cfbf5245e6830bcee6f59d34 100644
--- a/debian/tests/cli
+++ b/debian/tests/cli
@@ -3,6 +3,7 @@ set -e
 
 # Author: Robie Basak <robie.basak@ubuntu.com>
 
+GROUP=cli ./debian/tests/run-tests.sh sapi/cli/tests/
 cd "$ADTTMP"
 cat > hello.php <<EOT
 <?php echo "Hello, world!\n"; ?>
diff --git a/debian/tests/cli-amd64-failures.txt b/debian/tests/cli-amd64-failures.txt
new file mode 100644
index 0000000000000000000000000000000000000000..24db8dfa54ec5024f08ead868cb3a653057155e3
--- /dev/null
+++ b/debian/tests/cli-amd64-failures.txt
@@ -0,0 +1 @@
+sapi/cli/tests/017.phpt
diff --git a/debian/tests/control b/debian/tests/control
index 9ee24bbfa9f484caa451c91bd494ccbeb2050ab2..41b19b88b14857d16dc0c6621aa6cc22328ca776 100644
--- a/debian/tests/control
+++ b/debian/tests/control
@@ -12,3 +12,15 @@ Restrictions: needs-root
 Tests: fpm
 Depends: apache2, php7.2-fpm, wget
 Restrictions: needs-root
+
+Tests: run-basic-tests
+Depends: php7.2-cli, php7.2-cgi, php7.2-mbstring
+
+Tests: run-Zend-tests
+Depends: php7.2-cli
+
+Tests: run-ext-tests
+Depends: php7.2-cli
+
+Tests: run-ext-bz2-tests
+Depends: php7.2-cli, php7.2-bz2
diff --git a/debian/tests/diff-failures.php b/debian/tests/diff-failures.php
new file mode 100644
index 0000000000000000000000000000000000000000..f951c64efb7c1fad6b669aa42544d0045489e172
--- /dev/null
+++ b/debian/tests/diff-failures.php
@@ -0,0 +1,20 @@
+<?php
+// Usage:
+//  php run-tests.php -w actual-failures.txt
+//  php diff-failures.php expected-failures.txt actual-failures.txt
+// $argv[0] is the script, 1 is expected, 2 is actual
+
+$actual = str_replace(getcwd() . '/', '', file_get_contents($argv[2]));
+file_put_contents($argv[2], $actual);
+
+$output = [];
+$retVal = 0;
+if (!file_exists($argv[1])) {
+    // If expected doesn't exist, we expect no failures.
+    file_put_contents($argv[1], '');
+}
+exec("diff -u {$argv[1]} {$argv[2]}", $output, $retVal);
+echo implode("\n", $output) . "\n";
+
+exit($retVal);
+
diff --git a/debian/tests/ext-amd64-failures.txt b/debian/tests/ext-amd64-failures.txt
new file mode 100644
index 0000000000000000000000000000000000000000..856beef488b876123c3ad212058b450af410105a
--- /dev/null
+++ b/debian/tests/ext-amd64-failures.txt
@@ -0,0 +1,49 @@
+ext/date/tests/DateTime_add-fall-type2-type3.phpt
+ext/date/tests/DateTime_add-fall-type3-type2.phpt
+ext/date/tests/DateTime_add-fall-type3-type3.phpt
+ext/date/tests/DateTime_add-spring-type2-type3.phpt
+ext/date/tests/DateTime_add-spring-type3-type2.phpt
+ext/date/tests/DateTime_add-spring-type3-type3.phpt
+ext/date/tests/DateTime_diff-fall-type2-type3.phpt
+ext/date/tests/DateTime_diff-fall-type3-type2.phpt
+ext/date/tests/DateTime_diff-fall-type3-type3.phpt
+ext/date/tests/DateTime_diff-spring-type2-type3.phpt
+ext/date/tests/DateTime_diff-spring-type3-type2.phpt
+ext/date/tests/DateTime_diff-spring-type3-type3.phpt
+ext/date/tests/DateTime_sub-fall-type2-type3.phpt
+ext/date/tests/DateTime_sub-fall-type3-type2.phpt
+ext/date/tests/DateTime_sub-fall-type3-type3.phpt
+ext/date/tests/DateTime_sub-spring-type2-type3.phpt
+ext/date/tests/DateTime_sub-spring-type3-type2.phpt
+ext/date/tests/DateTime_sub-spring-type3-type3.phpt
+ext/date/tests/bug52480.phpt
+ext/date/tests/rfc-datetime_and_daylight_saving_time-type3-bd2.phpt
+ext/date/tests/rfc-datetime_and_daylight_saving_time-type3-fs.phpt
+ext/date/tests/timezone_version_get.phpt
+ext/date/tests/timezone_version_get_basic1.phpt
+ext/filter/tests/bug49184.phpt
+ext/filter/tests/bug67167.02.phpt
+ext/ftp/tests/004.phpt
+ext/ftp/tests/ftp_ssl_connect_error.phpt
+ext/openssl/tests/bug72165.phpt
+ext/openssl/tests/ecc.phpt
+ext/openssl/tests/openssl_error_string_basic.phpt
+ext/openssl/tests/stream_security_level.phpt
+ext/pcre/tests/bug37911.phpt
+ext/pcre/tests/grep2.phpt
+ext/pcre/tests/match_flags3.phpt
+ext/phar/tests/bug69958.phpt
+ext/session/tests/bug71162.phpt
+ext/session/tests/bug73529.phpt
+ext/session/tests/bug74514.phpt
+ext/session/tests/bug74936.phpt
+ext/session/tests/session_gc_basic.phpt
+ext/spl/tests/DirectoryIterator_getGroup_basic.phpt
+ext/standard/tests/checkdnsrr.phpt
+ext/standard/tests/directory/DirectoryClass_error_001-mb.phpt
+ext/standard/tests/dns_get_record.phpt
+ext/standard/tests/file/bug41655_1.phpt
+ext/standard/tests/file/glob_variation5.phpt
+ext/standard/tests/serialize/bug70219.phpt
+ext/standard/tests/streams/stream_context_tcp_nodelay_fopen.phpt
+ext/standard/tests/streams/stream_socket_sendto.phpt
diff --git a/debian/tests/run-Zend-tests b/debian/tests/run-Zend-tests
new file mode 100644
index 0000000000000000000000000000000000000000..43aecadaf0685536abfbb9e768fdfee5d5c1af24
--- /dev/null
+++ b/debian/tests/run-Zend-tests
@@ -0,0 +1,2 @@
+#!/bin/sh
+GROUP=Zend ./debian/tests/run-tests.sh Zend/tests/
diff --git a/debian/tests/run-basic-tests b/debian/tests/run-basic-tests
new file mode 100644
index 0000000000000000000000000000000000000000..35237bcbae3b38499ea76354268d66e6c3dd4e54
--- /dev/null
+++ b/debian/tests/run-basic-tests
@@ -0,0 +1,2 @@
+#!/bin/sh
+GROUP=basic ./debian/tests/run-tests.sh tests/
diff --git a/debian/tests/run-ext-bz2-tests b/debian/tests/run-ext-bz2-tests
new file mode 100644
index 0000000000000000000000000000000000000000..94a595f28a40c9933d68c56b84d098ebaad062b4
--- /dev/null
+++ b/debian/tests/run-ext-bz2-tests
@@ -0,0 +1,2 @@
+#!/bin/sh
+GROUP=ext-bz2 ./debian/tests/run-tests.sh ext/bz2/tests/
diff --git a/debian/tests/run-ext-tests b/debian/tests/run-ext-tests
new file mode 100644
index 0000000000000000000000000000000000000000..6e5da5517765f303b75935bce248fa80a223dd34
--- /dev/null
+++ b/debian/tests/run-ext-tests
@@ -0,0 +1,12 @@
+#!/bin/sh
+GROUP=ext ./debian/tests/run-tests.sh \
+ ext/calendar/tests/ ext/ctype/tests/ ext/date/tests/ \
+ ext/exif/tests/ ext/fileinfo/tests/ ext/filter/tests/ \
+ ext/ftp/tests/ ext/gettext/tests/ ext/hash/tests/ \
+ ext/iconv/tests/ ext/libxml/tests/ ext/openssl/tests/ \
+ ext/pcntl/tests/ ext/pcre/tests/ ext/pdo/tests/ \
+ ext/phar/tests/ ext/posix/tests/ ext/reflection/tests/ \
+ ext/session/tests/ ext/shmop/tests/ ext/sockets/tests/ \
+ ext/sodium/tests/ ext/spl/tests/ ext/standard/tests/ \
+ ext/sysvmsg/tests/  ext/sysvshm/tests/ ext/tokenizer/tests/ \
+ ext/zlib/tests/
diff --git a/debian/tests/run-tests.sh b/debian/tests/run-tests.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c0ea8951680228733495fd0eb14a711df2ce5282
--- /dev/null
+++ b/debian/tests/run-tests.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+# Save any failures to a file, and then diff it against our known failures
+# if we have any, to spot any new regressions (or improvements!).
+TEST_PHP_EXECUTABLE=/usr/bin/php7.2 php run-tests.php $@ -w failures.txt
+export ARCH=$(dpkg --print-architecture)
+php debian/tests/diff-failures.php "./debian/tests/$GROUP-$ARCH-failures.txt" failures.txt