diff --git a/debian/ruby-tests.rake b/debian/ruby-tests.rake
index 8297103336a7dd5695738c682d177476678d3ebc..80560427d23295bad834907abd376fce76e96dff 100644
--- a/debian/ruby-tests.rake
+++ b/debian/ruby-tests.rake
@@ -1,10 +1,15 @@
 require 'gem2deb/rake/spectask'
 
+# Upstream recommended to disable running the test suite, and everytime we try
+# to run it again, sooner or later we get FTBFS bugs due to random failures.
+# Instead of running the upstream test suite, let's run just some very basic
+# smoke tests of our own.
+#
+# Links:
+# https://github.com/grosser/parallel/issues/227
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=905648
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1019646
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1017218
 Gem2Deb::Rake::RSpecTask.new do |spec|
-  spec.pattern = './spec/**/*_spec.rb'
+  spec.pattern = 'debian/tests/*_spec.rb'
 end
-
-task :default
-  require "parallel"
-  puts "can load parallel gem"
-
diff --git a/debian/tests/smoke_tests_spec.rb b/debian/tests/smoke_tests_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..a0fcaeec1e40f39f31e4391bb6abaa8690e355d1
--- /dev/null
+++ b/debian/tests/smoke_tests_spec.rb
@@ -0,0 +1,12 @@
+require 'etc'
+require 'benchmark'
+require 'parallel'
+
+describe 'Parallel' do
+  it 'works for a simple case' do
+    skip if Etc.nprocessors < 2
+    sequential = Benchmark.measure { (0..100).each { sleep 0.01} }
+    parallel = Benchmark.measure { Parallel.each(0..100) { sleep 0.01} }
+    expect(parallel.real).to be < sequential.real
+  end
+end