diff --git a/setup.py b/setup.py
index 134d6f916..00a72b406 100755
--- a/setup.py
+++ b/setup.py
@@ -99,7 +99,9 @@ class picard_test(Command):
 
         tests = unittest.defaultTestLoader.loadTestsFromNames(names)
         t = unittest.TextTestRunner(verbosity=self.verbosity)
-        t.run(tests)
+        testresult = t.run(tests)
+        if not testresult.wasSuccessful():
+            raise SystemExit("At least one test failed.")
 
 
 class picard_build_locales(Command):