diff --git a/scripts/all-tests.sh b/scripts/all-tests.sh
index d4751ad858..d7dfbdd6b1 100755
--- a/scripts/all-tests.sh
+++ b/scripts/all-tests.sh
@@ -94,13 +94,13 @@ END
     "unit")
         case "$SHARD" in
             "lms")
-                paver test_system -s lms --extra_args="--with-flaky"
+                paver test_system -s lms --extra_args="--with-flaky" || { EXIT=1; }
                 paver coverage
                 ;;
             "cms-js-commonlib")
-                paver test_system -s cms --extra_args="--with-flaky"
-                paver test_js --coverage --skip_clean
-                paver test_lib --skip_clean --extra_args="--with-flaky"
+                paver test_system -s cms --extra_args="--with-flaky" || { EXIT=1; }
+                paver test_js --coverage --skip_clean || { EXIT=1; }
+                paver test_lib --skip_clean --extra_args="--with-flaky" || { EXIT=1; }
                 paver coverage
                 ;;
             *)
@@ -108,6 +108,8 @@ END
                 paver coverage
                 ;;
         esac
+
+        exit $EXIT
         ;;
 
     "lms-acceptance")
@@ -157,23 +159,23 @@ END
         case "$SHARD" in
 
             "all")
-                paver test_bokchoy
+                paver test_bokchoy || { EXIT=1; }
                 ;;
 
             "1")
-                paver test_bokchoy --extra_args="-a shard_1 --with-flaky"
+                paver test_bokchoy --extra_args="-a shard_1 --with-flaky" || { EXIT=1; }
                 ;;
 
             "2")
-                paver test_bokchoy --extra_args="-a 'shard_2' --with-flaky"
+                paver test_bokchoy --extra_args="-a 'shard_2' --with-flaky" || { EXIT=1; }
                 ;;
 
             "3")
-                paver test_bokchoy --extra_args="-a 'shard_3' --with-flaky"
+                paver test_bokchoy --extra_args="-a 'shard_3' --with-flaky" || { EXIT=1; }
                 ;;
 
             "4")
-                paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False,shard_3=False --with-flaky"
+                paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False,shard_3=False --with-flaky" || { EXIT=1; }
                 ;;
 
             # Default case because if we later define another bok-choy shard on Jenkins
@@ -196,6 +198,15 @@ END
 END
                 ;;
         esac
+
+        # Move the reports to a directory that is unique to the shard
+        # so that when they are 'slurped' to the main flow job, they
+        # do not conflict with and overwrite reports from other shards.
+        mv reports/ reports_tmp/
+        mkdir -p reports/${TEST_SUITE}/${SHARD}
+        mv reports_tmp/* reports/${TEST_SUITE}/${SHARD}
+        rm -r reports_tmp/
+        exit $EXIT
         ;;
 
 esac