diff --git a/data_diff/__main__.py b/data_diff/__main__.py
index ba006d36..016f3d7d 100644
--- a/data_diff/__main__.py
+++ b/data_diff/__main__.py
@@ -327,8 +327,7 @@ def main(conf, run, **kw):
             )
     except Exception as e:
         logging.error(e)
-        if kw["debug"]:
-            raise
+        raise
 
 
 def _data_diff(
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 1fc4833e..9dfd97c4 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -11,12 +11,17 @@
 
 def run_datadiff_cli(*args):
     try:
-        stdout = subprocess.check_output(
-            [sys.executable, "-m", "data_diff", "--no-tracking"] + list(args), stderr=subprocess.PIPE
+        p = subprocess.Popen(
+            [sys.executable, "-m", "data_diff", "--no-tracking"] + list(args),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
+        stdout, stderr = p.communicate()
     except subprocess.CalledProcessError as e:
         logging.error(e.stderr)
         raise
+    if stderr:
+        raise Exception(stderr)
     return stdout.splitlines()
 
 
@@ -48,6 +53,7 @@ def setUp(self) -> None:
     def test_basic(self):
         conn_str = CONN_STRINGS[self.db_cls]
         diff = run_datadiff_cli(conn_str, self.table_src_name, conn_str, self.table_dst_name)
+
         assert len(diff) == 1
 
     def test_options(self):