diff --git a/roles/bodhi2/backend/files/bodhi-automated-pushes.py b/roles/bodhi2/backend/files/bodhi-automated-pushes.py
index aacc81cfb1..e27c79b364 100644
--- a/roles/bodhi2/backend/files/bodhi-automated-pushes.py
+++ b/roles/bodhi2/backend/files/bodhi-automated-pushes.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import sys
 import requests
 import json
 import subprocess
@@ -7,6 +10,11 @@ req = requests.get('https://bodhi.fedoraproject.org/composes/')
 bodhi_composes = req.json()
 
 if len(bodhi_composes['composes']) == 0:
-    bodhi_push_cmd = ["bodhi-push", "--username", "releng"]
-    push = subprocess.Popen(bodhi_push_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
+    bodhi_push_cmd = ['bodhi-push', '--username', 'releng']
+    push = subprocess.Popen(bodhi_push_cmd, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
     push.stdin.write('y')
+    _, err = push.communicate()
+    push.wait()
+    if push.returncode != 0:
+        print(err, file=sys.stderr)
+        sys.exit(1)