diff --git a/ffsubsync/ffsubsync.py b/ffsubsync/ffsubsync.py
index 41b0c40..b0114cc 100755
--- a/ffsubsync/ffsubsync.py
+++ b/ffsubsync/ffsubsync.py
@@ -178,6 +178,8 @@ def try_sync(
                     reference_pipe.transform(args.reference),
                     srt_pipes,
                 )
+            if best_score < 0:
+                sync_was_successful = False
             logger.info("...done")
             offset_seconds = (
                 offset_samples / float(SAMPLE_RATE) + args.apply_offset_seconds