diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7b979ec28..7c6afdfb8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Following a breaking change in the NLTK API, we now depend on the most recent version only.
 - Added Tensorboard callbacks to the RC models
+- The error message you get when perl isn't installed is now more readable.
 
 ### Removed
 
diff --git a/allennlp_models/structured_prediction/metrics/srl_eval_scorer.py b/allennlp_models/structured_prediction/metrics/srl_eval_scorer.py
index 06ad0bdb5..672e51e3a 100644
--- a/allennlp_models/structured_prediction/metrics/srl_eval_scorer.py
+++ b/allennlp_models/structured_prediction/metrics/srl_eval_scorer.py
@@ -104,9 +104,14 @@ def __call__(
                     gold_tag_sequence,
                 )
         perl_script_command = ["perl", self._srl_eval_path, gold_path, predicted_path]
-        completed_process = subprocess.run(
-            perl_script_command, stdout=subprocess.PIPE, universal_newlines=True, check=True
-        )
+        try:
+            completed_process = subprocess.run(
+                perl_script_command, stdout=subprocess.PIPE, universal_newlines=True, check=True
+            )
+        except FileNotFoundError:
+            raise FileNotFoundError(
+                "'File not found' while running the evaluation. Do you have perl installed?"
+            )
         for line in completed_process.stdout.split("\n"):
             stripped = line.strip().split()
             if len(stripped) == 7: