Add warning when metric is skipped because it's not on the experiment (…

…facebook#2449)

Summary:
Pull Request resolved: facebook#2449

See discussion on D57165887 and D56634321.

Reviewed By: SebastianAment

Differential Revision: D57174994

fbshipit-source-id: 06b705ce7782db8014d596407abc3da463dddd05

ax/core/observation.py

@@ -380,6 +380,11 @@ def _filter_data_on_status(
         metric_name = g
         # Filter out any metrics that are not on the experiment.
         if metric_name not in experiment.metrics:
+            warnings.warn(
+                f"Metric {metric_name} not found on {experiment}. Not attaching to "
+                "observation.",
+                stacklevel=2,
+            )
             continue
         metric = experiment.metrics[metric_name]
         statuses_to_include_metric = (