diff --git a/tests/models/bridgetower/test_modeling_bridgetower.py b/tests/models/bridgetower/test_modeling_bridgetower.py
index 1a4d34d3967f98..9c40f376a7b573 100644
--- a/tests/models/bridgetower/test_modeling_bridgetower.py
+++ b/tests/models/bridgetower/test_modeling_bridgetower.py
@@ -627,7 +627,8 @@ def _get_non_used_layer_names(self, model_class):
         non_used_layer_names = ["text_model.pooler"]
         if model_class == BridgeTowerForMaskedLM:
             non_used_layer_names = non_used_layer_names + [
-                "cross_modal_image_layers.5",
+                # This number `1` actually depends on the number of layers in `cross_modal_image_layers` (by minus 1)
+                "cross_modal_image_layers.1",
                 "cross_modal_image_pooler",
                 "cross_modal_text_pooler",
             ]