diff --git a/tests/models/tvp/test_modeling_tvp.py b/tests/models/tvp/test_modeling_tvp.py
index ebdd4fb0b7694b..14ec02ed6fd9a9 100644
--- a/tests/models/tvp/test_modeling_tvp.py
+++ b/tests/models/tvp/test_modeling_tvp.py
@@ -230,10 +230,11 @@ def test_inference_no_head(self):
 
         image_processor = self.default_image_processor
         image = prepare_img()
-        encoding = image_processor(images=image, return_tensors="pt").to(torch_device)
+        encoding = image_processor(images=image, return_tensors="pt")
         input_ids = torch.tensor([[1, 2]])
         attention_mask = torch.tensor([[1, 1]])
         encoding.update({"input_ids": input_ids, "attention_mask": attention_mask})
+        encoding.to(torch_device)
 
         with torch.no_grad():
             outputs = model(**encoding)
@@ -250,10 +251,11 @@ def test_inference_with_head(self):
 
         image_processor = self.default_image_processor
         image = prepare_img()
-        encoding = image_processor(images=image, return_tensors="pt").to(torch_device)
+        encoding = image_processor(images=image, return_tensors="pt")
         input_ids = torch.tensor([[1, 2]])
         attention_mask = torch.tensor([[1, 1]])
         encoding.update({"input_ids": input_ids, "attention_mask": attention_mask})
+        encoding.to(torch_device)
 
         with torch.no_grad():
             outputs = model(**encoding)