diff --git a/controller/state.go b/controller/state.go
index bb4fdd3d21fed..b6bafcb5c83ac 100644
--- a/controller/state.go
+++ b/controller/state.go
@@ -965,6 +965,10 @@ func specEqualsCompareTo(spec v1alpha1.ApplicationSpec, comparedTo v1alpha1.Comp
 		currentSpec.Destination.Name = ""
 	}
 
+	// Set IsServerInferred to false on both, because that field is not important for comparison.
+	comparedTo.Destination.SetIsServerInferred(false)
+	currentSpec.Destination.SetIsServerInferred(false)
+
 	return reflect.DeepEqual(comparedTo, currentSpec)
 }
 
diff --git a/controller/state_test.go b/controller/state_test.go
index 971b7ac751273..658e72224aeb9 100644
--- a/controller/state_test.go
+++ b/controller/state_test.go
@@ -1528,6 +1528,10 @@ func TestUseDiffCache(t *testing.T) {
 				t.Fatalf("error merging app: %s", err)
 			}
 		}
+		if app.Spec.Destination.Name != "" && app.Spec.Destination.Server != "" {
+			// Simulate the controller's process for populating both of these fields.
+			app.Spec.Destination.SetInferredServer(app.Spec.Destination.Server)
+		}
 		return app
 	}
 
diff --git a/pkg/apis/application/v1alpha1/types.go b/pkg/apis/application/v1alpha1/types.go
index 4afeac5183615..998b1fbf1dfb6 100644
--- a/pkg/apis/application/v1alpha1/types.go
+++ b/pkg/apis/application/v1alpha1/types.go
@@ -1003,6 +1003,12 @@ type ApplicationDestination struct {
 	isServerInferred bool `json:"-"`
 }
 
+// SetIsServerInferred sets the isServerInferred flag. This is used to allow comparison between two destinations where
+// one server is inferred and the other is not.
+func (d *ApplicationDestination) SetIsServerInferred(inferred bool) {
+	d.isServerInferred = inferred
+}
+
 type ResourceHealthLocation string
 
 var (