diff --git a/controller/state.go b/controller/state.go
index 10b7c6b6a9d40..4e9b383a2e0cb 100644
--- a/controller/state.go
+++ b/controller/state.go
@@ -933,6 +933,10 @@ func specEqualsCompareTo(spec v1alpha1.ApplicationSpec, comparedTo v1alpha1.Comp
 		currentSpec.Destination.Name = ""
 	}
 
+	// Set IsServerInferred to false on both, because that field is not important for comparison.
+	comparedTo.Destination.SetIsServerInferred(false)
+	currentSpec.Destination.SetIsServerInferred(false)
+
 	return reflect.DeepEqual(comparedTo, currentSpec)
 }
 
diff --git a/controller/state_test.go b/controller/state_test.go
index bf87256eb5bae..0c92400e9106e 100644
--- a/controller/state_test.go
+++ b/controller/state_test.go
@@ -1529,6 +1529,10 @@ func TestUseDiffCache(t *testing.T) {
 				t.Fatalf("error merging app: %s", err)
 			}
 		}
+		if app.Spec.Destination.Name != "" && app.Spec.Destination.Server != "" {
+			// Simulate the controller's process for populating both of these fields.
+			app.Spec.Destination.SetInferredServer(app.Spec.Destination.Server)
+		}
 		return app
 	}
 
diff --git a/pkg/apis/application/v1alpha1/types.go b/pkg/apis/application/v1alpha1/types.go
index 2aa0c0ee34b81..a5d52f3d3f75a 100644
--- a/pkg/apis/application/v1alpha1/types.go
+++ b/pkg/apis/application/v1alpha1/types.go
@@ -938,6 +938,12 @@ type ApplicationDestination struct {
 	isServerInferred bool `json:"-"`
 }
 
+// SetIsServerInferred sets the isServerInferred flag. This is used to allow comparison between two destinations where
+// one server is inferred and the other is not.
+func (d *ApplicationDestination) SetIsServerInferred(inferred bool) {
+	d.isServerInferred = inferred
+}
+
 type ResourceHealthLocation string
 
 var (