Fix ReactiveStorageIT#testScaleDuringSplitOrClone

Sometimes the autoscaling decider returns an empty response when
the service does not have enough information to provide an autoscaling
decision, i.e. when a new node joins it tries to fetch the new node
memory info and this might take a while. This commits adds a busy
assertion to ensure that a valid autoscaling capacity is provided
eventually.

Closes elastic#88478

x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java

@@ -38,6 +38,8 @@
 import static org.elasticsearch.index.store.Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class ReactiveStorageIT extends AutoscalingStorageIntegTestCase {
@@ -356,7 +358,6 @@ public void testScaleWhileShrinking() throws Exception {
         ensureGreen();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/88478")
     public void testScaleDuringSplitOrClone() throws Exception {
         internalCluster().startMasterOnlyNode();
         final String dataNode1Name = internalCluster().startDataOnlyNode();
@@ -392,15 +393,20 @@ public void testScaleDuringSplitOrClone() throws Exception {
         setTotalSpace(dataNode1Name, enoughSpace);
         setTotalSpace(dataNode2Name, enoughSpace);
 
-        // validate initial state looks good
-        GetAutoscalingCapacityAction.Response response = capacity();
-        assertThat(response.results().keySet(), equalTo(Set.of(policyName)));
-        assertThat(response.results().get(policyName).currentCapacity().total().storage().getBytes(), equalTo(enoughSpace * 2));
-        assertThat(response.results().get(policyName).requiredCapacity().total().storage().getBytes(), equalTo(enoughSpace * 2));
-        assertThat(
-            response.results().get(policyName).requiredCapacity().node().storage().getBytes(),
-            equalTo(used + LOW_WATERMARK_BYTES + ReactiveStorageDeciderService.NODE_DISK_OVERHEAD)
-        );
+        // It might take a while until the autoscaling polls the node information of dataNode2 and
+        // provides a complete autoscaling capacity response
+        assertBusy(() -> {
+            // validate initial state looks good
+            GetAutoscalingCapacityAction.Response response = capacity();
+            assertThat(response.results().keySet(), equalTo(Set.of(policyName)));
+            assertThat(response.results().get(policyName).currentCapacity().total().storage().getBytes(), equalTo(enoughSpace * 2));
+            assertThat(response.results().get(policyName).requiredCapacity(), is(notNullValue()));
+            assertThat(response.results().get(policyName).requiredCapacity().total().storage().getBytes(), equalTo(enoughSpace * 2));
+            assertThat(
+                response.results().get(policyName).requiredCapacity().node().storage().getBytes(),
+                equalTo(used + LOW_WATERMARK_BYTES + ReactiveStorageDeciderService.NODE_DISK_OVERHEAD)
+            );
+        });
 
         assertAcked(
             client().admin()
@@ -430,7 +436,7 @@ public void testScaleDuringSplitOrClone() throws Exception {
         // * 2 since worst case is no hard links, see DiskThresholdDecider.getExpectedShardSize.
         long requiredSpaceForClone = used * 2 + LOW_WATERMARK_BYTES;
 
-        response = capacity();
+        GetAutoscalingCapacityAction.Response response = capacity();
         assertThat(response.results().keySet(), equalTo(Set.of(policyName)));
         assertThat(response.results().get(policyName).currentCapacity().total().storage().getBytes(), equalTo(enoughSpace * 2));
         // test that even when the shard cannot allocate due to disk space, we do not request a "total" scale up, only a node-level.