limit the max value of cluster.max_shards_per_node to avoid int overflow

CHANGELOG.md

@@ -75,6 +75,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix NPE on restore searchable snapshot ([#13911](https://github.com/opensearch-project/OpenSearch/pull/13911))
 - Fix double invocation of postCollection when MultiBucketCollector is present ([#14015](https://github.com/opensearch-project/OpenSearch/pull/14015))
 - Java high-level REST client bulk() is not respecting the bulkRequest.requireAlias(true) method call ([#14146](https://github.com/opensearch-project/OpenSearch/pull/14146))
+- Fix the computed max shards of cluster to avoid int overflow ([#14155](https://github.com/opensearch-project/OpenSearch/pull/14155))
 
 ### Security
 

server/src/main/java/org/opensearch/indices/ShardLimitValidator.java

@@ -261,14 +261,15 @@ static Optional<String> checkShardLimit(
             return Optional.empty();
         }
 
+        int computedMaxShards = (int) Math.min(Integer.MAX_VALUE, (long) maxShardsPerNodeSetting * nodeCount);
         int maxShardsInCluster = maxShardsPerClusterSetting;
         if (maxShardsInCluster == -1) {
-            maxShardsInCluster = maxShardsPerNodeSetting * nodeCount;
+            maxShardsInCluster = computedMaxShards;
         } else {
-            maxShardsInCluster = Math.min(maxShardsInCluster, maxShardsPerNodeSetting * nodeCount);
+            maxShardsInCluster = Math.min(maxShardsInCluster, computedMaxShards);
         }
 
-        int currentOpenShards = state.getMetadata().getTotalOpenIndexShards();
+        long currentOpenShards = state.getMetadata().getTotalOpenIndexShards();
         if ((currentOpenShards + newShards) > maxShardsInCluster) {
             String errorMessage = "this action would add ["
                 + newShards

server/src/test/java/org/opensearch/indices/ShardLimitValidatorTests.java

@@ -214,6 +214,25 @@ public void testNonSystemIndexCreationFailsWithMaxShardLimitOnCluster() {
         );
     }
 
+    public void testComputedMaxShardsOfClusterIntOverFlow() {
+        final int maxShardLimitPerNode = 500_000_000;
+        ClusterState state = createClusterForShardLimitTest(15, 1, 1);
+        Optional<String> errorMessage = ShardLimitValidator.checkShardLimit(2, state, maxShardLimitPerNode, -1);
+        assertFalse(errorMessage.isPresent());
+
+        errorMessage = ShardLimitValidator.checkShardLimit(Integer.MAX_VALUE - 1, state, maxShardLimitPerNode, -1);
+        assertEquals(
+            "this action would add ["
+                + (Integer.MAX_VALUE - 1)
+                + "] total shards, but this cluster currently has ["
+                + 2
+                + "]/["
+                + Integer.MAX_VALUE
+                + "] maximum shards open",
+            errorMessage.get()
+        );
+    }
+
     public void testNonSystemIndexCreationPassesWithMaxShardLimitOnCluster() {
         final int maxShardLimitOnCluster = 5;
         Settings limitOnlySettings = Settings.builder()