[PLAT-9803] Allow AWS universe to use any AWS instance type if runtim…

…e conf is set

Summary:
1. Retrieve pricing for all instance types (this doesn't increase file sizes by much).
2. When a runtime conf is set, all AWS instance types from pricing data will be created in the provider.
3. Ephemeral instance types or not are decided based on having a 'd' in the instance type code, which is the AWS convention. Also added the non-d named types to the ephemeral list.

Test Plan:
1. Add a unit test and verify a couple of instance types there.
2. Create an AWS universe with c6id.large and verify it is created with local instance storage and SQL queries work.
3. Spot check a few ephemeral instance types.

Reviewers: svarshney, yshchetinin

Reviewed By: svarshney

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D37692

managed/src/main/java/com/yugabyte/yw/cloud/aws/AWSInitializer.java

@@ -39,6 +39,7 @@
 import com.yugabyte.yw.cloud.PublicCloudConstants.Architecture;
 import com.yugabyte.yw.common.PlatformServiceException;
 import com.yugabyte.yw.common.config.GlobalConfKeys;
+import com.yugabyte.yw.common.config.ProviderConfKeys;
 import com.yugabyte.yw.common.config.RuntimeConfGetter;
 import com.yugabyte.yw.models.InstanceType;
 import com.yugabyte.yw.models.InstanceType.InstanceTypeDetails;
@@ -52,12 +53,14 @@
 import java.util.Iterator;
 import java.util.Map;
 import java.util.UUID;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import play.Environment;
 import play.libs.Json;
 
+@Slf4j
 // TODO: move pricing data fetch to ybcloud.
 @Singleton
 public class AWSInitializer extends AbstractInitializer {
@@ -82,13 +85,16 @@ public void initialize(UUID customerUUID, UUID providerUUID) {
     Provider provider = Provider.get(customerUUID, providerUUID);
     InitializationContext context = new InitializationContext(provider);
 
-    LOG.info("Initializing AWS instance type and pricing info.");
+    LOG.info(
+        "Initializing AWS instance type and pricing info for regions {}",
+        Json.toJson(provider.getRegions()));
     LOG.info("This operation may take a few minutes...");
     // Get the price Json object stored locally at conf/aws_pricing.
     for (Region region : provider.getRegions()) {
       JsonNode regionJson = null;
 
       String pricingFileName = "aws_pricing/" + region.getCode() + ".tar.gz";
+
       try (InputStream pricingStream = environment.resourceAsStream(pricingFileName);
           GzipCompressorInputStream gzipStream = new GzipCompressorInputStream(pricingStream);
           TarArchiveInputStream regionStream = new TarArchiveInputStream(gzipStream)) {
@@ -553,6 +559,7 @@ private void storeInstanceTypeInfoToDB(InitializationContext context) {
       // 12 x 2000 HDD
       // 2 x 900 GB NVMe SSD
       String[] parts = productAttrs.get("storage").replaceAll(",", "").split(" ");
+      log.trace("Storage details for instance type {} are {}", instanceTypeCode, parts);
       if (parts.length < 4) {
         if (!productAttrs.get("storage").equals("EBS only")) {
           String msg =
@@ -570,19 +577,38 @@ private void storeInstanceTypeInfoToDB(InitializationContext context) {
           volumeType = VolumeType.EBS;
         }
       } else {
-        if (parts[1].equals("x")) {
+
+        if (parts[1].toLowerCase().equals("x")) {
           volumeCount = Integer.parseInt(parts[0]);
           volumeSizeGB = Integer.parseInt(parts[2]);
-          if (parts[3].equals("GB")) {
-            volumeType = VolumeType.valueOf(parts[4].toUpperCase());
-          } else {
-            volumeType = VolumeType.valueOf(parts[3].toUpperCase());
+          try {
+            if (parts[3].equals("GB")) {
+              volumeType = VolumeType.valueOf(parts[4].toUpperCase());
+            } else {
+              volumeType = VolumeType.valueOf(parts[3].toUpperCase());
+            }
+          } catch (IllegalArgumentException ilex) {
+            LOG.warn(
+                "Unexpected error parsing instance type {} and volume types {}, skipping",
+                instanceTypeCode,
+                parts,
+                ilex);
+            continue;
           }
 
         } else {
           volumeCount = 1;
           volumeSizeGB = Integer.parseInt(parts[0]);
-          volumeType = VolumeType.valueOf(parts[2].toUpperCase());
+          try {
+            volumeType = VolumeType.valueOf(parts[2].toUpperCase());
+          } catch (IllegalArgumentException ilex) {
+            LOG.warn(
+                "Unexpected error parsing instance type {} and volume types {}, skipping",
+                instanceTypeCode,
+                parts,
+                ilex);
+            continue;
+          }
         }
       }
 
@@ -656,6 +682,11 @@ private boolean matches(Map<String, String> objAttrs, String name, FilterOp op,
 
   private boolean isInstanceTypeSupported(
       Provider provider, Map<String, String> productAttributes) {
+
+    if (runtimeConfGetter.getConfForScope(provider, ProviderConfKeys.allowUnsupportedInstances)) {
+      return true;
+    }
+
     return InstanceType.getAWSInstancePrefixesSupported(provider, runtimeConfGetter).stream()
         .anyMatch(productAttributes.getOrDefault("instanceType", "")::startsWith);
   }

managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/ManipulateDnsRecordTask.java

@@ -64,13 +64,21 @@ public void run() {
               nodeIpCsv)
           .processErrors();
     } catch (Exception e) {
-      if (taskParams().type != DnsManager.DnsCommandType.Delete || !taskParams().isForceDelete) {
+      if (taskParams().type != DnsManager.DnsCommandType.Delete) {
         throw e;
-      } else {
+      }
+      if (taskParams().isForceDelete) {
         log.info(
             "Ignoring error in dns record deletion for {} due to isForceDelete being set.",
             taskParams().domainNamePrefix,
             e);
+      } else if (e.getMessage().toLowerCase().contains("not found")) {
+        log.info(
+            "Ignoring error in dns record deletion for {} as it may have already been deleted.",
+            taskParams().domainNamePrefix,
+            e);
+      } else {
+        throw e;
       }
     }
   }

managed/src/main/java/com/yugabyte/yw/forms/UniverseDefinitionTaskParams.java

@@ -78,7 +78,9 @@
 public class UniverseDefinitionTaskParams extends UniverseTaskParams {
 
   private static final Set<String> AWS_INSTANCE_WITH_EPHEMERAL_STORAGE_ONLY =
-      ImmutableSet.of("i3.", "c5d.", "c6gd.");
+      ImmutableSet.of(
+          "g5.", "g6.", "g6e.", "gr6.", "i3.", "i3en.", "i4g.", "i4i.", "im4gn.", "is4gen.", "p5.",
+          "p5e.", "trn1.", "trn1n.", "x1.", "x1e.");
 
   public static final String UPDATING_TASK_UUID_FIELD = "updatingTaskUUID";
   public static final String PLACEMENT_MODIFICATION_TASK_UUID_FIELD =
@@ -560,6 +562,13 @@ public static boolean hasEphemeralStorage(
         if (instanceType.startsWith(prefix)) {
           return true;
         }
+        String[] typeParts = instanceType.split("\\.");
+        // AWS has a convention of using 'd' in the instance type for most local storage
+        // instance types. Ideally, we can figure this out from the pricing data instead
+        // so we don't have to hack this way.
+        if (typeParts.length >= 1 && typeParts[0].contains("d")) {
+          return true;
+        }
       }
       return false;
     } else if (providerType == CloudType.gcp) {

managed/src/main/resources/application.test.conf

@@ -2,7 +2,7 @@ include classpath("application.common.conf")
 
 db {
   default.driver="org.h2.Driver"
-  default.url="jdbc:h2:mem:play-test;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1"
+  default.url="jdbc:h2:mem:play-test;USER=sa;PASSWORD=123;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1"
   default.migration.locations=["common","h2"]
   perf_advisor.driver="org.h2.Driver"
   perf_advisor.url="jdbc:h2:mem:play-test;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1"
@@ -65,6 +65,12 @@ yb {
     # Disabling for now, will add tests later.
     verify_cluster_state = false
   }
+
+  test {
+    # Log all test output to the file defined in logback-test.xml
+    logtofile = false
+  }
+
 }
 
 ybc {
@@ -81,6 +87,7 @@ runtime_config{
   data_validation.enabled = true
 }
 
+
 # Only used by sbt swaggerGen
 swagger.filter = "com.yugabyte.yw.common.swagger.PlatformSwaggerSpecFilter"
 api.version = "v1"

managed/src/main/resources/aws_pricing/version_metadata.json

@@ -1,3 +1,3 @@
 {
-    "date": "2024-06-12"
+    "date": "2024-09-22"
 }

managed/src/main/resources/reference.conf

@@ -1313,6 +1313,10 @@ yb {
     enabled = true
     enabled = ${?ENV_PROXY_SELECTOR_ENABLED}
   }
+
+  test {
+    logtofile = false
+  }
 }
 
 ybc {

managed/src/test/java/com/yugabyte/yw/cloud/aws/AWSInitializerTest.java

@@ -0,0 +1,106 @@
+package com.yugabyte.yw.cloud.aws;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.yugabyte.yw.commissioner.Common.CloudType;
+import com.yugabyte.yw.common.FakeDBApplication;
+import com.yugabyte.yw.common.ModelFactory;
+import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
+import com.yugabyte.yw.models.AvailabilityZone;
+import com.yugabyte.yw.models.Customer;
+import com.yugabyte.yw.models.InstanceType;
+import com.yugabyte.yw.models.Provider;
+import com.yugabyte.yw.models.ProviderDetails;
+import com.yugabyte.yw.models.ProviderDetails.CloudInfo;
+import com.yugabyte.yw.models.Region;
+import com.yugabyte.yw.models.RuntimeConfigEntry;
+import com.yugabyte.yw.models.helpers.provider.AWSCloudInfo;
+import java.util.Arrays;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Before;
+import org.junit.Test;
+
+@Slf4j
+public class AWSInitializerTest extends FakeDBApplication {
+
+  private Customer customer;
+  private Provider defaultProvider;
+  private Region defaultRegion;
+  AWSInitializer awsInitializer;
+
+  @Before
+  public void setup() {
+    customer = ModelFactory.testCustomer();
+    defaultProvider = ModelFactory.awsProvider(customer);
+    defaultRegion = new Region();
+    defaultRegion.setProvider(defaultProvider);
+    defaultRegion.setCode("us-east-2");
+    defaultRegion.setName("us-east-2");
+    AvailabilityZone az = new AvailabilityZone();
+    az.setCode("us-east-2a");
+    az.setName("us-east-2a");
+    defaultRegion.setZones(Arrays.asList(az));
+    defaultProvider.getRegions().add(defaultRegion);
+    ProviderDetails providerDetails = new ProviderDetails();
+    CloudInfo cloudInfo = new CloudInfo();
+    cloudInfo.aws = new AWSCloudInfo();
+    cloudInfo.aws.setAwsAccessKeyID("accessKey");
+    cloudInfo.aws.setAwsAccessKeySecret("accessKeySecret");
+    providerDetails.setCloudInfo(cloudInfo);
+    defaultProvider.setDetails(providerDetails);
+    defaultProvider.save();
+  }
+
+  @Test
+  public void testAWSInitializerAllInstanceTypes() {
+
+    assertTrue(
+        UniverseDefinitionTaskParams.hasEphemeralStorage(CloudType.aws, "x2gd.12xlarge", null));
+    assertTrue(
+        UniverseDefinitionTaskParams.hasEphemeralStorage(CloudType.aws, "i3en.2xlarge", null));
+
+    RuntimeConfigEntry.upsertGlobal("yb.internal.allow_unsupported_instances", "true");
+    awsInitializer = app.injector().instanceOf(AWSInitializer.class);
+    awsInitializer.initialize(customer.getUuid(), defaultProvider.getUuid());
+
+    InstanceType c6a4x = InstanceType.get(defaultProvider.getUuid(), "c6a.4xlarge");
+    assertEquals(16, (int) c6a4x.getNumCores().doubleValue());
+    assertEquals(32, (int) c6a4x.getMemSizeGB().doubleValue());
+
+    InstanceType c6id4x = InstanceType.get(defaultProvider.getUuid(), "c6id.4xlarge");
+    assertEquals(16, (int) c6id4x.getNumCores().doubleValue());
+    assertEquals(32, (int) c6id4x.getMemSizeGB().doubleValue());
+    assertEquals(1, c6id4x.getInstanceTypeDetails().volumeDetailsList.size());
+    var volDetailsList = c6id4x.getInstanceTypeDetails().volumeDetailsList.get(0);
+    assertEquals(volDetailsList.volumeType, InstanceType.VolumeType.NVME);
+    assertEquals(950, volDetailsList.volumeSizeGB.intValue());
+
+    InstanceType g612x = InstanceType.get(defaultProvider.getUuid(), "g6.12xlarge");
+    assertEquals(48, (int) g612x.getNumCores().doubleValue());
+    assertEquals(192, (int) g612x.getMemSizeGB().doubleValue());
+    assertEquals(4, g612x.getInstanceTypeDetails().volumeDetailsList.size());
+    volDetailsList = g612x.getInstanceTypeDetails().volumeDetailsList.get(0);
+    assertEquals(volDetailsList.volumeType, InstanceType.VolumeType.NVME);
+    assertEquals(940, volDetailsList.volumeSizeGB.intValue());
+  }
+
+  @Test
+  public void testAWSInitializerAllowedInstanceTypes() {
+    RuntimeConfigEntry.upsertGlobal("yb.internal.allow_unsupported_instances", "false");
+    awsInitializer = app.injector().instanceOf(AWSInitializer.class);
+    awsInitializer.initialize(customer.getUuid(), defaultProvider.getUuid());
+
+    InstanceType c6a4x = InstanceType.get(defaultProvider.getUuid(), "c6a.4xlarge");
+    assertEquals(16, (int) c6a4x.getNumCores().doubleValue());
+    assertEquals(32, (int) c6a4x.getMemSizeGB().doubleValue());
+
+    InstanceType c6id4x = null;
+    try {
+      c6id4x = InstanceType.get(defaultProvider.getUuid(), "c6id.4xlarge");
+    } catch (Exception ex) {
+      // ignore
+    }
+    assertEquals(null, c6id4x);
+  }
+}

managed/src/test/resources/logback-test.xml

@@ -2,6 +2,8 @@
 <configuration>
 
   <property name="cloud" value="${yb.cloud.enabled}"/>
+  <property name="LOGTESTTOFILE" value="${yb.test.logtofile:-false}"/>
+
   <conversionRule conversionWord="coloredLevel" converterClass="play.api.libs.logback.ColoredLevel" />
 
 
@@ -53,6 +55,20 @@
           <level>${APPLICATION_CONSOLE_LOG_LEVEL:-${applicationLogLevel:-DEBUG}}</level>
         </filter>
       </appender>
+
+    <if condition='property("LOGTESTTOFILE").equals("true")'>
+    <then>
+      <appender name="FILE" class="ch.qos.logback.core.FileAppender">
+        <encoder>
+          <pattern>%d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC} %X{logType} %-5coloredLevel %X{correlation-id} %F:%L [%thread] %logger %msg%n</pattern>
+        </encoder>
+        <file>/tmp/yw.test.log</file>
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+          <level>${APPLICATION_CONSOLE_LOG_LEVEL:-${applicationLogLevel:-DEBUG}}</level>
+        </filter>
+      </appender>
+    </then>
+    </if>
     </else>
   </if>
 
@@ -61,6 +77,15 @@
     <includeCallerData>true</includeCallerData>
   </appender>
 
+<if condition='property("LOGTESTTOFILE").equals("true")'>
+    <then>
+  <appender name="ASYNCFILE" class="ch.qos.logback.classic.AsyncAppender">
+    <appender-ref ref="FILE" />
+    <includeCallerData>true</includeCallerData>
+  </appender>
+</then>
+</if>
+
   <logger name="play" level="INFO" />
   <logger name="org.yb" level="INFO" />
   <logger name="db.migration" level="INFO" />
@@ -73,8 +98,20 @@
 
   <logger name="oshi.software.os.linux.LinuxFileSystem" level="ERROR" />
 
+
+    <if condition='property("LOGTESTTOFILE").equals("true")'>
+    <then>
   <root level="WARN">
     <appender-ref ref="ASYNCSTDOUT" />
-  </root>
+    <appender-ref ref="ASYNCFILE" />
+     </root>
+</then>
+<else>
+  <root level="WARN">
+    <appender-ref ref="ASYNCSTDOUT" />
+     </root>
+</else>
+</if>
+
 
 </configuration>

managed/ui/src/components/universes/UniverseDetail/UniverseDetail.js

@@ -91,10 +91,13 @@ import ConnectionPoolDisabled from '../../../redesign/assets/connection-pool-dis
 
 import './UniverseDetail.scss';
 
-const INSTANCE_WITH_EPHEMERAL_STORAGE_ONLY = ['i3', 'c5d', 'c6gd'];
+const INSTANCE_WITH_EPHEMERAL_STORAGE_ONLY = ['g5','g6','g6e',
+  'gr6','i3','i3en','i4g','i4i','im4gn',
+  'is4gen','p5','p5e','trn1','trn1n','x1','x1e'];
 
 export const isEphemeralAwsStorageInstance = (instanceType) => {
-  return INSTANCE_WITH_EPHEMERAL_STORAGE_ONLY.includes(instanceType?.split?.('.')[0]);
+  return INSTANCE_WITH_EPHEMERAL_STORAGE_ONLY.includes(instanceType?.split?.('.')[0]) || 
+    instanceType?.split?.('.')[0].includes('d');
 };
 
 class UniverseDetail extends Component {