[PLAT-14944] Enable db scoped replication if runtime config is enabled

Summary:
Removed dependence on `dbScoped` in `DrConfigCreateForm` when hitting `POST    /customers/:cUUID/dr_configs` CREATE endpoint.

We now depend only on the runtime config `yb.xcluster.db_scoped.enabled` to determine whether to be db scoped when creating a new DR config. This runtime config is now at universe level. When the source universe's runtime config `yb.xcluster.db_scoped.creationEnabled` is set as true, we will try to create a db scoped replication.

Test Plan:
Added UTs to test the following:
1. `yb.xcluster.db_scoped.creationEnabled` = false globally. Then non-db scoped DR is created
2. `yb.xcluster.db_scoped.creationEnabled` = false on source universe and `yb.xcluster.db_scoped.creationEnabled`= true on target universe. Non-db scoped DR is created.
3. `yb.xcluster.db_scoped.creationEnabled` = true on source universe and `yb.xcluster.db_scoped.creationEnabled` = false on target universe. Db Scoped DR is created.

Local provider tests pass

Reviewers: jmak, hzare, sanketh

Reviewed By: hzare

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D37948

managed/RUNTIME-FLAGS.md

@@ -249,6 +249,7 @@
 | "Verify current cluster state (from db perspective) before running task" | "yb.task.verify_cluster_state" | "UNIVERSE" | "Verify current cluster state (from db perspective) before running task" | "Boolean" |
 | "Wait time for xcluster/DR replication setup and edit RPCs" | "yb.xcluster.operation_timeout" | "UNIVERSE" | "Wait time for xcluster/DR replication setup and edit RPCs." | "Duration" |
 | "Maximum timeout for xCluster bootstrap producer RPC call" | "yb.xcluster.bootstrap_producer_timeout" | "UNIVERSE" | "If the RPC call to create the bootstrap streams on the source universe does not return before this timeout, the task will retry with exponential backoff until it fails." | "Duration" |
+| "Flag to enable db scoped xCluster replication creation" | "yb.xcluster.db_scoped.creationEnabled" | "UNIVERSE" | "If flag is enabled, allows DR creation with db scoped xCluster replication" | "Boolean" |
 | "Leaderless tablets check enabled" | "yb.checks.leaderless_tablets.enabled" | "UNIVERSE" | " Whether to run CheckLeaderlessTablets subtask before running universe tasks" | "Boolean" |
 | "Leaderless tablets check timeout" | "yb.checks.leaderless_tablets.timeout" | "UNIVERSE" | "Controls the max time out when performing the CheckLeaderlessTablets subtask" | "Duration" |
 | "Enable Clock Sync check" | "yb.wait_for_clock_sync.enabled" | "UNIVERSE" | "Enable Clock Sync check" | "Boolean" |

managed/src/main/java/com/yugabyte/yw/common/config/GlobalConfKeys.java

@@ -785,14 +785,6 @@ public class GlobalConfKeys extends RuntimeConfigKeysModule {
           "It indicates whether creating disaster recovery configs are enabled",
           ConfDataType.BooleanType,
           ImmutableList.of(ConfKeyTags.PUBLIC));
-  public static final ConfKeyInfo<Boolean> dbScopedXClusterCreationEnabled =
-      new ConfKeyInfo<>(
-          "yb.xcluster.db_scoped.creationEnabled",
-          ScopeType.GLOBAL,
-          "Flag to enable db scoped xCluster replication creation",
-          "If flag is enabled, allows DR creation with db scoped xCluster replication",
-          ConfDataType.BooleanType,
-          ImmutableList.of(ConfKeyTags.INTERNAL));
   public static final ConfKeyInfo<Boolean> xclusterEnableAutoFlagValidation =
       new ConfKeyInfo<>(
           "yb.xcluster.enable_auto_flag_validation",

managed/src/main/java/com/yugabyte/yw/common/config/UniverseConfKeys.java

@@ -1115,7 +1115,14 @@ public class UniverseConfKeys extends RuntimeConfigKeysModule {
               + " fails.",
           ConfDataType.DurationType,
           ImmutableList.of(ConfKeyTags.PUBLIC));
-
+  public static final ConfKeyInfo<Boolean> dbScopedXClusterCreationEnabled =
+      new ConfKeyInfo<>(
+          "yb.xcluster.db_scoped.creationEnabled",
+          ScopeType.UNIVERSE,
+          "Flag to enable db scoped xCluster replication creation",
+          "If flag is enabled, allows DR creation with db scoped xCluster replication",
+          ConfDataType.BooleanType,
+          ImmutableList.of(ConfKeyTags.PUBLIC));
   public static final ConfKeyInfo<Boolean> leaderlessTabletsCheckEnabled =
       new ConfKeyInfo<>(
           "yb.checks.leaderless_tablets.enabled",

managed/src/main/java/com/yugabyte/yw/controllers/DrConfigController.java

@@ -183,16 +183,8 @@ public Result create(UUID customerUUID, Http.Request request) {
     }
 
     boolean isDbScoped =
-        confGetter.getGlobalConf(GlobalConfKeys.dbScopedXClusterCreationEnabled)
-            || createForm.dbScoped;
-    if (!confGetter.getGlobalConf(GlobalConfKeys.dbScopedXClusterCreationEnabled)
-        && createForm.dbScoped) {
-      throw new PlatformServiceException(
-          BAD_REQUEST,
-          "Support for db scoped disaster recovery configs is disabled in YBA. You may enable it "
-              + "by setting yb.xcluster.db_scoped.creationEnabled to true in the application.conf");
-    }
-
+        confGetter.getConfForScope(
+            sourceUniverse, UniverseConfKeys.dbScopedXClusterCreationEnabled);
     if (isDbScoped) {
       XClusterUtil.dbScopedXClusterPreChecks(sourceUniverse, targetUniverse);
     }

managed/src/main/java/com/yugabyte/yw/forms/DrConfigCreateForm.java

@@ -1,6 +1,5 @@
 package com.yugabyte.yw.forms;
 
-import com.yugabyte.yw.models.common.YbaApi;
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
 import java.util.Set;
@@ -49,12 +48,6 @@ public class DrConfigCreateForm {
   @ApiModelProperty("Run the pre-checks without actually running the subtasks")
   public boolean dryRun = false;
 
-  @ApiModelProperty(
-      value = "WARNING: This is a preview API that could change. Whether to enable db scoped DR",
-      hidden = true)
-  @YbaApi(visibility = YbaApi.YbaApiVisibility.PREVIEW, sinceYBAVersion = "2.23.0.0")
-  public Boolean dbScoped = false;
-
   @Valid
   @ApiModelProperty(
       "Parameters needed for creating PITR configs; if not specified, "

managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/DRDbScopedLocalTest.java

@@ -12,7 +12,7 @@
 import com.yugabyte.yw.common.PlacementInfoUtil;
 import com.yugabyte.yw.common.ReleaseManager;
 import com.yugabyte.yw.common.Util;
-import com.yugabyte.yw.common.config.GlobalConfKeys;
+import com.yugabyte.yw.common.config.UniverseConfKeys;
 import com.yugabyte.yw.common.gflags.SpecificGFlags;
 import com.yugabyte.yw.forms.DrConfigCreateForm;
 import com.yugabyte.yw.forms.DrConfigFailoverForm;
@@ -67,7 +67,7 @@ public void setupDrDbScoped() {
     runtimeConfService.setKey(
         customer.getUuid(),
         ScopedRuntimeConfig.GLOBAL_SCOPE_UUID,
-        GlobalConfKeys.dbScopedXClusterCreationEnabled.getKey(),
+        UniverseConfKeys.dbScopedXClusterCreationEnabled.getKey(),
         "true",
         true);
 
@@ -176,7 +176,6 @@ public void testDrDbScopedSetupWithBootstrap() throws InterruptedException {
     formData.sourceUniverseUUID = sourceUniverse.getUniverseUUID();
     formData.targetUniverseUUID = targetUniverse.getUniverseUUID();
     formData.name = "db-scoped-disaster-recovery-1";
-    formData.dbScoped = true;
     formData.dbs = new HashSet<String>();
     for (TableInfoForm.NamespaceInfoResp namespace : namespaceInfo) {
       if (namespaceNames.contains(namespace.name)) {
@@ -250,7 +249,6 @@ public void testDrDbScopedUpdate() throws InterruptedException {
     formData.sourceUniverseUUID = sourceUniverse.getUniverseUUID();
     formData.targetUniverseUUID = targetUniverse.getUniverseUUID();
     formData.name = "db-scoped-disaster-recovery-1";
-    formData.dbScoped = true;
     formData.dbs = new HashSet<String>();
     List<String> createNamespaceNames = Arrays.asList("dbnoncolocated");
     for (TableInfoForm.NamespaceInfoResp namespace : namespaceInfo) {
@@ -349,7 +347,6 @@ public void testDrDbScopedWithTLS() throws InterruptedException {
     formData.sourceUniverseUUID = sourceUniverse.getUniverseUUID();
     formData.targetUniverseUUID = targetUniverse.getUniverseUUID();
     formData.name = "db-scoped-disaster-recovery-1";
-    formData.dbScoped = true;
     formData.dbs = new HashSet<String>();
     for (TableInfoForm.NamespaceInfoResp namespace : namespaceInfo) {
       if (namespaceNames.contains(namespace.name)) {
@@ -723,7 +720,6 @@ private CreateDRMetadata defaultDbDRCreate(int numNodes, int replicationFactor)
     formData.sourceUniverseUUID = sourceUniverse.getUniverseUUID();
     formData.targetUniverseUUID = targetUniverse.getUniverseUUID();
     formData.name = "db-scoped-disaster-recovery-1";
-    formData.dbScoped = true;
     formData.dbs = new HashSet<String>();
     for (TableInfoForm.NamespaceInfoResp namespace : namespaceInfo) {
       if (namespaceNames.contains(namespace.name)) {

managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/DRLocalTest.java

@@ -11,12 +11,14 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.yugabyte.yw.common.ModelFactory;
 import com.yugabyte.yw.common.ShellResponse;
+import com.yugabyte.yw.common.config.UniverseConfKeys;
 import com.yugabyte.yw.common.gflags.SpecificGFlags;
 import com.yugabyte.yw.forms.DrConfigCreateForm;
 import com.yugabyte.yw.forms.TableInfoForm;
 import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
 import com.yugabyte.yw.forms.XClusterConfigCreateFormData;
 import com.yugabyte.yw.forms.XClusterConfigRestartFormData;
+import com.yugabyte.yw.models.ScopedRuntimeConfig;
 import com.yugabyte.yw.models.TaskInfo;
 import com.yugabyte.yw.models.Universe;
 import com.yugabyte.yw.models.configs.CustomerConfig;
@@ -26,13 +28,24 @@
 import java.util.List;
 import java.util.UUID;
 import lombok.extern.slf4j.Slf4j;
+import org.junit.Before;
 import org.junit.Test;
 import play.libs.Json;
 import play.mvc.Result;
 
 @Slf4j
 public class DRLocalTest extends DRLocalTestBase {
 
+  @Before
+  public void setupNonDbDr() {
+    runtimeConfService.setKey(
+        customer.getUuid(),
+        ScopedRuntimeConfig.GLOBAL_SCOPE_UUID,
+        UniverseConfKeys.dbScopedXClusterCreationEnabled.getKey(),
+        "false",
+        true /* isSuperAdmin */);
+  }
+
   @Test
   public void testDrConfigSetup() throws InterruptedException {
     UniverseDefinitionTaskParams.UserIntent userIntent =
@@ -61,7 +74,6 @@ public void testDrConfigSetup() throws InterruptedException {
     formData.sourceUniverseUUID = source.getUniverseUUID();
     formData.targetUniverseUUID = target.getUniverseUUID();
     formData.name = "DisasterRecovery-1";
-    formData.dbScoped = false;
     formData.dbs = new HashSet<>();
     for (TableInfoForm.NamespaceInfoResp namespaceInfo : resp) {
       if (namespaceInfo.name.equals("yugabyte")) {

managed/src/test/java/com/yugabyte/yw/controllers/DrConfigControllerTest.java

@@ -75,7 +75,9 @@
 import org.mockito.junit.MockitoJUnitRunner;
 import org.yb.CommonTypes;
 import org.yb.CommonTypes.TableType;
+import org.yb.Schema;
 import org.yb.client.GetMasterClusterConfigResponse;
+import org.yb.client.GetTableSchemaResponse;
 import org.yb.client.GetUniverseReplicationInfoResponse;
 import org.yb.client.ListTablesResponse;
 import org.yb.client.YBClient;
@@ -121,7 +123,7 @@ private CustomerConfig createData(Customer customer) {
     return CustomerConfig.createWithFormData(customer.getUuid(), formData);
   }
 
-  private DrConfigCreateForm createDefaultCreateForm(String name, Boolean dbScoped) {
+  private DrConfigCreateForm createDefaultCreateForm(String name) {
     DrConfigCreateForm createForm = new DrConfigCreateForm();
     createForm.name = name;
     createForm.sourceUniverseUUID = sourceUniverse.getUniverseUUID();
@@ -130,10 +132,6 @@ private DrConfigCreateForm createDefaultCreateForm(String name, Boolean dbScoped
     createForm.bootstrapParams = new RestartBootstrapParams();
     createForm.bootstrapParams.backupRequestParams = backupRequestParams;
 
-    if (dbScoped != null) {
-      createForm.dbScoped = dbScoped;
-    }
-
     List<MasterDdlOuterClass.ListTablesResponsePB.TableInfo> tableInfoList = new ArrayList<>();
     MasterDdlOuterClass.ListTablesResponsePB.TableInfo.Builder table1TableInfoBuilder =
         MasterDdlOuterClass.ListTablesResponsePB.TableInfo.newBuilder();
@@ -202,13 +200,59 @@ public void setUp() {
   }
 
   @Test
-  // Runtime config `yb.xcluster.db_scoped.creationEnabled` = true and db scoped parameter is passed
-  // in as true for request body.
-  public void testCreateDbScopedSuccess() {
+  // Runtime config `yb.xcluster.db_scoped.creationEnabled` = false.
+  public void testCreateSuccess() {
     settableRuntimeConfigFactory
         .globalRuntimeConf()
+        .setValue("yb.xcluster.db_scoped.creationEnabled", "false");
+    DrConfigCreateForm data = createDefaultCreateForm("txnDR");
+    UUID taskUUID = buildTaskInfo(null, TaskType.CreateDrConfig);
+    when(mockCommissioner.submit(any(), any())).thenReturn(taskUUID);
+    GetTableSchemaResponse mockTableSchemaResponseTable1 =
+        new GetTableSchemaResponse(
+            0,
+            "",
+            new Schema(Collections.emptyList()),
+            "db1",
+            "table_1",
+            "000030af000030008000000000004000",
+            null,
+            true,
+            CommonTypes.TableType.PGSQL_TABLE_TYPE,
+            Collections.emptyList(),
+            false);
+    try {
+      when(mockYBClient.getTableSchemaByUUID(any())).thenReturn(mockTableSchemaResponseTable1);
+    } catch (Exception ignored) {
+    }
+    Result result =
+        doRequestWithAuthTokenAndBody(
+            "POST",
+            "/api/customers/" + defaultCustomer.getUuid() + "/dr_configs",
+            authToken,
+            Json.toJson(data));
+
+    assertOk(result);
+    List<DrConfig> drConfigs =
+        DrConfig.getBetweenUniverses(
+            sourceUniverse.getUniverseUUID(), targetUniverse.getUniverseUUID());
+    assertEquals(1, drConfigs.size());
+    DrConfig drConfig = drConfigs.get(0);
+    assertNotNull(drConfig);
+    XClusterConfig xClusterConfig = drConfig.getActiveXClusterConfig();
+    assertEquals(xClusterConfig.getType(), ConfigType.Txn);
+  }
+
+  @Test
+  // Runtime config `yb.xcluster.db_scoped.creationEnabled` = true for source universe.
+  public void testCreateDbScopedSuccess() {
+    settableRuntimeConfigFactory
+        .forUniverse(sourceUniverse)
         .setValue("yb.xcluster.db_scoped.creationEnabled", "true");
-    DrConfigCreateForm data = createDefaultCreateForm("dbScopedDR", true);
+    settableRuntimeConfigFactory
+        .forUniverse(targetUniverse)
+        .setValue("yb.xcluster.db_scoped.creationEnabled", "false");
+    DrConfigCreateForm data = createDefaultCreateForm("dbScopedDR");
     UUID taskUUID = buildTaskInfo(null, TaskType.CreateDrConfig);
     when(mockCommissioner.submit(any(), any())).thenReturn(taskUUID);
     Result result =
@@ -230,12 +274,59 @@ public void testCreateDbScopedSuccess() {
   }
 
   @Test
-  // Runtime config `yb.xcluster.db_scoped.creationEnabled` = true with no parameter.
-  public void testSetDatabasesSuccess() {
+  // Only target universe has db scoped runtime config set as true.
+  public void testNonDbScopedCreate() {
+    settableRuntimeConfigFactory
+        .forUniverse(sourceUniverse)
+        .setValue("yb.xcluster.db_scoped.creationEnabled", "false");
+    settableRuntimeConfigFactory
+        .forUniverse(targetUniverse)
+        .setValue("yb.xcluster.db_scoped.creationEnabled", "true");
+    GetTableSchemaResponse mockTableSchemaResponseTable1 =
+        new GetTableSchemaResponse(
+            0,
+            "",
+            new Schema(Collections.emptyList()),
+            "db1",
+            "table_1",
+            "000030af000030008000000000004000",
+            null,
+            true,
+            CommonTypes.TableType.PGSQL_TABLE_TYPE,
+            Collections.emptyList(),
+            false);
+    try {
+      when(mockYBClient.getTableSchemaByUUID(any())).thenReturn(mockTableSchemaResponseTable1);
+    } catch (Exception ignored) {
+    }
+    DrConfigCreateForm data = createDefaultCreateForm("txnDR");
+    UUID taskUUID = buildTaskInfo(null, TaskType.CreateDrConfig);
+    when(mockCommissioner.submit(any(), any())).thenReturn(taskUUID);
+    Result result =
+        doRequestWithAuthTokenAndBody(
+            "POST",
+            "/api/customers/" + defaultCustomer.getUuid() + "/dr_configs",
+            authToken,
+            Json.toJson(data));
+
+    assertOk(result);
+    List<DrConfig> drConfigs =
+        DrConfig.getBetweenUniverses(
+            sourceUniverse.getUniverseUUID(), targetUniverse.getUniverseUUID());
+    assertEquals(1, drConfigs.size());
+    DrConfig drConfig = drConfigs.get(0);
+    assertNotNull(drConfig);
+    XClusterConfig xClusterConfig = drConfig.getActiveXClusterConfig();
+    assertEquals(xClusterConfig.getType(), ConfigType.Txn);
+  }
+
+  @Test
+  // Runtime config `yb.xcluster.db_scoped.creationEnabled` = true.
+  public void testSetDatabasesDbScopedSuccess() {
     settableRuntimeConfigFactory
         .globalRuntimeConf()
         .setValue("yb.xcluster.db_scoped.creationEnabled", "true");
-    DrConfigCreateForm data = createDefaultCreateForm("dbScopedDR", null);
+    DrConfigCreateForm data = createDefaultCreateForm("dbScopedDR");
     UUID taskUUID = buildTaskInfo(null, TaskType.CreateDrConfig);
     when(mockCommissioner.submit(any(), any())).thenReturn(taskUUID);
     Result result =
@@ -297,12 +388,12 @@ public void testSetDatabasesSuccess() {
   }
 
   @Test
-  // Runtime config `yb.xcluster.db_scoped.creationEnabled` = true with no parameter.
-  public void testSetDatabasesFailureNoChange() {
+  // Runtime config `yb.xcluster.db_scoped.creationEnabled` = true.
+  public void testSetDatabasesDbScopedFailureNoChange() {
     settableRuntimeConfigFactory
         .globalRuntimeConf()
         .setValue("yb.xcluster.db_scoped.creationEnabled", "true");
-    DrConfigCreateForm data = createDefaultCreateForm("dbScopedDR", null);
+    DrConfigCreateForm data = createDefaultCreateForm("dbScopedDR");
     UUID taskUUID = buildTaskInfo(null, TaskType.CreateDrConfig);
     when(mockCommissioner.submit(any(), any())).thenReturn(taskUUID);
     Result result =
@@ -347,12 +438,12 @@ public void testSetDatabasesFailureNoChange() {
   }
 
   @Test
-  // Runtime config `yb.xcluster.db_scoped.creationEnabled` = true with no parameter.
-  public void testSetDatabasesFailureNoDbs() {
+  // Runtime config `yb.xcluster.db_scoped.creationEnabled` = true.
+  public void testSetDatabasesDbScopedFailureNoDbs() {
     settableRuntimeConfigFactory
         .globalRuntimeConf()
         .setValue("yb.xcluster.db_scoped.creationEnabled", "true");
-    DrConfigCreateForm data = createDefaultCreateForm("dbScopedDR", null);
+    DrConfigCreateForm data = createDefaultCreateForm("dbScopedDR");
     UUID taskUUID = buildTaskInfo(null, TaskType.CreateDrConfig);
     when(mockCommissioner.submit(any(), any())).thenReturn(taskUUID);
     Result result =
@@ -396,28 +487,6 @@ public void testSetDatabasesFailureNoDbs() {
     assertThat(exception.getMessage(), containsString("error.required"));
   }
 
-  @Test
-  // Runtime config `yb.xcluster.db_scoped.creationEnabled` is disabled but db scoped parameter is
-  // passed in as true for request body.
-  public void testCreateDbScopedDisabledFailure() {
-    settableRuntimeConfigFactory
-        .globalRuntimeConf()
-        .setValue("yb.xcluster.db_scoped.creationEnabled", "false");
-    DrConfigCreateForm data = createDefaultCreateForm("dbScopedDR", true);
-    buildTaskInfo(null, TaskType.CreateDrConfig);
-
-    Result result =
-        assertPlatformException(
-            () ->
-                doRequestWithAuthTokenAndBody(
-                    "POST",
-                    "/api/customers/" + defaultCustomer.getUuid() + "/dr_configs",
-                    authToken,
-                    Json.toJson(data)));
-
-    assertBadRequest(result, "db scoped disaster recovery configs is disabled");
-  }
-
   private void setupMockGetUniverseReplicationInfo(
       DrConfig drConfig, String sourceNamespace, String targetNamespace) throws Exception {
     GetUniverseReplicationInfoResponse mockResponse =