From 817f0fcb1a7c9ccb9b4b780a9073cb5c77eb938a Mon Sep 17 00:00:00 2001 From: Nisarg Thakkar Date: Tue, 26 Mar 2024 12:22:21 -0700 Subject: [PATCH] [controller] Improve update store workflow and reduce Zk metadata updates --- .../java/com/linkedin/venice/AdminTool.java | 44 +- .../java/com/linkedin/venice/Command.java | 10 - clients/venice-pulsar/readme.md | 2 +- docker/venice-client/create-store.sh | 4 +- .../com/linkedin/venice/VeniceConstants.java | 4 - .../java/com/linkedin/venice/ConfigKeys.java | 83 +- .../controllerapi/ControllerApiConstants.java | 2 +- .../controllerapi/ControllerClient.java | 14 - .../venice/controllerapi/ControllerRoute.java | 3 - .../helix/ParentHelixOfflinePushAccessor.java | 33 +- .../helix/StoragePersonaRepository.java | 6 +- .../linkedin/venice/meta/BackupStrategy.java | 6 +- .../venice/meta/HybridStoreConfig.java | 5 + .../venice/meta/HybridStoreConfigImpl.java | 2 + .../venice/meta/PartitionerConfigImpl.java | 5 +- .../com/linkedin/venice/meta/ZKStore.java | 10 +- .../linkedin/venice/utils/PartitionUtils.java | 4 +- .../java/com/linkedin/venice/utils/Utils.java | 6 +- .../venice/meta/BackupStrategyTest.java | 23 + .../venice/meta/TestHybridStoreConfig.java | 22 + .../com/linkedin/venice/utils/UtilsTest.java | 12 + ...LevelConfigForActiveActiveReplication.java | 274 +--- ...lusterLevelConfigForNativeReplication.java | 134 +- .../venice/controller/TestFabricBuildout.java | 12 +- .../controller/TestIncrementalPush.java | 3 +- ...stParentControllerWithMultiDataCenter.java | 6 +- ...VeniceHelixAdminWithSharedEnvironment.java | 74 +- .../VeniceParentHelixAdminTest.java | 78 +- .../AdminConsumptionTaskIntegrationTest.java | 119 +- .../server/AbstractTestAdminSparkServer.java | 59 +- .../server/TestAdminSparkServer.java | 389 +++-- .../TestAdminSparkServerWithMultiServers.java | 26 +- .../ActiveActiveReplicationForHybridTest.java | 23 +- .../venice/endToEnd/MetaSystemStoreTest.java | 181 +-- .../venice/endToEnd/PartialUpdateTest.java | 5 +- .../venice/endToEnd/ParticipantStoreTest.java | 104 +- .../venice/endToEnd/PushJobDetailsTest.java | 2 +- .../PushStatusStoreMultiColoTest.java | 6 +- .../venice/endToEnd/PushStatusStoreTest.java | 6 +- .../venice/endToEnd/StoragePersonaTest.java | 25 +- ...TestActiveActiveReplicationForIncPush.java | 32 +- .../linkedin/venice/endToEnd/TestBatch.java | 66 +- .../linkedin/venice/endToEnd/TestHybrid.java | 272 ---- .../endToEnd/TestHybridMultiRegion.java | 355 +++++ .../endToEnd/TestHybridStoreDeletion.java | 156 +- ...tialUpdateWithActiveActiveReplication.java | 6 - .../TestPushJobWithNativeReplication.java | 176 +-- ...tPushJobWithSourceGridFabricSelection.java | 13 +- .../endToEnd/TestStaleDataVisibility.java | 53 +- .../venice/endToEnd/TestStoreMigration.java | 28 +- .../TestStoreUpdateStoragePersona.java | 39 +- .../endToEnd/TestWritePathComputation.java | 109 +- ...entIndividualFeatureConfigurationTest.java | 19 +- .../utils/AbstractClientEndToEndSetup.java | 5 +- .../venice/hadoop/TestVenicePushJob.java | 27 +- .../IngestionHeartBeatTest.java | 2 +- .../integration/utils/ServiceFactory.java | 80 +- .../utils/VeniceClusterCreateOptions.java | 12 + .../utils/VeniceClusterWrapper.java | 1 + .../utils/VeniceControllerCreateOptions.java | 12 + .../utils/VeniceControllerWrapper.java | 8 +- .../VeniceMultiClusterCreateOptions.java | 55 +- .../utils/VeniceMultiClusterWrapper.java | 11 +- ...VeniceMultiRegionClusterCreateOptions.java | 256 ++++ ...woLayerMultiRegionMultiClusterWrapper.java | 108 +- .../kafka/ssl/AdminChannelWithSSLTest.java | 48 +- .../TestMetadataOperationInMultiCluster.java | 6 +- ...tingSstFilesWithActiveActiveIngestion.java | 7 - .../utils/IntegrationTestPushUtils.java | 16 +- .../com/linkedin/venice/controller/Admin.java | 78 +- .../venice/controller/StoreViewUtils.java | 29 +- .../VeniceControllerClusterConfig.java | 137 +- .../controller/VeniceControllerConfig.java | 176 +-- .../VeniceControllerMultiClusterConfig.java | 32 +- .../controller/VeniceControllerService.java | 29 +- .../venice/controller/VeniceHelixAdmin.java | 1228 ++------------- .../controller/VeniceParentHelixAdmin.java | 1327 ++++++----------- .../SystemSchemaInitializationRoutine.java | 42 +- .../init/SystemStoreInitializationHelper.java | 23 +- .../kafka/consumer/AdminConsumptionTask.java | 2 + .../kafka/consumer/AdminExecutionTask.java | 22 +- .../controller/server/AdminSparkServer.java | 4 - .../controller/server/CreateVersion.java | 1 - .../controller/server/StoresRoutes.java | 45 +- .../venice/controller/util/AdminUtils.java | 73 + .../ParentControllerConfigUpdateUtils.java | 171 --- .../PrimaryControllerConfigUpdateUtils.java | 170 +++ .../controller/util/UpdateStoreUtils.java | 1134 ++++++++++++++ .../controller/util/UpdateStoreWrapper.java | 18 + .../AbstractTestVeniceParentHelixAdmin.java | 11 +- .../TestVeniceHelixAdminWithoutCluster.java | 56 - .../TestVeniceParentHelixAdmin.java | 186 +-- .../SystemStoreInitializationHelperTest.java | 3 + .../consumer/TestAdminConsumerService.java | 2 - .../controller/util/AdminUtilsTest.java | 134 ++ ...rimaryControllerConfigUpdateUtilsTest.java | 154 ++ .../controller/util/UpdateStoreUtilsTest.java | 1041 +++++++++++++ ...ParentControllerConfigUpdateUtilsTest.java | 308 ---- .../pulsar/sink/PulsarVeniceSinkTest.java | 3 +- 99 files changed, 5416 insertions(+), 5027 deletions(-) create mode 100644 internal/venice-common/src/test/java/com/linkedin/venice/meta/BackupStrategyTest.java create mode 100644 internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java create mode 100644 internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java create mode 100644 services/venice-controller/src/main/java/com/linkedin/venice/controller/util/AdminUtils.java delete mode 100644 services/venice-controller/src/main/java/com/linkedin/venice/controller/util/ParentControllerConfigUpdateUtils.java create mode 100644 services/venice-controller/src/main/java/com/linkedin/venice/controller/util/PrimaryControllerConfigUpdateUtils.java create mode 100644 services/venice-controller/src/main/java/com/linkedin/venice/controller/util/UpdateStoreUtils.java create mode 100644 services/venice-controller/src/main/java/com/linkedin/venice/controller/util/UpdateStoreWrapper.java create mode 100644 services/venice-controller/src/test/java/com/linkedin/venice/controller/util/AdminUtilsTest.java create mode 100644 services/venice-controller/src/test/java/com/linkedin/venice/controller/util/PrimaryControllerConfigUpdateUtilsTest.java create mode 100644 services/venice-controller/src/test/java/com/linkedin/venice/controller/util/UpdateStoreUtilsTest.java delete mode 100644 services/venice-controller/src/test/java/com/linkedin/venice/controller/utils/ParentControllerConfigUpdateUtilsTest.java diff --git a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java index cf73a1f5c3..4f3092b56a 100644 --- a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java +++ b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java @@ -471,12 +471,6 @@ public static void main(String[] args) throws Exception { case REMOVE_FROM_STORE_ACL: removeFromStoreAcl(cmd); break; - case ENABLE_NATIVE_REPLICATION_FOR_CLUSTER: - enableNativeReplicationForCluster(cmd); - break; - case DISABLE_NATIVE_REPLICATION_FOR_CLUSTER: - disableNativeReplicationForCluster(cmd); - break; case ENABLE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER: enableActiveActiveReplicationForCluster(cmd); break; @@ -1115,7 +1109,11 @@ static UpdateStoreQueryParams getUpdateStoreQueryParams(CommandLine cmd) { booleanParam(cmd, Arg.RMD_CHUNKING_ENABLED, p -> params.setRmdChunkingEnabled(p), argSet); integerParam(cmd, Arg.BATCH_GET_LIMIT, p -> params.setBatchGetLimit(p), argSet); integerParam(cmd, Arg.NUM_VERSIONS_TO_PRESERVE, p -> params.setNumVersionsToPreserve(p), argSet); - booleanParam(cmd, Arg.INCREMENTAL_PUSH_ENABLED, p -> params.setIncrementalPushEnabled(p), argSet); + booleanParam(cmd, Arg.INCREMENTAL_PUSH_ENABLED, p -> { + System.out.println( + "Setting incremental push config is deprecated. Please set the appropriate source-of-truth configs."); + params.setIncrementalPushEnabled(p); + }, argSet); booleanParam(cmd, Arg.WRITE_COMPUTATION_ENABLED, p -> params.setWriteComputationEnabled(p), argSet); booleanParam(cmd, Arg.READ_COMPUTATION_ENABLED, p -> params.setReadComputationEnabled(p), argSet); integerParam( @@ -1807,7 +1805,7 @@ public static void checkMigrationStatus( printSystemStoreMigrationStatus(destControllerClient, storeName, printFunction); } else { // This is a parent controller - System.err.println("\n=================== Parent Controllers ===================="); + printFunction.apply("\n=================== Parent Controllers ===================="); printMigrationStatus(srcControllerClient, storeName, printFunction); printMigrationStatus(destControllerClient, storeName, printFunction); @@ -1818,7 +1816,7 @@ public static void checkMigrationStatus( Map destChildControllerClientMap = getControllerClientMap(destClusterName, response); for (Map.Entry entry: srcChildControllerClientMap.entrySet()) { - System.err.println("\n\n=================== Child Datacenter " + entry.getKey() + " ===================="); + printFunction.apply("\n\n=================== Child Datacenter " + entry.getKey() + " ===================="); ControllerClient srcChildController = entry.getValue(); ControllerClient destChildController = destChildControllerClientMap.get(entry.getKey()); @@ -2583,34 +2581,6 @@ private static void removeFromStoreAcl(CommandLine cmd) throws Exception { } } - private static void enableNativeReplicationForCluster(CommandLine cmd) { - String storeType = getRequiredArgument(cmd, Arg.STORE_TYPE); - String sourceRegionParam = getOptionalArgument(cmd, Arg.NATIVE_REPLICATION_SOURCE_FABRIC); - Optional sourceRegion = - StringUtils.isEmpty(sourceRegionParam) ? Optional.empty() : Optional.of(sourceRegionParam); - String regionsFilterParam = getOptionalArgument(cmd, Arg.REGIONS_FILTER); - Optional regionsFilter = - StringUtils.isEmpty(regionsFilterParam) ? Optional.empty() : Optional.of(regionsFilterParam); - - ControllerResponse response = - controllerClient.configureNativeReplicationForCluster(true, storeType, sourceRegion, regionsFilter); - printObject(response); - } - - private static void disableNativeReplicationForCluster(CommandLine cmd) { - String storeType = getRequiredArgument(cmd, Arg.STORE_TYPE); - String sourceFabricParam = getOptionalArgument(cmd, Arg.NATIVE_REPLICATION_SOURCE_FABRIC); - Optional sourceFabric = - StringUtils.isEmpty(sourceFabricParam) ? Optional.empty() : Optional.of(sourceFabricParam); - String regionsFilterParam = getOptionalArgument(cmd, Arg.REGIONS_FILTER); - Optional regionsFilter = - StringUtils.isEmpty(regionsFilterParam) ? Optional.empty() : Optional.of(regionsFilterParam); - - ControllerResponse response = - controllerClient.configureNativeReplicationForCluster(false, storeType, sourceFabric, regionsFilter); - printObject(response); - } - private static void enableActiveActiveReplicationForCluster(CommandLine cmd) { String storeType = getRequiredArgument(cmd, Arg.STORE_TYPE); String regionsFilterParam = getOptionalArgument(cmd, Arg.REGIONS_FILTER); diff --git a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java index 69900a996c..a352a619fb 100644 --- a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java +++ b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/Command.java @@ -384,16 +384,6 @@ public enum Command { "remove-from-store-acl", "Remove a principal from ACL's for an existing store", new Arg[] { URL, STORE, PRINCIPAL }, new Arg[] { CLUSTER, READABILITY, WRITEABILITY } ), - ENABLE_NATIVE_REPLICATION_FOR_CLUSTER( - "enable-native-replication-for-cluster", - "enable native replication for certain stores (batch-only, hybrid-only, incremental-push, hybrid-or-incremental, all) in a cluster", - new Arg[] { URL, STORE_TYPE }, new Arg[] { CLUSTER, REGIONS_FILTER, NATIVE_REPLICATION_SOURCE_FABRIC } - ), - DISABLE_NATIVE_REPLICATION_FOR_CLUSTER( - "disable-native-replication-for-cluster", - "disable native replication for certain stores (batch-only, hybrid-only, incremental-push, hybrid-or-incremental, all) in a cluster", - new Arg[] { URL, CLUSTER, STORE_TYPE }, new Arg[] { REGIONS_FILTER, NATIVE_REPLICATION_SOURCE_FABRIC } - ), ENABLE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER( "enable-active-active-replication-for-cluster", "enable active active replication for certain stores (batch-only, hybrid-only, incremental-push, hybrid-or-incremental, all) in a cluster", diff --git a/clients/venice-pulsar/readme.md b/clients/venice-pulsar/readme.md index 6a44dc5d37..47696da4b8 100644 --- a/clients/venice-pulsar/readme.md +++ b/clients/venice-pulsar/readme.md @@ -31,7 +31,7 @@ cd ~/src/venice ```shell java -jar bin/venice-admin-tool-all.jar --new-store --url http://venice-controller:5555 --cluster venice-cluster0 --store t1_n1_s1 --key-schema-file /tmp/key.asvc --value-schema-file /tmp/value.asvc - java -jar bin/venice-admin-tool-all.jar --update-store --url http://venice-controller:5555 --cluster venice-cluster0 --store t1_n1_s1 --storage-quota -1 --incremental-push-enabled true + java -jar bin/venice-admin-tool-all.jar --update-store --url http://venice-controller:5555 --cluster venice-cluster0 --store t1_n1_s1 --storage-quota -1 --hybrid-rewind-seconds 86400 --hybrid-offset-lag 1000 --hybrid-data-replication-policy NONE java -jar bin/venice-admin-tool-all.jar --update-store --url http://venice-controller:5555 --cluster venice-cluster0 --store t1_n1_s1 --read-quota 1000000 java -jar bin/venice-admin-tool-all.jar --empty-push --url http://venice-controller:5555 --cluster venice-cluster0 --store t1_n1_s1 --push-id init --store-size 1000 diff --git a/docker/venice-client/create-store.sh b/docker/venice-client/create-store.sh index 009b2c4823..a0530fee33 100755 --- a/docker/venice-client/create-store.sh +++ b/docker/venice-client/create-store.sh @@ -10,5 +10,5 @@ jar=/opt/venice/bin/venice-admin-tool-all.jar # create store java -jar $jar --new-store --url $url --cluster $clusterName --store $storeName --key-schema-file $keySchema --value-schema-file $valueSchema -# update quota and enabled incremental push -java -jar $jar --update-store --url $url --cluster $clusterName --store $storeName --storage-quota -1 --incremental-push-enabled true +# update quota and enable hybrid to allow incremental push +java -jar $jar --update-store --url $url --cluster $clusterName --store $storeName --storage-quota -1 --hybrid-rewind-seconds 86400 --hybrid-offset-lag 1000 --hybrid-data-replication-policy NONE diff --git a/internal/venice-client-common/src/main/java/com/linkedin/venice/VeniceConstants.java b/internal/venice-client-common/src/main/java/com/linkedin/venice/VeniceConstants.java index 66de8f7272..474cc0d391 100644 --- a/internal/venice-client-common/src/main/java/com/linkedin/venice/VeniceConstants.java +++ b/internal/venice-client-common/src/main/java/com/linkedin/venice/VeniceConstants.java @@ -68,10 +68,6 @@ public class VeniceConstants { public static final String SYSTEM_PROPERTY_FOR_APP_RUNNING_REGION = "com.linkedin.app.env"; - // public static final String TIMESTAMP_FIELD_NAME = "timestamp"; // - // - // public static final String REPLICATION_CHECKPOINT_VECTOR_FIELD = "replication_checkpoint_vector"; - /** * This is a sentinel value to be used in TopicSwitch message rewindStartTimestamp field between controller and server. * When controller specifies this, Leader server nodes will calculate the rewind start time itself. diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java index 560220bb93..e60269e777 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/ConfigKeys.java @@ -144,39 +144,6 @@ private ConfigKeys() { */ public static final String KAFKA_REPLICATION_FACTOR_RT_TOPICS = "kafka.replication.factor.rt.topics"; - /** - * TODO: the following 3 configs will be deprecated after the native replication migration is changed to a two-step - * process: 1. Turn on the cluster level config that takes care of newly created stores; 2. Run admin command - * to convert existing stores to native replication. - */ - /** - * Cluster-level config to enable native replication for all batch-only stores. - */ - public static final String ENABLE_NATIVE_REPLICATION_FOR_BATCH_ONLY = "enable.native.replication.for.batch.only"; - - /** - * Cluster-level config to enable native replication for all hybrid stores. - */ - public static final String ENABLE_NATIVE_REPLICATION_FOR_HYBRID = "enable.native.replication.for.hybrid"; - - /** - * Cluster-level config to enable native replication for new batch-only stores. - */ - public static final String ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY = - "enable.native.replication.as.default.for.batch.only"; - - /** - * Cluster-level config to enable native replication for new hybrid stores. - */ - public static final String ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID = - "enable.native.replication.as.default.for.hybrid"; - - /** - * Cluster-level config to enable active-active replication for new batch-only stores. - */ - public static final String ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY_STORE = - "enable.active.active.replication.as.default.for.batch.only.store"; - /** * Cluster-level config to enable active-active replication for new hybrid stores. */ @@ -189,7 +156,7 @@ private ConfigKeys() { public static final String ENABLE_BLOB_TRANSFER = "enable.blob.transfer"; /** - * Sets the default for whether or not do schema validation for all stores + * Sets the default for whether to do schema validation or not for all stores */ public static final String CONTROLLER_SCHEMA_VALIDATION_ENABLED = "controller.schema.validation.enabled"; @@ -304,9 +271,9 @@ private ConfigKeys() { public static final String CONTROLLER_ENFORCE_SSL = "controller.enforce.ssl"; /** - * Whether child controllers will directly consume the source admin topic in the parent Kafka cluster. + * This config specifies if Venice is deployed in a multi-region mode */ - public static final String ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED = "admin.topic.remote.consumption.enabled"; + public static final String MULTI_REGION = "multi.region"; /** * This config defines the source region name of the admin topic @@ -314,7 +281,8 @@ private ConfigKeys() { public static final String ADMIN_TOPIC_SOURCE_REGION = "admin.topic.source.region"; /** - * This following config defines whether admin consumption should be enabled or not, and this config will only control the behavior in Child Controller. + * This following config defines whether admin consumption should be enabled or not, and this config will only control + * the behavior in Child Controller. This is used for store migration. */ public static final String CHILD_CONTROLLER_ADMIN_TOPIC_CONSUMPTION_ENABLED = "child.controller.admin.topic.consumption.enabled"; @@ -359,8 +327,15 @@ private ConfigKeys() { "controller.store.graveyard.cleanup.sleep.interval.between.list.fetch.minutes"; /** - * Whether the superset schema generation in Parent Controller should be done via passed callback or not. + * Whether the superset schema generation in Primary Controller should be done via passed callback or not. */ + public static final String CONTROLLER_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED = + "controller.external.superset.schema.generation.enabled"; + + /** + * Whether the superset schema generation in Primary Controller should be done via passed callback or not. + */ + @Deprecated public static final String CONTROLLER_PARENT_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED = "controller.parent.external.superset.schema.generation.enabled"; @@ -1182,12 +1157,6 @@ private ConfigKeys() { public static final String NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_HYBRID_STORES = "native.replication.source.fabric.as.default.for.hybrid.stores"; - /** - * We will use this config to determine whether we should enable incremental push for hybrid active-active user stores. - * If this config is set to true, we will enable incremental push for hybrid active-active user stores. - */ - public static final String ENABLE_INCREMENTAL_PUSH_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES = - "enable.incremental.push.for.hybrid.active.active.user.stores"; /** * We will use this config to determine whether we should enable partial update for hybrid active-active user stores. * If this config is set to true, we will enable partial update for hybrid active-active user stores whose latest value @@ -1211,7 +1180,7 @@ private ConfigKeys() { // go/inclusivecode deprecated(alias="child.cluster.allowlist") @Deprecated - public static final String CHILD_CLUSTER_WHITELIST = "child.cluster.whitelist"; + public static final String CHILD_CLUSTER_ALLOWLIST_LEGACY = "child.cluster.whitelist"; /** * Only required when controller.parent.mode=true @@ -1236,7 +1205,7 @@ private ConfigKeys() { // go/inclusivecode deprecated(alias="native.replication.fabric.allowlist") @Deprecated - public static final String NATIVE_REPLICATION_FABRIC_WHITELIST = "native.replication.fabric.whitelist"; + public static final String NATIVE_REPLICATION_FABRIC_ALLOWLIST_LEGACY = "native.replication.fabric.whitelist"; /** * Previously {@link #CHILD_CLUSTER_ALLOWLIST} is used to also represent the allowlist of source fabrics @@ -1255,15 +1224,9 @@ private ConfigKeys() { public static final String PARENT_KAFKA_CLUSTER_FABRIC_LIST = "parent.kafka.cluster.fabric.list"; /** - * Whether A/A is enabled on the controller. When it is true, all A/A required config (e.g. - * {@link #ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST}) must be set. - */ - public static final String ACTIVE_ACTIVE_ENABLED_ON_CONTROLLER = "active.active.enabled.on.controller"; - - /** - * A list of fabrics that are source(s) of the active active real time replication. When active-active replication - * is enabled on the controller {@link #ACTIVE_ACTIVE_ENABLED_ON_CONTROLLER} is true, this list should contain fabrics - * where the Venice server should consume from when it accepts the TS (TopicSwitch) message. + * A list of regions that are source(s) of the Active/Active real time replication. When running in a multi-region + * mode, this list should contain region names where the Venice server should consume from when it accepts the + * TS (TopicSwitch) message. * Example value of this config: "dc-0, dc-1, dc-2". */ public static final String ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST = "active.active.real.time.source.fabric.list"; @@ -1561,16 +1524,6 @@ private ConfigKeys() { */ public static final String CONTROLLER_HAAS_SUPER_CLUSTER_NAME = "controller.haas.super.cluster.name"; - /** - * Whether to enable batch push (including GF job) from Admin in Child Controller. - * In theory, we should disable batch push in Child Controller no matter what, but the fact is that today there are - * many tests, which are doing batch pushes to an individual cluster setup (only Child Controller), so disabling batch push from Admin - * in Child Controller will require a lot of refactoring. - * So the current strategy is to enable it by default, but disable it in EI and PROD. - */ - public static final String CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD = - "controller.enable.batch.push.from.admin.in.child"; - /** * A config that turns the key/value profiling stats on and off. This config can be placed in both Router and SNs and it * is off by default. When switching it on, We will emit a fine grained histogram that reflects the distribution of diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerApiConstants.java b/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerApiConstants.java index c2707b2056..94951636c8 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerApiConstants.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerApiConstants.java @@ -231,7 +231,7 @@ public class ControllerApiConstants { public static final String UNUSED_SCHEMA_DELETION_ENABLED = "unused_schema_deletion_enabled"; - public static final String BLOB_TRANSFER_ENABLED = "blob.transfer.enabled"; + public static final String BLOB_TRANSFER_ENABLED = "blob_transfer_enabled"; public static final String HEARTBEAT_TIMESTAMP = "heartbeat_timestamp"; } diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerClient.java b/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerClient.java index 0e708e6b7b..b1fe560ec5 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerClient.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerClient.java @@ -29,7 +29,6 @@ import static com.linkedin.venice.controllerapi.ControllerApiConstants.LOCKED_NODE_ID_LIST_SEPARATOR; import static com.linkedin.venice.controllerapi.ControllerApiConstants.LOCKED_STORAGE_NODE_IDS; import static com.linkedin.venice.controllerapi.ControllerApiConstants.NAME; -import static com.linkedin.venice.controllerapi.ControllerApiConstants.NATIVE_REPLICATION_SOURCE_FABRIC; import static com.linkedin.venice.controllerapi.ControllerApiConstants.OFFSET; import static com.linkedin.venice.controllerapi.ControllerApiConstants.OPERATION; import static com.linkedin.venice.controllerapi.ControllerApiConstants.OWNER; @@ -1101,19 +1100,6 @@ public SystemStoreHeartbeatResponse getHeartbeatFromSystemStore(String storeName SystemStoreHeartbeatResponse.class); } - public ControllerResponse configureNativeReplicationForCluster( - boolean enableNativeReplication, - String storeType, - Optional sourceFabric, - Optional regionsFilter) { - // Verify the input storeType is valid - VeniceUserStoreType.valueOf(storeType.toUpperCase()); - QueryParams params = newParams().add(STATUS, enableNativeReplication).add(STORE_TYPE, storeType); - sourceFabric.ifPresent(s -> params.add(NATIVE_REPLICATION_SOURCE_FABRIC, s)); - regionsFilter.ifPresent(f -> params.add(REGIONS_FILTER, f)); - return request(ControllerRoute.CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER, params, ControllerResponse.class); - } - public ControllerResponse configureActiveActiveReplicationForCluster( boolean enableActiveActiveReplication, String storeType, diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerRoute.java b/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerRoute.java index 037131ab96..83d2352b7f 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerRoute.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/controllerapi/ControllerRoute.java @@ -231,9 +231,6 @@ public enum ControllerRoute { UPDATE_ACL("/update_acl", HttpMethod.POST, Arrays.asList(CLUSTER, NAME, ACCESS_PERMISSION)), GET_ACL("/get_acl", HttpMethod.GET, Arrays.asList(CLUSTER, NAME)), DELETE_ACL("/delete_acl", HttpMethod.GET, Arrays.asList(CLUSTER, NAME)), - CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER( - "/configure_native_replication_for_cluster", HttpMethod.POST, Arrays.asList(CLUSTER, STORE_TYPE, STATUS) - ), CONFIGURE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER( "/configure_active_active_replication_for_cluster", HttpMethod.POST, Arrays.asList(CLUSTER, STORE_TYPE, STATUS) ), GET_DELETABLE_STORE_TOPICS("/get_deletable_store_topics", HttpMethod.GET, Collections.emptyList()), diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/helix/ParentHelixOfflinePushAccessor.java b/internal/venice-common/src/main/java/com/linkedin/venice/helix/ParentHelixOfflinePushAccessor.java index 5d26594ac0..655a9572a3 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/helix/ParentHelixOfflinePushAccessor.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/helix/ParentHelixOfflinePushAccessor.java @@ -1,53 +1,22 @@ package com.linkedin.venice.helix; -import com.linkedin.venice.exceptions.VeniceException; -import com.linkedin.venice.pushmonitor.OfflinePushStatus; import com.linkedin.venice.utils.HelixUtils; import com.linkedin.venice.utils.PathResourceRegistry; -import java.util.List; -import org.apache.helix.AccessOption; -import org.apache.helix.manager.zk.ZkBaseDataAccessor; import org.apache.helix.zookeeper.impl.client.ZkClient; public class ParentHelixOfflinePushAccessor { // Use the different path from the normal offline push status znodes to prevent impacting the offline push monitor. public static final String OFFLINE_PUSH_SUB_PATH = "ParentOfflinePushes"; - private final ZkClient zkClient; - /** - * Zk accessor for offline push status ZNodes. - */ - private final ZkBaseDataAccessor offlinePushStatusAccessor; public ParentHelixOfflinePushAccessor(ZkClient zkClient, HelixAdapterSerializer adapter) { - this.zkClient = zkClient; String offlinePushStatusPattern = getOfflinePushStatuesParentPath(PathResourceRegistry.WILDCARD_MATCH_ANY) + "/" + PathResourceRegistry.WILDCARD_MATCH_ANY; adapter.registerSerializer(offlinePushStatusPattern, new OfflinePushStatusJSONSerializer()); - this.zkClient.setZkSerializer(adapter); - this.offlinePushStatusAccessor = new ZkBaseDataAccessor<>(zkClient); - } - - public OfflinePushStatus getOfflinePushStatus(String clusterName, String kafkaTopic) { - OfflinePushStatus offlinePushStatus = - offlinePushStatusAccessor.get(getOfflinePushStatusPath(clusterName, kafkaTopic), null, AccessOption.PERSISTENT); - if (offlinePushStatus == null) { - throw new VeniceException( - "Can not find offline push status in ZK from path:" + getOfflinePushStatusPath(clusterName, kafkaTopic)); - } - return offlinePushStatus; - } - - public List getAllPushNames(String clusterName) { - return offlinePushStatusAccessor - .getChildNames(getOfflinePushStatuesParentPath(clusterName), AccessOption.PERSISTENT); + zkClient.setZkSerializer(adapter); } private String getOfflinePushStatuesParentPath(String clusterName) { return HelixUtils.getHelixClusterZkPath(clusterName) + "/" + OFFLINE_PUSH_SUB_PATH; } - - private String getOfflinePushStatusPath(String clusterNaem, String topic) { - return getOfflinePushStatuesParentPath(clusterNaem) + "/" + topic; - } } diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/helix/StoragePersonaRepository.java b/internal/venice-common/src/main/java/com/linkedin/venice/helix/StoragePersonaRepository.java index cde1c6a5ed..42f235bd9e 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/helix/StoragePersonaRepository.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/helix/StoragePersonaRepository.java @@ -179,15 +179,15 @@ private void deleteStores(List storeNames) { public StoragePersona getPersonaContainingStore(String storeName) { String personaName = storeNamePersonaMap.get(storeName); - if (personaName == null) + if (personaName == null) { return null; + } return getPersona(personaName); } private boolean isStoreSetValid(StoragePersona persona, Optional additionalStore) { Set setToValidate = new HashSet<>(); - if (additionalStore.isPresent()) - setToValidate.add(additionalStore.get().getName()); + additionalStore.ifPresent(store -> setToValidate.add(store.getName())); setToValidate.addAll(persona.getStoresToEnforce()); return setToValidate.stream() .allMatch( diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/meta/BackupStrategy.java b/internal/venice-common/src/main/java/com/linkedin/venice/meta/BackupStrategy.java index c1ea17b8be..d6422747d6 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/meta/BackupStrategy.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/meta/BackupStrategy.java @@ -24,7 +24,7 @@ public enum BackupStrategy { // KEEP_IN_KAFKA_ONLY, /** Keep in user-specified store eg HDD, other DB */ // KEEP_IN_USER_STORE; - private int value; + private final int value; BackupStrategy(int v) { this.value = v; @@ -35,6 +35,10 @@ public enum BackupStrategy { Arrays.stream(values()).forEach(s -> idMapping.put(s.value, s)); } + public int getValue() { + return value; + } + public static BackupStrategy fromInt(int i) { BackupStrategy strategy = idMapping.get(i); if (strategy == null) { diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/meta/HybridStoreConfig.java b/internal/venice-common/src/main/java/com/linkedin/venice/meta/HybridStoreConfig.java index 061098317f..3952f8f4a2 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/meta/HybridStoreConfig.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/meta/HybridStoreConfig.java @@ -27,4 +27,9 @@ public interface HybridStoreConfig extends DataModelBackedStructure= 0 && (this.getOffsetLagThresholdToGoOnline() >= 0 + || this.getProducerTimestampLagThresholdToGoOnlineInSeconds() >= 0); + } } diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/meta/HybridStoreConfigImpl.java b/internal/venice-common/src/main/java/com/linkedin/venice/meta/HybridStoreConfigImpl.java index fda0d9a33d..5eb35d14b5 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/meta/HybridStoreConfigImpl.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/meta/HybridStoreConfigImpl.java @@ -14,8 +14,10 @@ */ @JsonIgnoreProperties(ignoreUnknown = true) public class HybridStoreConfigImpl implements HybridStoreConfig { + @Deprecated public static final long DEFAULT_REWIND_TIME_IN_SECONDS = Time.SECONDS_PER_DAY; public static final long DEFAULT_HYBRID_TIME_LAG_THRESHOLD = -1L; + @Deprecated public static final long DEFAULT_HYBRID_OFFSET_LAG_THRESHOLD = 1000L; private final StoreHybridConfig hybridConfig; diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/meta/PartitionerConfigImpl.java b/internal/venice-common/src/main/java/com/linkedin/venice/meta/PartitionerConfigImpl.java index ccd727d7f2..00692908aa 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/meta/PartitionerConfigImpl.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/meta/PartitionerConfigImpl.java @@ -92,6 +92,9 @@ public int hashCode() { @JsonIgnore public PartitionerConfig clone() { - return new PartitionerConfigImpl(getPartitionerClass(), getPartitionerParams(), getAmplificationFactor()); + return new PartitionerConfigImpl( + getPartitionerClass(), + new HashMap<>(getPartitionerParams()), + getAmplificationFactor()); } } diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/meta/ZKStore.java b/internal/venice-common/src/main/java/com/linkedin/venice/meta/ZKStore.java index c4e9de7531..913e5b62a0 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/meta/ZKStore.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/meta/ZKStore.java @@ -208,7 +208,7 @@ public ZKStore(Store store) { setSchemaAutoRegisterFromPushJobEnabled(store.isSchemaAutoRegisterFromPushJobEnabled()); setLatestSuperSetValueSchemaId(store.getLatestSuperSetValueSchemaId()); setHybridStoreDiskQuotaEnabled(store.isHybridStoreDiskQuotaEnabled()); - setEtlStoreConfig(store.getEtlStoreConfig()); + setEtlStoreConfig(store.getEtlStoreConfig().clone()); setStoreMetadataSystemStoreEnabled(store.isStoreMetadataSystemStoreEnabled()); setLatestVersionPromoteToCurrentTimestamp(store.getLatestVersionPromoteToCurrentTimestamp()); setBackupVersionRetentionMs(store.getBackupVersionRetentionMs()); @@ -219,7 +219,7 @@ public ZKStore(Store store) { setStoreMetaSystemStoreEnabled(store.isStoreMetaSystemStoreEnabled()); setActiveActiveReplicationEnabled(store.isActiveActiveReplicationEnabled()); setRmdVersion(store.getRmdVersion()); - setViewConfigs(store.getViewConfigs()); + setViewConfigs(new HashMap<>(store.getViewConfigs())); setStorageNodeReadQuotaEnabled(store.isStorageNodeReadQuotaEnabled()); setUnusedSchemaDeletionEnabled(store.isUnusedSchemaDeletionEnabled()); setMinCompactionLagSeconds(store.getMinCompactionLagSeconds()); @@ -361,11 +361,7 @@ public void setLargestUsedVersionNumber(int largestUsedVersionNumber) { @SuppressWarnings("unused") // Used by Serializer/De-serializer for storing to Zoo Keeper @Override public long getStorageQuotaInByte() { - // This is a safeguard in case that some old stores do not have storage quota field - return (this.storeProperties.storageQuotaInByte <= 0 - && this.storeProperties.storageQuotaInByte != UNLIMITED_STORAGE_QUOTA) - ? DEFAULT_STORAGE_QUOTA - : this.storeProperties.storageQuotaInByte; + return this.storeProperties.storageQuotaInByte; } @SuppressWarnings("unused") // Used by Serializer/De-serializer for storing to Zoo Keeper diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/utils/PartitionUtils.java b/internal/venice-common/src/main/java/com/linkedin/venice/utils/PartitionUtils.java index 4ecf0f4456..c3e84fde88 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/utils/PartitionUtils.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/utils/PartitionUtils.java @@ -68,7 +68,9 @@ public static int calculatePartitionCount( partitionCount, storageQuota, storeName); - return (int) partitionCount; + + // At least 1 partition + return partitionCount <= 0 ? 1 : (int) partitionCount; } public static VenicePartitioner getVenicePartitioner(PartitionerConfig config) { diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java b/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java index 7d6c988799..633febbf96 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java @@ -469,7 +469,7 @@ public static List parseCommaSeparatedStringToList(String rawString) { } public static Set parseCommaSeparatedStringToSet(String rawString) { - if (rawString == null || rawString.length() == 0) { + if (StringUtils.isEmpty(rawString)) { return Collections.emptySet(); } return Utils.setOf(rawString.split(",\\s*")); @@ -736,10 +736,6 @@ public static Set mutableSetOf(T... objs) { return new HashSet<>(Arrays.asList(objs)); } - public static long calculateDurationMs(Time time, long startTimeMs) { - return time.getMilliseconds() - startTimeMs; - } - public static void closeQuietlyWithErrorLogged(Closeable... closeables) { if (closeables == null) { return; diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/meta/BackupStrategyTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/meta/BackupStrategyTest.java new file mode 100644 index 0000000000..29fa48cf72 --- /dev/null +++ b/internal/venice-common/src/test/java/com/linkedin/venice/meta/BackupStrategyTest.java @@ -0,0 +1,23 @@ +package com.linkedin.venice.meta; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +import com.linkedin.venice.exceptions.VeniceException; +import org.testng.annotations.Test; + + +public class BackupStrategyTest { + @Test + public void testFromInt() { + assertEquals(BackupStrategy.fromInt(0), BackupStrategy.KEEP_MIN_VERSIONS); + assertEquals(BackupStrategy.fromInt(1), BackupStrategy.DELETE_ON_NEW_PUSH_START); + assertThrows(VeniceException.class, () -> BackupStrategy.fromInt(2)); + } + + @Test + public void testGetValue() { + assertEquals(BackupStrategy.KEEP_MIN_VERSIONS.getValue(), 0); + assertEquals(BackupStrategy.DELETE_ON_NEW_PUSH_START.getValue(), 1); + } +} diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/meta/TestHybridStoreConfig.java b/internal/venice-common/src/test/java/com/linkedin/venice/meta/TestHybridStoreConfig.java index c4dd2368d6..2f75104798 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/meta/TestHybridStoreConfig.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/meta/TestHybridStoreConfig.java @@ -22,4 +22,26 @@ public void deserializes() throws IOException { Assert.assertEquals(fasterXml.getRewindTimeInSeconds(), 123L); Assert.assertEquals(fasterXml.getDataReplicationPolicy(), DataReplicationPolicy.NON_AGGREGATE); } + + @Test + public void testIsHybrid() { + HybridStoreConfig hybridStoreConfig; + hybridStoreConfig = new HybridStoreConfigImpl(-1, -1, -1, null, null); + Assert.assertFalse(hybridStoreConfig.isHybrid()); + + hybridStoreConfig = new HybridStoreConfigImpl(100, -1, -1, null, null); + Assert.assertFalse(hybridStoreConfig.isHybrid()); + + hybridStoreConfig = new HybridStoreConfigImpl(100, 100, -1, null, null); + Assert.assertTrue(hybridStoreConfig.isHybrid()); + + hybridStoreConfig = new HybridStoreConfigImpl(100, 100, 100, null, null); + Assert.assertTrue(hybridStoreConfig.isHybrid()); + + hybridStoreConfig = new HybridStoreConfigImpl(100, -1, 100, null, null); + Assert.assertTrue(hybridStoreConfig.isHybrid()); + + hybridStoreConfig = new HybridStoreConfigImpl(-1, -1, 100, null, null); + Assert.assertFalse(hybridStoreConfig.isHybrid()); + } } diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java index 24f3cc4acf..1459858577 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java @@ -15,6 +15,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import org.testng.Assert; import org.testng.annotations.Test; @@ -183,4 +184,15 @@ public void testParseMap() { public void testSanitizingStringForLogger() { Assert.assertEquals(Utils.getSanitizedStringForLogger(".abc.123."), "_abc_123_"); } + + @Test + public void testParseCommaSeparatedStringToSet() { + Assert.assertTrue(Utils.parseCommaSeparatedStringToSet(null).isEmpty()); + Assert.assertTrue(Utils.parseCommaSeparatedStringToSet("").isEmpty()); + Set set = Utils.parseCommaSeparatedStringToSet("a,b,c"); + Assert.assertEquals(set.size(), 3); + Assert.assertTrue(set.contains("a")); + Assert.assertTrue(set.contains("b")); + Assert.assertTrue(set.contains("c")); + } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForActiveActiveReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForActiveActiveReplication.java index 362e47aa2d..8a622d0ea7 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForActiveActiveReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForActiveActiveReplication.java @@ -1,267 +1,79 @@ package com.linkedin.venice.controller; -import static com.linkedin.venice.ConfigKeys.ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY_STORE; import static com.linkedin.venice.ConfigKeys.ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY; -import static com.linkedin.venice.ConfigKeys.PARTICIPANT_MESSAGE_STORE_ENABLED; -import static com.linkedin.venice.controller.VeniceHelixAdmin.VERSION_ID_UNSET; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; +import static com.linkedin.venice.utils.TestUtils.assertCommand; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; -import com.linkedin.venice.common.VeniceSystemStoreUtils; +import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; -import com.linkedin.venice.integration.utils.D2TestUtils; -import com.linkedin.venice.meta.Store; -import com.linkedin.venice.meta.Version; -import com.linkedin.venice.pubsub.manager.TopicManager; -import com.linkedin.venice.pubsub.manager.TopicManagerRepository; -import com.linkedin.venice.utils.TestUtils; +import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; +import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; +import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; -import com.linkedin.venice.utils.VeniceProperties; -import io.tehuti.metrics.MetricsRepository; -import java.io.IOException; -import java.util.Optional; import java.util.Properties; -import java.util.concurrent.TimeUnit; -import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -public class TestClusterLevelConfigForActiveActiveReplication extends AbstractTestVeniceHelixAdmin { +public class TestClusterLevelConfigForActiveActiveReplication { private static final long TEST_TIMEOUT = 30 * Time.MS_PER_SECOND; + private VeniceTwoLayerMultiRegionMultiClusterWrapper multiRegionMultiClusterWrapper; + private ControllerClient parentControllerClient; + @BeforeClass(alwaysRun = true) - public void setUp() throws Exception { - setupCluster(); + public void setUp() { + Utils.thisIsLocalhost(); + Properties childControllerProps = new Properties(); + childControllerProps + .setProperty(ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE, Boolean.toString(true)); + multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfRouters(1) + .numberOfServers(1) + .parentControllerProperties(childControllerProps) + .build()); + String clusterName = multiRegionMultiClusterWrapper.getClusterNames()[0]; + parentControllerClient = + new ControllerClient(clusterName, multiRegionMultiClusterWrapper.getControllerConnectString()); } @AfterClass(alwaysRun = true) public void cleanUp() { - cleanupCluster(); - } - - @Test(timeOut = TEST_TIMEOUT) - public void testClusterLevelActiveActiveReplicationConfigForNewHybridStores() throws IOException { - TopicManagerRepository originalTopicManagerRepository = prepareCluster(true, false); - String storeNameHybrid = Utils.getUniqueString("test-store-hybrid"); - String pushJobId1 = "test-push-job-id-1"; - /** - * Do not enable any store-level config for leader/follower mode or native replication feature. - */ - veniceAdmin.createStore(clusterName, storeNameHybrid, "test-owner", KEY_SCHEMA, VALUE_SCHEMA); - /** - * Add a version - */ - veniceAdmin.addVersionAndTopicOnly( - clusterName, - storeNameHybrid, - pushJobId1, - VERSION_ID_UNSET, - 1, - 1, - false, - true, - Version.PushType.STREAM, - null, - null, - Optional.empty(), - -1, - 1, - Optional.empty(), - false); - - // Version 1 should exist. - assertEquals(veniceAdmin.getStore(clusterName, storeNameHybrid).getVersions().size(), 1); - // Check store level Active/Active is enabled or not - assertFalse(veniceAdmin.getStore(clusterName, storeNameHybrid).isActiveActiveReplicationEnabled()); - veniceAdmin.updateStore( - clusterName, - storeNameHybrid, - new UpdateStoreQueryParams().setHybridRewindSeconds(1000L).setHybridOffsetLagThreshold(1000L)); - assertTrue(veniceAdmin.getStore(clusterName, storeNameHybrid).isActiveActiveReplicationEnabled()); - - veniceAdmin.updateStore( - clusterName, - storeNameHybrid, - new UpdateStoreQueryParams().setHybridRewindSeconds(-1).setHybridOffsetLagThreshold(-1)); - assertTrue(veniceAdmin.getStore(clusterName, storeNameHybrid).isActiveActiveReplicationEnabled()); - - // Set topic original topic manager back - veniceAdmin.setTopicManagerRepository(originalTopicManagerRepository); + Utils.closeQuietlyWithErrorLogged(multiRegionMultiClusterWrapper); } @Test(timeOut = TEST_TIMEOUT) - public void testClusterLevelActiveActiveReplicationConfigForNewIncrementalPushStores() throws IOException { - TopicManagerRepository originalTopicManagerRepository = prepareCluster(true, false); - String storeNameIncremental = Utils.getUniqueString("test-store-incremental"); + public void testClusterLevelActiveActiveReplicationConfigForNewHybridStores() { + String storeName = Utils.getUniqueString("test-store-hybrid"); String pushJobId1 = "test-push-job-id-1"; - /** - * Do not enable any store-level config for leader/follower mode or native replication feature. - */ - veniceAdmin.createStore(clusterName, storeNameIncremental, "test-owner", KEY_SCHEMA, VALUE_SCHEMA); - /** - * Add a version - */ - veniceAdmin.addVersionAndTopicOnly( - clusterName, - storeNameIncremental, - pushJobId1, - VERSION_ID_UNSET, - 1, - 1, - false, - true, - Version.PushType.STREAM, - null, - null, - Optional.empty(), - -1, - 1, - Optional.empty(), - false); + parentControllerClient.createNewStore(storeName, "test-owner", "\"string\"", "\"string\""); + parentControllerClient.emptyPush(storeName, pushJobId1, 1); // Version 1 should exist. - assertEquals(veniceAdmin.getStore(clusterName, storeNameIncremental).getVersions().size(), 1); + StoreInfo store = assertCommand(parentControllerClient.getStore(storeName)).getStore(); + assertEquals(store.getVersions().size(), 1); // Check store level Active/Active is enabled or not - veniceAdmin.setIncrementalPushEnabled(clusterName, storeNameIncremental, false); - assertFalse(veniceAdmin.getStore(clusterName, storeNameIncremental).isIncrementalPushEnabled()); - assertFalse(veniceAdmin.getStore(clusterName, storeNameIncremental).isActiveActiveReplicationEnabled()); - - veniceAdmin.setIncrementalPushEnabled(clusterName, storeNameIncremental, true); - assertTrue(veniceAdmin.getStore(clusterName, storeNameIncremental).isIncrementalPushEnabled()); - assertTrue(veniceAdmin.getStore(clusterName, storeNameIncremental).isActiveActiveReplicationEnabled()); - - // After inc push is disabled, even default A/A config for pure hybrid store is false, - // original store A/A config is enabled. - veniceAdmin.setIncrementalPushEnabled(clusterName, storeNameIncremental, false); - assertFalse(veniceAdmin.getStore(clusterName, storeNameIncremental).isIncrementalPushEnabled()); - assertTrue(veniceAdmin.getStore(clusterName, storeNameIncremental).isActiveActiveReplicationEnabled()); - - // Set topic original topic manager back - veniceAdmin.setTopicManagerRepository(originalTopicManagerRepository); - } + assertFalse(store.isActiveActiveReplicationEnabled()); - @Test(timeOut = TEST_TIMEOUT) - public void testClusterLevelActiveActiveReplicationConfigForNewBatchOnlyStores() throws IOException { - TopicManagerRepository originalTopicManagerRepository = prepareCluster(false, true); - String storeNameBatchOnly = Utils.getUniqueString("test-store-batch-only"); - String pushJobId1 = "test-push-job-id-1"; - /** - * Do not enable any store-level config for leader/follower mode or native replication feature. - */ - veniceAdmin.createStore(clusterName, storeNameBatchOnly, "test-owner", KEY_SCHEMA, VALUE_SCHEMA); - /** - * Add a version - */ - veniceAdmin.addVersionAndTopicOnly( - clusterName, - storeNameBatchOnly, - pushJobId1, - VERSION_ID_UNSET, - 1, - 1, - false, - true, - Version.PushType.STREAM, - null, - null, - Optional.empty(), - -1, - 1, - Optional.empty(), - false); - - // Version 1 should exist. - assertEquals(veniceAdmin.getStore(clusterName, storeNameBatchOnly).getVersions().size(), 1); - - // Store level Active/Active replication should be enabled since this store is a batch-only store by default - assertTrue(veniceAdmin.getStore(clusterName, storeNameBatchOnly).isActiveActiveReplicationEnabled()); - - // After updating the store to have incremental push enabled, it's A/A is still enabled - veniceAdmin.setIncrementalPushEnabled(clusterName, storeNameBatchOnly, true); - assertTrue(veniceAdmin.getStore(clusterName, storeNameBatchOnly).isActiveActiveReplicationEnabled()); - - // Let's disable the A/A config for the store. - veniceAdmin.setActiveActiveReplicationEnabled(clusterName, storeNameBatchOnly, false); - assertFalse(veniceAdmin.getStore(clusterName, storeNameBatchOnly).isActiveActiveReplicationEnabled()); - - // After updating the store back to a batch-only store, it's A/A becomes enabled again - veniceAdmin.setIncrementalPushEnabled(clusterName, storeNameBatchOnly, false); - assertTrue(veniceAdmin.getStore(clusterName, storeNameBatchOnly).isActiveActiveReplicationEnabled()); - - // After updating the store to be a hybrid store, it's A/A should still be enabled. - veniceAdmin.updateStore( - clusterName, - storeNameBatchOnly, + // Convert to hybrid store + parentControllerClient.updateStore( + storeName, new UpdateStoreQueryParams().setHybridRewindSeconds(1000L).setHybridOffsetLagThreshold(1000L)); - assertTrue(veniceAdmin.getStore(clusterName, storeNameBatchOnly).isActiveActiveReplicationEnabled()); - - // Set topic original topic manager back - veniceAdmin.setTopicManagerRepository(originalTopicManagerRepository); - } - - private TopicManagerRepository prepareCluster( - boolean enableActiveActiveForHybrid, - boolean enableActiveActiveForBatchOnly) throws IOException { - veniceAdmin.stop(clusterName); - veniceAdmin.close(); - Properties controllerProperties = - getActiveActiveControllerProperties(clusterName, enableActiveActiveForHybrid, enableActiveActiveForBatchOnly); - veniceAdmin = new VeniceHelixAdmin( - TestUtils.getMultiClusterConfigFromOneCluster( - new VeniceControllerConfig(new VeniceProperties(controllerProperties))), - new MetricsRepository(), - D2TestUtils.getAndStartD2Client(zkAddress), - pubSubTopicRepository, - pubSubBrokerWrapper.getPubSubClientsFactory()); + assertTrue(parentControllerClient.getStore(storeName).getStore().isActiveActiveReplicationEnabled()); - veniceAdmin.initStorageCluster(clusterName); - TopicManagerRepository originalTopicManagerRepository = veniceAdmin.getTopicManagerRepository(); - - TopicManager mockedTopicManager = mock(TopicManager.class); - TopicManagerRepository mockedTopicManageRepository = mock(TopicManagerRepository.class); - doReturn(mockedTopicManager).when(mockedTopicManageRepository).getLocalTopicManager(); - doReturn(mockedTopicManager).when(mockedTopicManageRepository).getTopicManager(any(String.class)); - doReturn(mockedTopicManager).when(mockedTopicManageRepository).getTopicManager(anyString()); - veniceAdmin.setTopicManagerRepository(mockedTopicManageRepository); - TestUtils - .waitForNonDeterministicCompletion(5, TimeUnit.SECONDS, () -> veniceAdmin.isLeaderControllerFor(clusterName)); - Object createParticipantStoreFromProp = controllerProperties.get(PARTICIPANT_MESSAGE_STORE_ENABLED); - if (createParticipantStoreFromProp != null && Boolean.parseBoolean(createParticipantStoreFromProp.toString())) { - // Wait for participant store to finish materializing - TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, () -> { - Store store = - veniceAdmin.getStore(clusterName, VeniceSystemStoreUtils.getParticipantStoreNameForCluster(clusterName)); - Assert.assertNotNull(store); - assertEquals(store.getCurrentVersion(), 1); - }); - } - return originalTopicManagerRepository; - } - - private Properties getActiveActiveControllerProperties( - String clusterName, - boolean enableActiveActiveForHybrid, - boolean enableActiveActiveForBatchOnly) throws IOException { - Properties props = super.getControllerProperties(clusterName); - props.setProperty(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY, "true"); - // Enable Active/Active replication for hybrid stores through cluster-level config - props.setProperty( - ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE, - Boolean.toString(enableActiveActiveForHybrid)); - // Enable Active/Active replication for batch-only stores through cluster-level config - props.setProperty( - ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY_STORE, - Boolean.toString(enableActiveActiveForBatchOnly)); - return props; + // Reverting hybrid configs disables A/A mode + parentControllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setHybridRewindSeconds(-1).setHybridOffsetLagThreshold(-1)); + assertFalse(parentControllerClient.getStore(storeName).getStore().isActiveActiveReplicationEnabled()); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForNativeReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForNativeReplication.java index aa64d0a333..c049bb2458 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForNativeReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestClusterLevelConfigForNativeReplication.java @@ -1,21 +1,18 @@ package com.linkedin.venice.controller; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_BATCH_ONLY_STORES; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_HYBRID_STORES; -import static com.linkedin.venice.controller.VeniceHelixAdmin.VERSION_ID_UNSET; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; +import static com.linkedin.venice.utils.TestUtils.assertCommand; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; -import com.linkedin.venice.meta.Version; -import com.linkedin.venice.pubsub.manager.TopicManager; -import com.linkedin.venice.pubsub.manager.TopicManagerRepository; +import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; +import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; +import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.utils.Utils; -import java.io.IOException; -import java.util.Optional; import java.util.Properties; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -23,88 +20,67 @@ import org.testng.annotations.Test; -public class TestClusterLevelConfigForNativeReplication extends AbstractTestVeniceHelixAdmin { +public class TestClusterLevelConfigForNativeReplication { + private VeniceTwoLayerMultiRegionMultiClusterWrapper multiRegionMultiClusterWrapper; + private ControllerClient parentControllerClient; + @BeforeClass(alwaysRun = true) - public void setUp() throws Exception { - setupCluster(); + public void setUp() { + Utils.thisIsLocalhost(); + Properties childControllerProps = new Properties(); + // enable native replication for batch-only stores through cluster-level config + childControllerProps.setProperty(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_BATCH_ONLY_STORES, "dc-batch"); + childControllerProps.setProperty(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_HYBRID_STORES, "dc-hybrid"); + + multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfRouters(1) + .numberOfServers(1) + .parentControllerProperties(childControllerProps) + .build()); + String clusterName = multiRegionMultiClusterWrapper.getClusterNames()[0]; + parentControllerClient = + new ControllerClient(clusterName, multiRegionMultiClusterWrapper.getControllerConnectString()); } @AfterClass(alwaysRun = true) public void cleanUp() { - cleanupCluster(); - } - - @Override - Properties getControllerProperties(String clusterName) throws IOException { - Properties props = super.getControllerProperties(clusterName); - // enable native replication for batch-only stores through cluster-level config - props.setProperty(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY, "true"); - props.setProperty(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_BATCH_ONLY_STORES, "dc-batch"); - props.setProperty(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_HYBRID_STORES, "dc-hybrid"); - return props; + Utils.closeQuietlyWithErrorLogged(multiRegionMultiClusterWrapper); } @Test public void testClusterLevelNativeReplicationConfigForNewStores() { - TopicManagerRepository originalTopicManagerRepository = veniceAdmin.getTopicManagerRepository(); - - TopicManager mockedTopicManager = mock(TopicManager.class); - TopicManagerRepository mockedTopicManageRepository = mock(TopicManagerRepository.class); - doReturn(mockedTopicManager).when(mockedTopicManageRepository).getLocalTopicManager(); - doReturn(mockedTopicManager).when(mockedTopicManageRepository).getTopicManager(any(String.class)); - doReturn(mockedTopicManager).when(mockedTopicManageRepository).getTopicManager(anyString()); - veniceAdmin.setTopicManagerRepository(mockedTopicManageRepository); String storeName = Utils.getUniqueString("test-store"); String pushJobId1 = "test-push-job-id-1"; - /** - * Do not enable any store-level config for leader/follower mode or native replication feature. - */ - veniceAdmin.createStore(clusterName, storeName, "test-owner", KEY_SCHEMA, VALUE_SCHEMA); + parentControllerClient.createNewStore(storeName, "test-owner", "\"string\"", "\"string\""); + parentControllerClient.emptyPush(storeName, pushJobId1, 1); - /** - * Add a version - */ - veniceAdmin.addVersionAndTopicOnly( - clusterName, - storeName, - pushJobId1, - VERSION_ID_UNSET, - 1, - 1, - false, - true, - Version.PushType.BATCH, - null, - null, - Optional.empty(), - -1, - 1, - Optional.empty(), - false); // Version 1 should exist. - Assert.assertEquals(veniceAdmin.getStore(clusterName, storeName).getVersions().size(), 1); + StoreInfo store = assertCommand(parentControllerClient.getStore(storeName)).getStore(); + assertEquals(store.getVersions().size(), 1); + // native replication should be enabled by cluster-level config - Assert.assertEquals(veniceAdmin.getStore(clusterName, storeName).isNativeReplicationEnabled(), true); - Assert.assertEquals(veniceAdmin.getStore(clusterName, storeName).getNativeReplicationSourceFabric(), "dc-batch"); - veniceAdmin.updateStore( - clusterName, - storeName, - new UpdateStoreQueryParams().setHybridRewindSeconds(1L).setHybridOffsetLagThreshold(1L)); - Assert.assertEquals(veniceAdmin.getStore(clusterName, storeName).getNativeReplicationSourceFabric(), "dc-hybrid"); - veniceAdmin.updateStore( - clusterName, - storeName, - new UpdateStoreQueryParams().setHybridRewindSeconds(-1L).setHybridOffsetLagThreshold(-1L)); - Assert.assertEquals(veniceAdmin.getStore(clusterName, storeName).getNativeReplicationSourceFabric(), "dc-batch"); - veniceAdmin.updateStore( - clusterName, - storeName, - new UpdateStoreQueryParams().setIncrementalPushEnabled(true) - .setHybridRewindSeconds(1L) - .setHybridOffsetLagThreshold(10)); - Assert.assertEquals(veniceAdmin.getStore(clusterName, storeName).getNativeReplicationSourceFabric(), "dc-hybrid"); + assertTrue(store.isNativeReplicationEnabled()); + Assert.assertEquals(store.getNativeReplicationSourceFabric(), "dc-batch"); + + // Convert to hybrid store + assertCommand( + parentControllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setHybridRewindSeconds(1000L).setHybridOffsetLagThreshold(1000L))); + store = assertCommand(parentControllerClient.getStore(storeName)).getStore(); + assertTrue(store.isNativeReplicationEnabled()); + Assert.assertEquals(store.getNativeReplicationSourceFabric(), "dc-hybrid"); - // Set topic original topic manager back - veniceAdmin.setTopicManagerRepository(originalTopicManagerRepository); + // Reverting hybrid configs reverts NR source region + assertCommand( + parentControllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setHybridRewindSeconds(-1).setHybridOffsetLagThreshold(-1))); + store = assertCommand(parentControllerClient.getStore(storeName)).getStore(); + assertTrue(store.isNativeReplicationEnabled()); + Assert.assertEquals(store.getNativeReplicationSourceFabric(), "dc-batch"); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java index 59ea1f13e6..6bb682135b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java @@ -1,8 +1,6 @@ package com.linkedin.venice.controller; import static com.linkedin.venice.ConfigKeys.ALLOW_CLUSTER_WIPE; -import static com.linkedin.venice.ConfigKeys.CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY; import static com.linkedin.venice.ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS; import com.linkedin.venice.AdminTool; @@ -29,9 +27,17 @@ import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.Ignore; import org.testng.annotations.Test; +/** + * Note: These tests use an unsupported mode during setup - by creating stores and running pushes + * to them directly in child regions. This is not how new regions will be added. We need the test setup to support + * adding blank regions so that we can simulate the true fabric buildout process. Because of this, I've disabled these + * tests for now. + */ +@Ignore public class TestFabricBuildout { private static final int TEST_TIMEOUT = 90_000; // ms @@ -47,9 +53,7 @@ public class TestFabricBuildout { @BeforeClass public void setUp() { Properties childControllerProperties = new Properties(); - childControllerProperties.setProperty(CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD, "true"); childControllerProperties.setProperty(ALLOW_CLUSTER_WIPE, "true"); - childControllerProperties.setProperty(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY, "true"); Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestIncrementalPush.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestIncrementalPush.java index 8d86282986..9cd4e5def2 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestIncrementalPush.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestIncrementalPush.java @@ -11,6 +11,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.BackupStrategy; +import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pushmonitor.ExecutionStatus; @@ -59,7 +60,7 @@ public void setUp() { cluster.getNewStore(storeName); long storageQuota = 2L * PARTITION_SIZE; UpdateStoreQueryParams params = new UpdateStoreQueryParams(); - params.setIncrementalPushEnabled(true) + params.setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) .setHybridRewindSeconds(1) .setHybridOffsetLagThreshold(1) .setStorageQuotaInByte(storageQuota) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java index 045c0d020b..d6a39c3b7b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java @@ -21,6 +21,7 @@ import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.BufferReplayPolicy; +import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.meta.Version; @@ -31,6 +32,7 @@ import com.linkedin.venice.utils.IntegrationTestPushUtils; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.TestWriteUtils; +import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; import java.io.File; import java.io.IOException; @@ -194,7 +196,9 @@ public void testUpdateStore() { incPushStoreName, parentControllerClient, new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) - .setIncrementalPushEnabled(true)); + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) + .setHybridRewindSeconds(Time.SECONDS_PER_DAY) + .setHybridOffsetLagThreshold(1000)); TestUtils.waitForNonDeterministicAssertion(30, TimeUnit.SECONDS, false, true, () -> { for (ControllerClient controllerClient: controllerClients) { StoreResponse storeResponse = controllerClient.getStore(incPushStoreName); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithSharedEnvironment.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithSharedEnvironment.java index 71cf2426e1..5e71e1a12e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithSharedEnvironment.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithSharedEnvironment.java @@ -58,6 +58,7 @@ import com.linkedin.venice.utils.PropertyBuilder; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.TestWriteUtils; +import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import com.linkedin.venice.views.ChangeCaptureView; @@ -480,27 +481,6 @@ public void testUpdateStoreMetadata() throws Exception { PartitionerConfig partitionerConfig = new PartitionerConfigImpl(); veniceAdmin.setStorePartitionerConfig(clusterName, storeName, partitionerConfig); - veniceAdmin.setIncrementalPushEnabled(clusterName, storeName, true); - Assert.assertTrue(veniceAdmin.getStore(clusterName, storeName).isIncrementalPushEnabled()); - - veniceAdmin.setBootstrapToOnlineTimeoutInHours(clusterName, storeName, 48); - Assert.assertEquals(veniceAdmin.getStore(clusterName, storeName).getBootstrapToOnlineTimeoutInHours(), 48); - - veniceAdmin.setHybridStoreDiskQuotaEnabled(clusterName, storeName, true); - Assert.assertTrue(veniceAdmin.getStore(clusterName, storeName).isHybridStoreDiskQuotaEnabled()); - - // test setting per-store RMD (replication metadata) version ID - int rmdVersion = veniceAdmin.getStore(clusterName, storeName).getRmdVersion(); - Assert.assertEquals(rmdVersion, -1); - - veniceAdmin.setReplicationMetadataVersionID(clusterName, storeName, 2); - rmdVersion = veniceAdmin.getStore(clusterName, storeName).getRmdVersion(); - Assert.assertNotEquals(rmdVersion, -1); - Assert.assertEquals(rmdVersion, 2); - - // test hybrid config - // set incrementalPushEnabled to be false as hybrid and incremental are mutex - veniceAdmin.setIncrementalPushEnabled(clusterName, storeName, false); Assert.assertFalse(veniceAdmin.getStore(clusterName, storeName).isHybrid()); veniceAdmin.updateStore( clusterName, @@ -1468,33 +1448,27 @@ public void leakyTopicTruncation() { } } - @Test(timeOut = TOTAL_TIMEOUT_FOR_LONG_TEST_MS) - public void testSetLargestUsedVersion() { - String storeName = "testSetLargestUsedVersion"; - veniceAdmin.createStore(clusterName, storeName, storeOwner, KEY_SCHEMA, VALUE_SCHEMA); - Store store = veniceAdmin.getStore(clusterName, storeName); - Assert.assertEquals(store.getLargestUsedVersionNumber(), 0); - - Version version = - veniceAdmin.incrementVersionIdempotent(clusterName, storeName, Version.guidBasedDummyPushId(), 1, 1); - store = veniceAdmin.getStore(clusterName, storeName); - Assert.assertTrue(version.getNumber() > 0); - Assert.assertEquals(store.getLargestUsedVersionNumber(), version.getNumber()); - - veniceAdmin.setStoreLargestUsedVersion(clusterName, storeName, 0); - store = veniceAdmin.getStore(clusterName, storeName); - Assert.assertEquals(store.getLargestUsedVersionNumber(), 0); - } - @Test(timeOut = TOTAL_TIMEOUT_FOR_LONG_TEST_MS) public void testWriteComputationEnabled() { String storeName = Utils.getUniqueString("test_store"); - veniceAdmin.createStore(clusterName, storeName, storeOwner, "\"string\"", "\"string\""); + String VALUE_FIELD_NAME = "int_field"; + String SECOND_VALUE_FIELD_NAME = "opt_int_field"; + String VALUE_SCHEMA_V2_STR = "{\n" + "\"type\": \"record\",\n" + "\"name\": \"TestValueSchema\",\n" + + "\"namespace\": \"com.linkedin.venice.fastclient.schema\",\n" + "\"fields\": [\n" + " {\"name\": \"" + + VALUE_FIELD_NAME + "\", \"type\": \"int\", \"default\": 10},\n" + "{\"name\": \"" + SECOND_VALUE_FIELD_NAME + + "\", \"type\": [\"null\", \"int\"], \"default\": null}]\n" + "}"; + + veniceAdmin.createStore(clusterName, storeName, storeOwner, "\"string\"", VALUE_SCHEMA_V2_STR); Store store = veniceAdmin.getStore(clusterName, storeName); Assert.assertFalse(store.isWriteComputationEnabled()); - veniceAdmin.updateStore(clusterName, storeName, new UpdateStoreQueryParams().setWriteComputationEnabled(true)); + veniceAdmin.updateStore( + clusterName, + storeName, + new UpdateStoreQueryParams().setHybridRewindSeconds(1000) + .setHybridOffsetLagThreshold(1000) + .setWriteComputationEnabled(true)); store = veniceAdmin.getStore(clusterName, storeName); Assert.assertTrue(store.isWriteComputationEnabled()); } @@ -1673,7 +1647,7 @@ public void testGetIncrementalPushVersion() { incrementalAndHybridEnabledStoreName, new UpdateStoreQueryParams().setHybridOffsetLagThreshold(1) .setHybridRewindSeconds(0) - .setIncrementalPushEnabled(true)); + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE)); veniceAdmin.incrementVersionIdempotent( clusterName, incrementalAndHybridEnabledStoreName, @@ -1703,7 +1677,10 @@ public void testGetIncrementalPushVersion() { public void testEarlyDeleteBackup() { String testDeleteStore = Utils.getUniqueString("testDeleteStore"); veniceAdmin.createStore(clusterName, testDeleteStore, storeOwner, "\"string\"", "\"string\""); - veniceAdmin.updateStore(clusterName, testDeleteStore, new UpdateStoreQueryParams().setIncrementalPushEnabled(true)); + veniceAdmin.updateStore( + clusterName, + testDeleteStore, + new UpdateStoreQueryParams().setHybridRewindSeconds(Time.SECONDS_PER_DAY).setHybridOffsetLagThreshold(1000)); veniceAdmin.incrementVersionIdempotent(clusterName, testDeleteStore, Version.guidBasedDummyPushId(), 1, 1); TestUtils.waitForNonDeterministicCompletion( TOTAL_TIMEOUT_FOR_SHORT_TEST_MS, @@ -1747,8 +1724,10 @@ public void testVersionLevelActiveActiveReplicationConfig() { /** * Enable L/F and Active/Active replication */ - veniceAdmin - .updateStore(clusterName, storeName, new UpdateStoreQueryParams().setActiveActiveReplicationEnabled(true)); + veniceAdmin.updateStore( + clusterName, + storeName, + new UpdateStoreQueryParams().setNativeReplicationEnabled(true).setActiveActiveReplicationEnabled(true)); /** * Add version 1 @@ -1888,7 +1867,10 @@ public void testUpdateStoreWithVersionInheritedConfigs() { veniceAdmin.updateStore( clusterName, storeName, - new UpdateStoreQueryParams().setHybridOffsetLagThreshold(1) + new UpdateStoreQueryParams().setNativeReplicationEnabled(true) + .setActiveActiveReplicationEnabled(true) + .setChunkingEnabled(true) + .setHybridOffsetLagThreshold(1) .setHybridRewindSeconds(1) .setStoreViews(viewConfig)); veniceAdmin.incrementVersionIdempotent(clusterName, storeName, Version.guidBasedDummyPushId(), 1, 1); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java index 900d44efe6..35096a31a2 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java @@ -2,7 +2,7 @@ import static com.linkedin.venice.ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE; import static com.linkedin.venice.ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE; -import static com.linkedin.venice.ConfigKeys.CONTROLLER_PARENT_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED; +import static com.linkedin.venice.ConfigKeys.CONTROLLER_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED; import static com.linkedin.venice.ConfigKeys.TERMINAL_STATE_TOPIC_CHECK_DELAY_MS; import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS; import static com.linkedin.venice.controller.SchemaConstants.BAD_VALUE_SCHEMA_FOR_WRITE_COMPUTE_V2; @@ -10,8 +10,6 @@ import static com.linkedin.venice.controller.SchemaConstants.VALUE_SCHEMA_FOR_WRITE_COMPUTE_V3; import static com.linkedin.venice.controller.SchemaConstants.VALUE_SCHEMA_FOR_WRITE_COMPUTE_V4; import static com.linkedin.venice.controller.SchemaConstants.VALUE_SCHEMA_FOR_WRITE_COMPUTE_V5; -import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.CHILD_REGION_NAME_PREFIX; -import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_PARENT_DATA_CENTER_REGION_NAME; import static com.linkedin.venice.utils.TestUtils.assertCommand; import static com.linkedin.venice.utils.TestUtils.waitForNonDeterministicAssertion; import static com.linkedin.venice.utils.TestUtils.waitForNonDeterministicPushCompletion; @@ -32,14 +30,11 @@ import com.linkedin.venice.controllerapi.StoreResponse; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; -import com.linkedin.venice.integration.utils.PubSubBrokerConfigs; -import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; -import com.linkedin.venice.integration.utils.VeniceControllerCreateOptions; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; -import com.linkedin.venice.integration.utils.ZkServerWrapper; import com.linkedin.venice.meta.ETLStoreConfig; import com.linkedin.venice.meta.HybridStoreConfig; import com.linkedin.venice.meta.StoreInfo; @@ -350,7 +345,7 @@ public void testHybridAndETLStoreConfig() { Assert.assertFalse(etlStoreConfig.isFutureVersionETLEnabled()); Assert.assertTrue( controllerResponse.getError() - .contains("Cannot enable ETL for this store " + "because etled user proxy account is not set")); + .contains("Cannot enable ETL for this store because etled user proxy account is not set")); // test enabling ETL with empty proxy account, expected failure params = new UpdateStoreQueryParams(); @@ -362,7 +357,7 @@ public void testHybridAndETLStoreConfig() { Assert.assertFalse(etlStoreConfig.isFutureVersionETLEnabled()); Assert.assertTrue( controllerResponse.getError() - .contains("Cannot enable ETL for this store " + "because etled user proxy account is not set")); + .contains("Cannot enable ETL for this store because etled user proxy account is not set")); // test enabling ETL with etl proxy account, expected success params = new UpdateStoreQueryParams(); @@ -409,7 +404,7 @@ public void testSupersetSchemaWithCustomSupersetSchemaGenerator() throws IOExcep // This cluster setup don't have server, we cannot perform push here. properties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, String.valueOf(false)); properties.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, String.valueOf(false)); - properties.setProperty(CONTROLLER_PARENT_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED, String.valueOf(true)); + properties.setProperty(CONTROLLER_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED, String.valueOf(true)); properties .put(VeniceControllerWrapper.SUPERSET_SCHEMA_GENERATOR, new SupersetSchemaGeneratorWithCustomProp(CUSTOM_PROP)); @@ -444,8 +439,11 @@ public void testSupersetSchemaWithCustomSupersetSchemaGenerator() throws IOExcep Assert.assertNotNull(newStoreResponse); Assert.assertFalse(newStoreResponse.isError(), "error in newStoreResponse: " + newStoreResponse.getError()); // Enable write compute - ControllerResponse updateStoreResponse = parentControllerClient - .updateStore(storeName, new UpdateStoreQueryParams().setWriteComputationEnabled(true)); + ControllerResponse updateStoreResponse = parentControllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setHybridRewindSeconds(86400) + .setHybridOffsetLagThreshold(1000) + .setWriteComputationEnabled(true)); Assert.assertFalse(updateStoreResponse.isError()); MultiSchemaResponse schemaResponse = parentControllerClient.getAllValueSchema(storeName); @@ -560,40 +558,32 @@ public static Object[][] controllerSSLAndSupersetSchemaGenerator() { public void testStoreMetaDataUpdateFromParentToChildController( boolean isControllerSslEnabled, boolean isSupersetSchemaGeneratorEnabled) throws IOException { - Properties properties = new Properties(); + Properties parentControllerProps = new Properties(); // This cluster setup don't have server, we cannot perform push here. - properties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, String.valueOf(false)); - properties.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, String.valueOf(false)); + parentControllerProps.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, String.valueOf(false)); + parentControllerProps + .setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, String.valueOf(false)); if (isSupersetSchemaGeneratorEnabled) { - properties.setProperty(CONTROLLER_PARENT_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED, String.valueOf(true)); - properties.put( + parentControllerProps.setProperty(CONTROLLER_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED, String.valueOf(true)); + parentControllerProps.put( VeniceControllerWrapper.SUPERSET_SCHEMA_GENERATOR, new SupersetSchemaGeneratorWithCustomProp("test_prop")); } - try (ZkServerWrapper zkServer = ServiceFactory.getZkServer(); - PubSubBrokerWrapper pubSubBrokerWrapper = ServiceFactory.getPubSubBroker( - new PubSubBrokerConfigs.Builder().setZkWrapper(zkServer) - .setRegionName(DEFAULT_PARENT_DATA_CENTER_REGION_NAME) - .build()); - VeniceControllerWrapper childControllerWrapper = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder(clusterName, zkServer, pubSubBrokerWrapper) - .sslToKafka(isControllerSslEnabled) - .regionName(CHILD_REGION_NAME_PREFIX + "0") - .build()); - ZkServerWrapper parentZk = ServiceFactory.getZkServer(); - VeniceControllerWrapper parentControllerWrapper = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder(clusterName, parentZk, pubSubBrokerWrapper) - .childControllers(new VeniceControllerWrapper[] { childControllerWrapper }) - .extraProperties(properties) + try (VeniceTwoLayerMultiRegionMultiClusterWrapper venice = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .parentControllerProperties(parentControllerProps) .sslToKafka(isControllerSslEnabled) .build())) { - String childControllerUrl = isControllerSslEnabled - ? childControllerWrapper.getSecureControllerUrl() - : childControllerWrapper.getControllerUrl(); - String parentControllerUrl = isControllerSslEnabled - ? parentControllerWrapper.getSecureControllerUrl() - : parentControllerWrapper.getControllerUrl(); + String childControllerUrl = venice.getChildRegions().get(0).getControllerConnectString(); + String parentControllerUrl = venice.getControllerConnectString(); Optional sslFactory = isControllerSslEnabled ? Optional.of(SslUtils.getVeniceLocalSslFactory()) : Optional.empty(); try (ControllerClient parentControllerClient = new ControllerClient(clusterName, parentControllerUrl, sslFactory); @@ -1024,7 +1014,9 @@ private void testWriteComputeSchemaAutoGenerationFailure(ControllerClient parent private void validateEnablingWriteComputeFailed(String storeName, ControllerClient parentControllerClient) { UpdateStoreQueryParams updateStoreQueryParams = new UpdateStoreQueryParams(); - updateStoreQueryParams.setWriteComputationEnabled(true); + updateStoreQueryParams.setHybridRewindSeconds(86400) + .setHybridOffsetLagThreshold(1000) + .setWriteComputationEnabled(true); ControllerResponse response = parentControllerClient.updateStore(storeName, updateStoreQueryParams); Assert.assertTrue( response.isError(), @@ -1051,7 +1043,9 @@ private void testWriteComputeSchemaAutoGeneration(ControllerClient parentControl // Step 2. Update this store to enable write compute. UpdateStoreQueryParams updateStoreQueryParams = new UpdateStoreQueryParams(); - updateStoreQueryParams.setWriteComputationEnabled(true); + updateStoreQueryParams.setHybridOffsetLagThreshold(1000) + .setHybridRewindSeconds(86400) + .setWriteComputationEnabled(true); parentControllerClient.updateStore(storeName, updateStoreQueryParams); // Step 3. Get value schema and write compute schema generated by the controller. @@ -1114,7 +1108,9 @@ private void testWriteComputeSchemaEnable(ControllerClient parentControllerClien // Step 2. Update this store to enable write compute. UpdateStoreQueryParams updateStoreQueryParams = new UpdateStoreQueryParams(); - updateStoreQueryParams.setWriteComputationEnabled(true); + updateStoreQueryParams.setHybridOffsetLagThreshold(1000) + .setHybridRewindSeconds(86400) + .setWriteComputationEnabled(true); parentControllerClient.updateStore(storeName, updateStoreQueryParams); // Could not enable write compute bad schema did not have defaults diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/kafka/consumer/AdminConsumptionTaskIntegrationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/kafka/consumer/AdminConsumptionTaskIntegrationTest.java index 50fc16f8d7..f8b36fa549 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/kafka/consumer/AdminConsumptionTaskIntegrationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/kafka/consumer/AdminConsumptionTaskIntegrationTest.java @@ -2,9 +2,8 @@ import static com.linkedin.venice.ConfigKeys.ADMIN_CONSUMPTION_CYCLE_TIMEOUT_MS; import static com.linkedin.venice.ConfigKeys.ADMIN_CONSUMPTION_MAX_WORKER_THREAD_POOL_SIZE; -import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.STANDALONE_REGION_NAME; -import static com.linkedin.venice.pubsub.PubSubConstants.PUBSUB_OPERATION_TIMEOUT_MS_DEFAULT_VALUE; +import com.linkedin.venice.controller.Admin; import com.linkedin.venice.controller.VeniceHelixAdmin; import com.linkedin.venice.controller.kafka.AdminTopicUtils; import com.linkedin.venice.controller.kafka.protocol.admin.AdminOperation; @@ -17,12 +16,11 @@ import com.linkedin.venice.controller.kafka.protocol.enums.SchemaType; import com.linkedin.venice.controller.kafka.protocol.serializer.AdminOperationSerializer; import com.linkedin.venice.controllerapi.ControllerClient; -import com.linkedin.venice.integration.utils.PubSubBrokerConfigs; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; -import com.linkedin.venice.integration.utils.VeniceControllerCreateOptions; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; -import com.linkedin.venice.integration.utils.ZkServerWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; +import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; import com.linkedin.venice.pubsub.PubSubTopicRepository; import com.linkedin.venice.pubsub.api.PubSubTopic; @@ -49,42 +47,40 @@ @Test(priority = -5) public class AdminConsumptionTaskIntegrationTest { private static final int TIMEOUT = 1 * Time.MS_PER_MINUTE; - - private String clusterName = Utils.getUniqueString("test-cluster"); private final AdminOperationSerializer adminOperationSerializer = new AdminOperationSerializer(); private static final String owner = "test_owner"; private static final String keySchema = "\"string\""; private static final String valueSchema = "\"string\""; - private Properties extraProperties = new Properties(); - private final PubSubTopicRepository pubSubTopicRepository = new PubSubTopicRepository(); - /** * This test is flaky on slower hardware, with a short timeout ): */ @Test(timeOut = TIMEOUT) public void testSkipMessageEndToEnd() throws ExecutionException, InterruptedException, IOException { - try (ZkServerWrapper zkServer = ServiceFactory.getZkServer(); - PubSubBrokerWrapper pubSubBrokerWrapper = ServiceFactory.getPubSubBroker( - new PubSubBrokerConfigs.Builder().setZkWrapper(zkServer).setRegionName(STANDALONE_REGION_NAME).build()); - TopicManager topicManager = - IntegrationTestPushUtils - .getTopicManagerRepo( - PUBSUB_OPERATION_TIMEOUT_MS_DEFAULT_VALUE, - 100, - 0l, - pubSubBrokerWrapper, - pubSubTopicRepository) - .getLocalTopicManager()) { + try ( + VeniceTwoLayerMultiRegionMultiClusterWrapper venice = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .build()); + ControllerClient parentControllerClient = new ControllerClient( + venice.getClusterNames()[0], + venice.getParentControllers().get(0).getControllerUrl())) { + String clusterName = venice.getClusterNames()[0]; + Admin admin = venice.getParentControllers().get(0).getVeniceAdmin(); + PubSubTopicRepository pubSubTopicRepository = admin.getPubSubTopicRepository(); + TopicManager topicManager = admin.getTopicManager(); PubSubTopic adminTopic = pubSubTopicRepository.getTopic(AdminTopicUtils.getTopicNameFromClusterName(clusterName)); topicManager.createTopic(adminTopic, 1, 1, true); String storeName = "test-store"; + PubSubBrokerWrapper pubSubBrokerWrapper = venice.getParentKafkaBrokerWrapper(); try ( - VeniceControllerWrapper controller = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder(clusterName, zkServer, pubSubBrokerWrapper) - .regionName(STANDALONE_REGION_NAME) - .build()); PubSubProducerAdapterFactory pubSubProducerAdapterFactory = pubSubBrokerWrapper.getPubSubClientsFactory().getProducerAdapterFactory(); VeniceWriter writer = @@ -99,13 +95,11 @@ public void testSkipMessageEndToEnd() throws ExecutionException, InterruptedExce writer.put(new byte[0], goodMessage, AdminOperationSerializer.LATEST_SCHEMA_ID_FOR_ADMIN_OPERATION); Thread.sleep(5000); // Non-deterministic, but whatever. This should never fail. - Assert.assertFalse(controller.getVeniceAdmin().hasStore(clusterName, storeName)); + Assert.assertTrue(parentControllerClient.getStore(storeName).isError()); - try (ControllerClient controllerClient = new ControllerClient(clusterName, controller.getControllerUrl())) { - controllerClient.skipAdminMessage(Long.toString(badOffset), false); - } + parentControllerClient.skipAdminMessage(Long.toString(badOffset), false); TestUtils.waitForNonDeterministicAssertion(TIMEOUT * 3, TimeUnit.MILLISECONDS, () -> { - Assert.assertTrue(controller.getVeniceAdmin().hasStore(clusterName, storeName)); + Assert.assertFalse(parentControllerClient.getStore(storeName).isError()); }); } } @@ -113,30 +107,36 @@ public void testSkipMessageEndToEnd() throws ExecutionException, InterruptedExce @Test(timeOut = TIMEOUT) public void testParallelAdminExecutionTasks() throws IOException, InterruptedException { - try (ZkServerWrapper zkServer = ServiceFactory.getZkServer(); - PubSubBrokerWrapper pubSubBrokerWrapper = ServiceFactory.getPubSubBroker( - new PubSubBrokerConfigs.Builder().setZkWrapper(zkServer).setRegionName(STANDALONE_REGION_NAME).build()); - TopicManager topicManager = - IntegrationTestPushUtils - .getTopicManagerRepo( - PUBSUB_OPERATION_TIMEOUT_MS_DEFAULT_VALUE, - 100, - 0l, - pubSubBrokerWrapper, - pubSubTopicRepository) - .getLocalTopicManager()) { + int adminConsumptionMaxWorkerPoolSize = 3; + + Properties parentControllerProps = new Properties(); + parentControllerProps.put(ADMIN_CONSUMPTION_MAX_WORKER_THREAD_POOL_SIZE, adminConsumptionMaxWorkerPoolSize); + parentControllerProps.put(ADMIN_CONSUMPTION_CYCLE_TIMEOUT_MS, 3000); + + try ( + VeniceTwoLayerMultiRegionMultiClusterWrapper venice = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .parentControllerProperties(parentControllerProps) + .build()); + ControllerClient parentControllerClient = new ControllerClient( + venice.getClusterNames()[0], + venice.getParentControllers().get(0).getControllerUrl())) { + String clusterName = venice.getClusterNames()[0]; + Admin admin = venice.getParentControllers().get(0).getVeniceAdmin(); + PubSubTopicRepository pubSubTopicRepository = admin.getPubSubTopicRepository(); + TopicManager topicManager = admin.getTopicManager(); PubSubTopic adminTopic = pubSubTopicRepository.getTopic(AdminTopicUtils.getTopicNameFromClusterName(clusterName)); topicManager.createTopic(adminTopic, 1, 1, true); String storeName = "test-store"; - int adminConsumptionMaxWorkerPoolSize = 3; - extraProperties.put(ADMIN_CONSUMPTION_MAX_WORKER_THREAD_POOL_SIZE, adminConsumptionMaxWorkerPoolSize); - extraProperties.put(ADMIN_CONSUMPTION_CYCLE_TIMEOUT_MS, 3000); + PubSubBrokerWrapper pubSubBrokerWrapper = venice.getParentKafkaBrokerWrapper(); try ( - VeniceControllerWrapper controller = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder(clusterName, zkServer, pubSubBrokerWrapper) - .regionName(STANDALONE_REGION_NAME) - .extraProperties(extraProperties) - .build()); PubSubProducerAdapterFactory pubSubProducerAdapterFactory = pubSubBrokerWrapper.getPubSubClientsFactory().getProducerAdapterFactory(); VeniceWriter writer = @@ -148,12 +148,12 @@ public void testParallelAdminExecutionTasks() throws IOException, InterruptedExc writer.put(new byte[0], goodMessage, AdminOperationSerializer.LATEST_SCHEMA_ID_FOR_ADMIN_OPERATION); TestUtils.waitForNonDeterministicAssertion(TIMEOUT, TimeUnit.MILLISECONDS, () -> { - Assert.assertTrue(controller.getVeniceAdmin().hasStore(clusterName, storeName)); + Assert.assertTrue(parentControllerClient.getStore(storeName).isError()); }); // Spin up a thread to occupy the store write lock to simulate the blocking admin execution task thread. CountDownLatch lockOccupyThreadStartedSignal = new CountDownLatch(1); - Runnable infiniteLockOccupy = getRunnable(controller, storeName, lockOccupyThreadStartedSignal); + Runnable infiniteLockOccupy = getRunnable(venice, storeName, lockOccupyThreadStartedSignal); Thread infiniteLockThread = new Thread(infiniteLockOccupy, "infiniteLockOccupy: " + storeName); infiniteLockThread.start(); Assert.assertTrue(lockOccupyThreadStartedSignal.await(5, TimeUnit.SECONDS)); @@ -181,7 +181,7 @@ public void testParallelAdminExecutionTasks() throws IOException, InterruptedExc writer.put(new byte[0], otherStoreMessage, AdminOperationSerializer.LATEST_SCHEMA_ID_FOR_ADMIN_OPERATION); TestUtils.waitForNonDeterministicAssertion(TIMEOUT, TimeUnit.MILLISECONDS, () -> { - Assert.assertTrue(controller.getVeniceAdmin().hasStore(clusterName, otherStoreName)); + Assert.assertFalse(parentControllerClient.getStore(storeName).isError()); }); infiniteLockThread.interrupt(); // This will release the lock @@ -190,14 +190,19 @@ public void testParallelAdminExecutionTasks() throws IOException, InterruptedExc byte[] storeDeletionMessage = getStoreDeletionMessage(clusterName, storeName, executionId); writer.put(new byte[0], storeDeletionMessage, AdminOperationSerializer.LATEST_SCHEMA_ID_FOR_ADMIN_OPERATION); TestUtils.waitForNonDeterministicAssertion(TIMEOUT, TimeUnit.MILLISECONDS, () -> { - Assert.assertFalse(controller.getVeniceAdmin().hasStore(clusterName, storeName)); + Assert.assertFalse(parentControllerClient.getStore(storeName).isError()); }); } } } - private Runnable getRunnable(VeniceControllerWrapper controller, String storeName, CountDownLatch latch) { - VeniceHelixAdmin admin = controller.getVeniceHelixAdmin(); + private Runnable getRunnable( + VeniceTwoLayerMultiRegionMultiClusterWrapper venice, + String storeName, + CountDownLatch latch) { + String clusterName = venice.getClusterNames()[0]; + VeniceControllerWrapper parentController = venice.getParentControllers().get(0); + VeniceHelixAdmin admin = parentController.getVeniceHelixAdmin(); return () -> { try (AutoCloseableLock ignore = admin.getHelixVeniceClusterResources(clusterName).getClusterLockManager().createStoreWriteLock(storeName)) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/AbstractTestAdminSparkServer.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/AbstractTestAdminSparkServer.java index d37e2113f1..6573d914c7 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/AbstractTestAdminSparkServer.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/AbstractTestAdminSparkServer.java @@ -6,10 +6,9 @@ import com.linkedin.venice.authorization.AuthorizerService; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.integration.utils.ServiceFactory; -import com.linkedin.venice.integration.utils.VeniceClusterWrapper; -import com.linkedin.venice.integration.utils.VeniceControllerCreateOptions; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; -import com.linkedin.venice.integration.utils.ZkServerWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; +import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; @@ -25,48 +24,60 @@ public class AbstractTestAdminSparkServer { protected static final int TEST_TIMEOUT = 300 * Time.MS_PER_SECOND; protected static final int STORAGE_NODE_COUNT = 1; - protected VeniceClusterWrapper cluster; + protected VeniceTwoLayerMultiRegionMultiClusterWrapper venice; protected ControllerClient controllerClient; protected VeniceControllerWrapper parentController; - protected ZkServerWrapper parentZk; + protected ControllerClient parentControllerClient; public void setUp( boolean useParentRestEndpoint, Optional authorizerService, Properties extraProperties) { - cluster = ServiceFactory.getVeniceCluster(1, STORAGE_NODE_COUNT, 0, 1, 100, false, false, extraProperties); + Properties parentControllerProps = new Properties(); + parentControllerProps.putAll(extraProperties); + Properties childControllerProps = new Properties(); + childControllerProps.putAll(extraProperties); - parentZk = ServiceFactory.getZkServer(); // The cluster does not have router setup - extraProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, "false"); - extraProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, "false"); - VeniceControllerCreateOptions options = - new VeniceControllerCreateOptions.Builder(cluster.getClusterName(), parentZk, cluster.getPubSubBrokerWrapper()) + parentControllerProps.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, "false"); + parentControllerProps.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, "false"); + + childControllerProps.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, "false"); + childControllerProps.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, "false"); + + venice = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(STORAGE_NODE_COUNT) + .numberOfRouters(0) .replicationFactor(1) - .childControllers(new VeniceControllerWrapper[] { cluster.getLeaderVeniceController() }) - .extraProperties(extraProperties) - .authorizerService(authorizerService.orElse(null)) - .build(); - parentController = ServiceFactory.getVeniceController(options); + .parentControllerProperties(parentControllerProps) + .childControllerProperties(childControllerProps) + .serverProperties(extraProperties) + .parentAuthorizerService(authorizerService.orElse(null)) + .build()); + parentController = venice.getParentControllers().get(0); + parentControllerClient = new ControllerClient(venice.getClusterNames()[0], parentController.getControllerUrl()); + String clusterName = venice.getClusterNames()[0]; if (!useParentRestEndpoint) { - controllerClient = - ControllerClient.constructClusterControllerClient(cluster.getClusterName(), cluster.getAllControllersURLs()); - } else { controllerClient = ControllerClient - .constructClusterControllerClient(cluster.getClusterName(), parentController.getControllerUrl()); + .constructClusterControllerClient(clusterName, venice.getChildRegions().get(0).getControllerConnectString()); + } else { + controllerClient = + ControllerClient.constructClusterControllerClient(clusterName, parentController.getControllerUrl()); } TestUtils.waitForNonDeterministicCompletion( TEST_TIMEOUT, TimeUnit.MILLISECONDS, - () -> parentController.isLeaderController(cluster.getClusterName())); + () -> parentController.isLeaderController(clusterName)); } public void cleanUp() { - Utils.closeQuietlyWithErrorLogged(parentController); Utils.closeQuietlyWithErrorLogged(controllerClient); - Utils.closeQuietlyWithErrorLogged(cluster); - Utils.closeQuietlyWithErrorLogged(parentZk); + Utils.closeQuietlyWithErrorLogged(venice); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServer.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServer.java index 27264ef753..24e0dcecd6 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServer.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServer.java @@ -1,5 +1,7 @@ package com.linkedin.venice.controller.server; +import static com.linkedin.venice.utils.TestUtils.assertCommand; + import com.linkedin.venice.ConfigKeys; import com.linkedin.venice.LastSucceedExecutionIdResponse; import com.linkedin.venice.common.VeniceSystemStoreType; @@ -30,8 +32,12 @@ import com.linkedin.venice.exceptions.ErrorType; import com.linkedin.venice.exceptions.ExceptionType; import com.linkedin.venice.httpclient.HttpClientUtils; +import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; +import com.linkedin.venice.meta.DataReplicationPolicy; +import com.linkedin.venice.meta.HybridStoreConfig; import com.linkedin.venice.meta.InstanceStatus; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.StoreInfo; @@ -41,6 +47,7 @@ import com.linkedin.venice.utils.EncodingUtils; import com.linkedin.venice.utils.ObjectMapperFactory; import com.linkedin.venice.utils.TestUtils; +import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; import java.io.IOException; import java.util.ArrayList; @@ -60,6 +67,7 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.message.BasicNameValuePair; +import org.apache.logging.log4j.LogManager; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -115,7 +123,11 @@ public void controllerClientCanQueryInstanceStatusInCluster() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanQueryReplicasOnAStorageNode() { - String storeName = cluster.getNewStoreVersion().getName(); + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + assertCommand( + parentControllerClient + .sendEmptyPushAndWait(storeName, Utils.getUniqueString(storeName), 1024, 60 * Time.MS_PER_SECOND)); try { MultiNodeResponse nodeResponse = controllerClient.listStorageNodes(); String nodeId = nodeResponse.getNodes()[0]; @@ -128,14 +140,16 @@ public void controllerClientCanQueryReplicasOnAStorageNode() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanQueryReplicasForTopic() { - VersionCreationResponse versionCreationResponse = cluster.getNewStoreVersion(); + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); - String storeName = versionCreationResponse.getName(); try { - String kafkaTopic = cluster.getNewStoreVersion().getKafkaTopic(); + String kafkaTopic = versionCreationResponse.getKafkaTopic(); Assert.assertNotNull( kafkaTopic, - "venice.getNewStoreVersion() should not return a null topic name\n" + versionCreationResponse.toString()); + "parentControllerClient.emptyPush should not return a null topic name\n" + versionCreationResponse); String store = Version.parseStoreFromKafkaTopicName(kafkaTopic); int version = Version.parseVersionFromKafkaTopicName(kafkaTopic); @@ -155,11 +169,12 @@ public void controllerClientCanCreateNewStore() throws IOException, ExecutionExc String valueSchema = "\"long\""; // create Store - NewStoreResponse newStoreResponse = controllerClient.createNewStore(storeToCreate, "owner", keySchema, valueSchema); + NewStoreResponse newStoreResponse = + parentControllerClient.createNewStore(storeToCreate, "owner", keySchema, valueSchema); Assert.assertFalse(newStoreResponse.isError(), "create new store should succeed for a store that doesn't exist"); try { NewStoreResponse duplicateNewStoreResponse = - controllerClient.createNewStore(storeToCreate, "owner", keySchema, valueSchema); + parentControllerClient.createNewStore(storeToCreate, "owner", keySchema, valueSchema); Assert .assertTrue(duplicateNewStoreResponse.isError(), "create new store should fail for duplicate store creation"); @@ -167,12 +182,12 @@ public void controllerClientCanCreateNewStore() throws IOException, ExecutionExc CloseableHttpAsyncClient httpClient = HttpClientUtils.getMinimalHttpClient(1, 1, Optional.empty()); httpClient.start(); List params = new ArrayList<>(); - params.add(new BasicNameValuePair(ControllerApiConstants.CLUSTER, cluster.getClusterName())); + params.add(new BasicNameValuePair(ControllerApiConstants.CLUSTER, venice.getClusterNames()[0])); params.add(new BasicNameValuePair(ControllerApiConstants.NAME, storeToCreate)); params.add(new BasicNameValuePair(ControllerApiConstants.OWNER, "owner")); params.add(new BasicNameValuePair(ControllerApiConstants.KEY_SCHEMA, keySchema)); params.add(new BasicNameValuePair(ControllerApiConstants.VALUE_SCHEMA, valueSchema)); - final HttpPost post = new HttpPost(cluster.getAllControllersURLs() + ControllerRoute.NEW_STORE.getPath()); + final HttpPost post = new HttpPost(venice.getControllerConnectString() + ControllerRoute.NEW_STORE.getPath()); post.setEntity(new UrlEncodedFormEntity(params)); HttpResponse duplicateStoreCreationHttpResponse = httpClient.execute(post, null).get(); Assert.assertEquals( @@ -195,13 +210,13 @@ public void controllerClientGetKeySchema() { SchemaResponse sr0 = controllerClient.getKeySchema(storeToCreate); Assert.assertTrue(sr0.isError()); // Create Store - NewStoreResponse newStoreResponse = - controllerClient.createNewStore(storeToCreate, "owner", keySchemaStr, valueSchemaStr); + assertCommand(parentControllerClient.createNewStore(storeToCreate, "owner", keySchemaStr, valueSchemaStr)); try { - Assert.assertFalse(newStoreResponse.isError(), "create new store should succeed for a store that doesn't exist"); - SchemaResponse sr1 = controllerClient.getKeySchema(storeToCreate); - Assert.assertEquals(sr1.getId(), 1); - Assert.assertEquals(sr1.getSchemaStr(), keySchemaStr); + TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, false, true, () -> { + SchemaResponse sr1 = assertCommand(controllerClient.getKeySchema(storeToCreate)); + Assert.assertEquals(sr1.getId(), 1); + Assert.assertEquals(sr1.getSchemaStr(), keySchemaStr); + }); } finally { // clear the store since the cluster is shared by other test cases deleteStore(storeToCreate); @@ -238,28 +253,29 @@ public void controllerClientManageValueSchema() { String incompatibleSchema = "\"string\""; // Add value schema to non-existed store - SchemaResponse sr0 = controllerClient.addValueSchema(storeToCreate, schema1); + SchemaResponse sr0 = parentControllerClient.addValueSchema(storeToCreate, schema1); Assert.assertTrue(sr0.isError()); // Add value schema to an existing store - NewStoreResponse newStoreResponse = controllerClient.createNewStore(storeToCreate, "owner", keySchemaStr, schema1); + NewStoreResponse newStoreResponse = + parentControllerClient.createNewStore(storeToCreate, "owner", keySchemaStr, schema1); Assert.assertFalse(newStoreResponse.isError(), "create new store should succeed for a store that doesn't exist"); try { - SchemaResponse sr1 = controllerClient.addValueSchema(storeToCreate, schema1); + SchemaResponse sr1 = parentControllerClient.addValueSchema(storeToCreate, schema1); Assert.assertFalse(sr1.isError()); Assert.assertEquals(sr1.getId(), 1); // Add same value schema - SchemaResponse sr2 = controllerClient.addValueSchema(storeToCreate, schema1); + SchemaResponse sr2 = parentControllerClient.addValueSchema(storeToCreate, schema1); Assert.assertFalse(sr2.isError()); Assert.assertEquals(sr2.getId(), sr1.getId()); // Add a new value schema - SchemaResponse sr3 = controllerClient.addValueSchema(storeToCreate, schema2); + SchemaResponse sr3 = parentControllerClient.addValueSchema(storeToCreate, schema2); Assert.assertFalse(sr3.isError()); Assert.assertEquals(sr3.getId(), 2); // Add invalid schema - SchemaResponse sr4 = controllerClient.addValueSchema(storeToCreate, invalidSchema); + SchemaResponse sr4 = parentControllerClient.addValueSchema(storeToCreate, invalidSchema); Assert.assertTrue(sr4.isError()); // Add incompatible schema - SchemaResponse sr5 = controllerClient.addValueSchema(storeToCreate, incompatibleSchema); + SchemaResponse sr5 = parentControllerClient.addValueSchema(storeToCreate, incompatibleSchema); Assert.assertTrue(sr5.isError()); Assert.assertEquals(sr5.getErrorType(), ErrorType.INVALID_SCHEMA); Assert.assertEquals(sr5.getExceptionType(), ExceptionType.INVALID_SCHEMA); @@ -268,30 +284,30 @@ public void controllerClientManageValueSchema() { String formattedSchemaStr1 = formatSchema(schema1); String formattedSchemaStr2 = formatSchema(schema2); // Get schema by id - SchemaResponse sr6 = controllerClient.getValueSchema(storeToCreate, 1); + SchemaResponse sr6 = parentControllerClient.getValueSchema(storeToCreate, 1); Assert.assertFalse(sr6.isError()); Assert.assertEquals(sr6.getSchemaStr(), formattedSchemaStr1); - SchemaResponse sr7 = controllerClient.getValueSchema(storeToCreate, 2); + SchemaResponse sr7 = parentControllerClient.getValueSchema(storeToCreate, 2); Assert.assertFalse(sr7.isError()); Assert.assertEquals(sr7.getSchemaStr(), formattedSchemaStr2); // Get schema by non-existed schema id - SchemaResponse sr8 = controllerClient.getValueSchema(storeToCreate, 3); + SchemaResponse sr8 = parentControllerClient.getValueSchema(storeToCreate, 3); Assert.assertTrue(sr8.isError()); // Get value schema by schema - SchemaResponse sr9 = controllerClient.getValueSchemaID(storeToCreate, schema1); + SchemaResponse sr9 = parentControllerClient.getValueSchemaID(storeToCreate, schema1); Assert.assertFalse(sr9.isError()); Assert.assertEquals(sr9.getId(), 1); - SchemaResponse sr10 = controllerClient.getValueSchemaID(storeToCreate, schema2); + SchemaResponse sr10 = parentControllerClient.getValueSchemaID(storeToCreate, schema2); Assert.assertFalse(sr10.isError()); Assert.assertEquals(sr10.getId(), 2); - SchemaResponse sr11 = controllerClient.getValueSchemaID(storeToCreate, invalidSchema); + SchemaResponse sr11 = parentControllerClient.getValueSchemaID(storeToCreate, invalidSchema); Assert.assertTrue(sr11.isError()); - SchemaResponse sr12 = controllerClient.getValueSchemaID(storeToCreate, incompatibleSchema); + SchemaResponse sr12 = parentControllerClient.getValueSchemaID(storeToCreate, incompatibleSchema); Assert.assertTrue(sr12.isError()); // Get all value schema - MultiSchemaResponse msr = controllerClient.getAllValueSchema(storeToCreate); + MultiSchemaResponse msr = parentControllerClient.getAllValueSchema(storeToCreate); Assert.assertFalse(msr.isError()); MultiSchemaResponse.Schema[] schemas = msr.getSchemas(); Assert.assertEquals(schemas.length, 2); @@ -307,19 +323,19 @@ public void controllerClientManageValueSchema() { String prefixForLotsOfSchemas = schemaPrefix + salaryFieldWithDefault; // add incorrect schema - sr1 = controllerClient.addValueSchema(storeToCreate, schemaStr); + sr1 = parentControllerClient.addValueSchema(storeToCreate, schemaStr); Assert.assertTrue(sr1.isError()); for (int i = 3; i < allSchemas.length; i++) { prefixForLotsOfSchemas += "," + " {\"name\": \"newField" + i + "\", \"type\": \"long\", \"default\": 123 }\n"; String schema = formatSchema(prefixForLotsOfSchemas + schemaSuffix); allSchemas[i - 1] = schema; - SchemaResponse sr = controllerClient.addValueSchema(storeToCreate, schema); + SchemaResponse sr = parentControllerClient.addValueSchema(storeToCreate, schema); Assert.assertFalse(sr.isError()); Assert.assertEquals(sr.getId(), i); // At each new schema we create, we test that the ordering is correct - MultiSchemaResponse msr2 = controllerClient.getAllValueSchema(storeToCreate); + MultiSchemaResponse msr2 = parentControllerClient.getAllValueSchema(storeToCreate); Assert.assertFalse(msr2.isError()); MultiSchemaResponse.Schema[] schemasFromController = msr2.getSchemas(); Assert.assertEquals( @@ -361,8 +377,11 @@ public void controllerClientSchemaOperationsAgainstInvalidStore() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanGetStoreInfo() { - String topic = cluster.getNewStoreVersion().getKafkaTopic(); - String storeName = Version.parseStoreFromKafkaTopicName(topic); + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); try { StoreResponse storeResponse = controllerClient.getStore(storeName); Assert.assertFalse(storeResponse.isError(), storeResponse.getError()); @@ -384,8 +403,11 @@ public void controllerClientCanGetStoreInfo() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanDisableStoresWrite() { - String topic = cluster.getNewStoreVersion().getKafkaTopic(); - String storeName = Version.parseStoreFromKafkaTopicName(topic); + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); try { StoreInfo store = controllerClient.getStore(storeName).getStore(); Assert.assertTrue(store.isEnableStoreWrites(), "Store should NOT be disabled after creating new store-version"); @@ -402,10 +424,11 @@ public void controllerClientCanDisableStoresWrite() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanDisableStoresRead() { - String topic = cluster.getNewStoreVersion().getKafkaTopic(); - - String storeName = Version.parseStoreFromKafkaTopicName(topic); - + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); try { StoreInfo store = controllerClient.getStore(storeName).getStore(); Assert.assertTrue(store.isEnableStoreReads(), "Store should NOT be disabled after creating new store-version"); @@ -423,9 +446,11 @@ public void controllerClientCanDisableStoresRead() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanDisableStoresReadWrite() { - String topic = cluster.getNewStoreVersion().getKafkaTopic(); - - String storeName = Version.parseStoreFromKafkaTopicName(topic); + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); try { StoreInfo store = controllerClient.getStore(storeName).getStore(); Assert.assertTrue(store.isEnableStoreReads(), "Store should NOT be disabled after creating new store-version"); @@ -448,20 +473,27 @@ public void controllerClientCanSetStoreMetadata() { String owner = Utils.getUniqueString("owner"); int partitionCount = 2; - cluster.getNewStore(storeName); + assertCommand(parentControllerClient.createNewStore(storeName, owner, "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); try { OwnerResponse ownerRes = controllerClient.setStoreOwner(storeName, owner); Assert.assertFalse(ownerRes.isError(), ownerRes.getError()); Assert.assertEquals(ownerRes.getOwner(), owner); - UpdateStoreQueryParams updateStoreQueryParams = - new UpdateStoreQueryParams().setPartitionCount(partitionCount).setIncrementalPushEnabled(true); - ControllerResponse partitionRes = controllerClient.updateStore(storeName, updateStoreQueryParams); + UpdateStoreQueryParams updateStoreQueryParams = new UpdateStoreQueryParams().setPartitionCount(partitionCount) + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) + .setHybridRewindSeconds(Time.SECONDS_PER_DAY) + .setHybridOffsetLagThreshold(1000); + ControllerResponse partitionRes = parentControllerClient.updateStore(storeName, updateStoreQueryParams); Assert.assertFalse(partitionRes.isError(), partitionRes.getError()); - StoreResponse storeResponse = controllerClient.getStore(storeName); - Assert.assertEquals(storeResponse.getStore().getPartitionCount(), partitionCount); - Assert.assertEquals(storeResponse.getStore().isIncrementalPushEnabled(), true); + TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, false, true, () -> { + StoreResponse storeResponse = controllerClient.getStore(storeName); + Assert.assertEquals(storeResponse.getStore().getPartitionCount(), partitionCount); + Assert.assertTrue(storeResponse.getStore().isIncrementalPushEnabled()); + }); } finally { deleteStore(storeName); } @@ -469,7 +501,10 @@ public void controllerClientCanSetStoreMetadata() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanQueryRemovability() { - VeniceServerWrapper server = cluster.getVeniceServers().get(0); + VeniceMultiClusterWrapper multiClusterWrapper = venice.getChildRegions().get(0); + String clusterName = multiClusterWrapper.getClusterNames()[0]; + VeniceClusterWrapper venice = multiClusterWrapper.getClusters().get(clusterName); + VeniceServerWrapper server = venice.getVeniceServers().get(0); String nodeId = Utils.getHelixNodeIdentifier(Utils.getHostName(), server.getPort()); ControllerResponse response = controllerClient.isNodeRemovable(nodeId); @@ -478,7 +513,11 @@ public void controllerClientCanQueryRemovability() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanDeleteAllVersion() { - String storeName = cluster.getNewStoreVersion().getName(); + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); try { controllerClient.enableStoreReads(storeName, false); controllerClient.enableStoreWrites(storeName, false); @@ -500,7 +539,11 @@ public void controllerClientCanDeleteAllVersion() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanDeleteOldVersion() { - String storeName = cluster.getNewStoreVersion().getName(); + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); try { VersionResponse response = controllerClient.deleteOldVersion(storeName, 1); Assert.assertFalse(response.isError(), response.getError()); @@ -522,7 +565,7 @@ public void controllerClientCanGetLastSucceedExecutionId() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanGetExecutionOfDeleteAllVersions() { - String clusterName = cluster.getClusterName(); + String clusterName = venice.getClusterNames()[0]; String storeName = Utils.getUniqueString("controllerClientCanDeleteAllVersion"); parentController.getVeniceAdmin().createStore(clusterName, storeName, "test", "\"string\"", "\"string\""); @@ -551,39 +594,41 @@ public void controllerClientCanListStoresStatuses() { String storePrefix = "controllerClientCanListStoresStatusesTestStore"; int storeCount = 2; for (int i = 0; i < storeCount; i++) { - storeNames.add(cluster.getNewStore(storePrefix + i).getName()); + String storeName = Utils.getUniqueString(storePrefix); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + storeNames.add(storeName); } try { - MultiStoreStatusResponse storeResponse = controllerClient.listStoresStatuses(); - Assert.assertFalse(storeResponse.isError()); - // since all test cases share VeniceClusterWrapper, we get the total number of stores from the Wrapper. - List storesInCluster = - storeResponse.getStoreStatusMap().entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); - for (String storeName: storeNames) { - Assert.assertTrue( - storesInCluster.contains(storeName), - "Result of listing store status should contain all stores we created."); - } - List storeStatuses = storeResponse.getStoreStatusMap() - .entrySet() - .stream() - .filter(e -> e.getKey().contains(storePrefix)) - .map(Map.Entry::getValue) - .collect(Collectors.toList()); - Assert.assertTrue(storeStatuses.size() == storeCount); - for (String status: storeStatuses) { - Assert.assertEquals( - status, - StoreStatus.UNAVAILABLE.toString(), - "Store should be unavailable because we have not created a version for this store. " - + storeResponse.getStoreStatusMap()); - } - for (String expectedStore: storeNames) { - Assert.assertTrue( - storeResponse.getStoreStatusMap().containsKey(expectedStore), - "Result of list store status should contain the store we created: " + expectedStore); - } + TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, false, true, () -> { + MultiStoreStatusResponse storeResponse = assertCommand(controllerClient.listStoresStatuses()); + // since all test cases share VeniceClusterWrapper, we get the total number of stores from the Wrapper. + List storesInCluster = new ArrayList<>(storeResponse.getStoreStatusMap().keySet()); + for (String storeName: storeNames) { + Assert.assertTrue( + storesInCluster.contains(storeName), + "Result of listing store status should contain all stores we created."); + } + List storeStatuses = storeResponse.getStoreStatusMap() + .entrySet() + .stream() + .filter(e -> e.getKey().contains(storePrefix)) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + Assert.assertEquals(storeStatuses.size(), storeCount); + for (String status: storeStatuses) { + Assert.assertEquals( + status, + StoreStatus.UNAVAILABLE.toString(), + "Store should be unavailable because we have not created a version for this store. " + + storeResponse.getStoreStatusMap()); + } + for (String expectedStore: storeNames) { + Assert.assertTrue( + storeResponse.getStoreStatusMap().containsKey(expectedStore), + "Result of list store status should contain the store we created: " + expectedStore); + } + }); } finally { storeNames.forEach(this::deleteStore); } @@ -591,13 +636,17 @@ public void controllerClientCanListStoresStatuses() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanListFutureStoreVersions() { + String clusterName = venice.getClusterNames()[0]; List storeNames = new ArrayList<>(); try { - ControllerClient parentControllerClient = ControllerClient - .constructClusterControllerClient(cluster.getClusterName(), parentController.getControllerUrl()); - storeNames.add(parentControllerClient.createNewStore("testStore", "owner", "\"string\"", "\"string\"").getName()); + ControllerClient parentControllerClient = + ControllerClient.constructClusterControllerClient(clusterName, parentController.getControllerUrl()); + String storeName = Utils.getUniqueString("testStore"); + NewStoreResponse newStoreResponse = + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + storeNames.add(newStoreResponse.getName()); MultiStoreStatusResponse storeResponse = - parentControllerClient.getFutureVersions(cluster.getClusterName(), storeNames.get(0)); + assertCommand(parentControllerClient.getFutureVersions(clusterName, storeNames.get(0))); // There's no version for this store and no future version coming, so we expect an entry with // Store.NON_EXISTING_VERSION @@ -610,20 +659,16 @@ public void controllerClientCanListFutureStoreVersions() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanUpdateAllowList() { - Admin admin = cluster.getLeaderVeniceController().getVeniceAdmin(); + String clusterName = venice.getClusterNames()[0]; + Admin admin = venice.getChildRegions().get(0).getLeaderController(clusterName).getVeniceAdmin(); String nodeId = Utils.getHelixNodeIdentifier(Utils.getHostName(), 34567); - Assert.assertFalse( - admin.getAllowlist(cluster.getClusterName()).contains(nodeId), - nodeId + " has not been added into allowlist."); + Assert + .assertFalse(admin.getAllowlist(clusterName).contains(nodeId), nodeId + " has not been added into allowlist."); controllerClient.addNodeIntoAllowList(nodeId); - Assert.assertTrue( - admin.getAllowlist(cluster.getClusterName()).contains(nodeId), - nodeId + " has been added into allowlist."); + Assert.assertTrue(admin.getAllowlist(clusterName).contains(nodeId), nodeId + " has been added into allowlist."); controllerClient.removeNodeFromAllowList(nodeId); - Assert.assertFalse( - admin.getAllowlist(cluster.getClusterName()).contains(nodeId), - nodeId + " has been removed from allowlist."); + Assert.assertFalse(admin.getAllowlist(clusterName).contains(nodeId), nodeId + " has been removed from allowlist."); } @Test(timeOut = TEST_TIMEOUT) @@ -639,7 +684,12 @@ public void controllerClientCanSetStore() { long readQuotaInCU = 200l; int numVersionToPreserve = 100; - String storeName = cluster.getNewStoreVersion().getName(); + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); + // Disable writes at first and test could we enable writes again through the update store method. Assert.assertFalse( controllerClient.enableStoreReadWrites(storeName, false).isError(), @@ -655,6 +705,10 @@ public void controllerClientCanSetStore() { .setAccessControlled(accessControlled) .setNumVersionsToPreserve(numVersionToPreserve); + VeniceMultiClusterWrapper multiClusterWrapper = venice.getChildRegions().get(0); + String clusterName = venice.getClusterNames()[0]; + VeniceClusterWrapper cluster = multiClusterWrapper.getClusters().get(clusterName); + try { ControllerResponse response = controllerClient.updateStore(storeName, queryParams); @@ -691,7 +745,11 @@ public void controllerClientCanSetStoreMissingSomeFields() { int current = 1; boolean enableReads = false; - storeName = cluster.getNewStoreVersion().getName(); + storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); ControllerResponse response = controllerClient.updateStore( storeName, new UpdateStoreQueryParams().setPartitionCount(partitionCount) @@ -699,10 +757,10 @@ public void controllerClientCanSetStoreMissingSomeFields() { .setEnableReads(enableReads)); Assert.assertFalse(response.isError(), response.getError()); - Store store = cluster.getLeaderVeniceController().getVeniceAdmin().getStore(cluster.getClusterName(), storeName); + StoreInfo store = controllerClient.getStore(storeName).getStore(); Assert.assertEquals(store.getPartitionCount(), partitionCount); Assert.assertEquals(store.getCurrentVersion(), current); - Assert.assertEquals(store.isEnableReads(), enableReads); + Assert.assertEquals(store.isEnableStoreReads(), enableReads); } finally { if (storeName != null) { deleteStore(storeName); @@ -714,14 +772,19 @@ public void controllerClientCanSetStoreMissingSomeFields() { public void canCreateAHybridStore() { String storeName = Utils.getUniqueString("store"); String owner = Utils.getUniqueString("owner"); - controllerClient.createNewStore(storeName, owner, "\"string\"", "\"string\""); + parentControllerClient.createNewStore(storeName, owner, "\"string\"", "\"string\""); try { - controllerClient.updateStore( + parentControllerClient.updateStore( storeName, new UpdateStoreQueryParams().setHybridRewindSeconds(123L).setHybridOffsetLagThreshold(1515L)); - StoreResponse storeResponse = controllerClient.getStore(storeName); - Assert.assertEquals(storeResponse.getStore().getHybridStoreConfig().getRewindTimeInSeconds(), 123L); - Assert.assertEquals(storeResponse.getStore().getHybridStoreConfig().getOffsetLagThresholdToGoOnline(), 1515L); + + TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, false, true, () -> { + StoreResponse storeResponse = controllerClient.getStore(storeName); + HybridStoreConfig hybridStoreConfig = storeResponse.getStore().getHybridStoreConfig(); + Assert.assertNotNull(hybridStoreConfig); + Assert.assertEquals(hybridStoreConfig.getRewindTimeInSeconds(), 123L); + Assert.assertEquals(hybridStoreConfig.getOffsetLagThresholdToGoOnline(), 1515L); + }); } finally { deleteStore(storeName); } @@ -729,7 +792,11 @@ public void canCreateAHybridStore() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanGetStorageEngineOverheadRatio() { - String storeName = cluster.getNewStoreVersion().getName(); + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); try { StorageEngineOverheadRatioResponse response = controllerClient.getStorageEngineOverheadRatio(storeName); @@ -744,7 +811,11 @@ public void controllerClientCanGetStorageEngineOverheadRatio() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanDeleteStore() { - String storeName = cluster.getNewStoreVersion().getName(); + String storeName = Utils.getUniqueString("test-store"); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + VersionCreationResponse versionCreationResponse = + parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1024); + Assert.assertFalse(versionCreationResponse.isError(), versionCreationResponse.getError()); try { controllerClient.enableStoreReads(storeName, false); controllerClient.enableStoreWrites(storeName, false); @@ -765,7 +836,7 @@ public void controllerClientCanDeleteStore() { @Test(timeOut = TEST_TIMEOUT) public void controllerClientCanGetExecutionOfDeleteStore() { - String clusterName = cluster.getClusterName(); + String clusterName = venice.getClusterNames()[0]; String storeName = Utils.getUniqueString("controllerClientCanGetExecutionOfDeleteStore"); parentController.getVeniceAdmin().createStore(clusterName, storeName, "test", "\"string\"", "\"string\""); @@ -837,7 +908,6 @@ public void controllerClientCanEnableThrottling() { Assert.assertFalse(controllerClient.getRoutersClusterConfig().getConfig().isThrottlingEnabled()); controllerClient.enableThrottling(true); Assert.assertTrue(controllerClient.getRoutersClusterConfig().getConfig().isThrottlingEnabled()); - } @Test(timeOut = TEST_TIMEOUT) @@ -852,12 +922,17 @@ public void controllerClientCanEnableMaxCapacityProtection() { public void controllerClientCanDiscoverCluster() { String storeName = Utils.getUniqueString("controllerClientCanDiscoverCluster"); controllerClient.createNewStore(storeName, "test", "\"string\"", "\"string\""); + String clusterName = venice.getClusterNames()[0]; try { Assert.assertEquals( ControllerClient - .discoverCluster(cluster.getLeaderVeniceController().getControllerUrl(), storeName, Optional.empty(), 1) + .discoverCluster( + venice.getChildRegions().get(0).getControllerConnectString(), + storeName, + Optional.empty(), + 1) .getCluster(), - cluster.getClusterName(), + clusterName, "Should be able to find the cluster which the given store belongs to."); } finally { deleteStore(storeName); @@ -874,9 +949,9 @@ public void controllerCanHandleLargePayload() throws IOException { String largeDictionary = EncodingUtils.base64EncodeToString(largeDictionaryBytes); - controllerClient.createNewStore(storeName, "test", "\"string\"", "\"string\""); + parentControllerClient.createNewStore(storeName, "test", "\"string\"", "\"string\""); - VersionCreationResponse vcr = controllerClient.requestTopicForWrites( + VersionCreationResponse vcr = parentControllerClient.requestTopicForWrites( storeName, 1L, Version.PushType.BATCH, @@ -898,49 +973,46 @@ public void controllerCanHandleLargePayload() throws IOException { public void controllerCanGetDeletableStoreTopics() { // The parent controller here is sharing the same kafka as child controllers. String storeName = Utils.getUniqueString("canGetDeletableStoreTopics"); - ControllerClient parentControllerClient = - new ControllerClient(cluster.getClusterName(), parentController.getControllerUrl()); + String clusterName = venice.getClusterNames()[0]; + ControllerClient parentControllerClient = new ControllerClient(clusterName, parentController.getControllerUrl()); try { - Assert - .assertFalse(parentControllerClient.createNewStore(storeName, "test", "\"string\"", "\"string\"").isError()); + assertCommand(parentControllerClient.createNewStore(storeName, "test", "\"string\"", "\"string\"")); String metaSystemStoreName = VeniceSystemStoreType.META_STORE.getSystemStoreName(storeName); // Add some system store and RT topics in the mix to make sure the request can still return the right values. - Assert - .assertFalse(parentControllerClient.emptyPush(metaSystemStoreName, "meta-store-push-1", 1024000L).isError()); - Assert.assertFalse(parentControllerClient.emptyPush(storeName, "push-1", 1024000L).isError()); + assertCommand(parentControllerClient.emptyPush(metaSystemStoreName, "meta-store-push-1", 1024000L)); + + assertCommand(parentControllerClient.emptyPush(storeName, "push-1", 1024000L)); // Store version topic v1 should be truncated after polling for completion by parent controller. TestUtils.waitForNonDeterministicPushCompletion( Version.composeKafkaTopic(storeName, 1), parentControllerClient, 10, TimeUnit.SECONDS); - Assert.assertFalse(parentControllerClient.emptyPush(storeName, "push-2", 1024000L).isError()); + + assertCommand(parentControllerClient.emptyPush(storeName, "push-2", 1024000L)); TestUtils.waitForNonDeterministicPushCompletion( Version.composeKafkaTopic(storeName, 2), - controllerClient, + parentControllerClient, 10, TimeUnit.SECONDS); - Assert.assertFalse(parentControllerClient.deleteOldVersion(storeName, 1).isError()); - MultiStoreTopicsResponse parentMultiStoreTopicResponse = parentControllerClient.getDeletableStoreTopics(); - Assert.assertFalse(parentMultiStoreTopicResponse.isError()); - Assert.assertTrue(parentMultiStoreTopicResponse.getTopics().contains(Version.composeKafkaTopic(storeName, 1))); - Assert.assertFalse(parentMultiStoreTopicResponse.getTopics().contains(Version.composeKafkaTopic(storeName, 2))); - Assert.assertFalse( - parentMultiStoreTopicResponse.getTopics().contains(Version.composeKafkaTopic(metaSystemStoreName, 1))); - Assert.assertFalse( - parentMultiStoreTopicResponse.getTopics().contains(Version.composeRealTimeTopic(metaSystemStoreName))); - // Child fabric should return the same result since they are sharing kafka. Wait for resource of v1 to be cleaned - // up since for child fabric we only consider a topic is deletable if its resource is deleted. + assertCommand(parentControllerClient.deleteOldVersion(storeName, 1)); + // Wait for resource of v1 to be cleaned up since for child fabric we only consider a topic is deletable if its + // resource is deleted. TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, () -> { Assert.assertFalse( - cluster.getLeaderVeniceController() + venice.getChildRegions() + .get(0) + .getLeaderController(clusterName) .getVeniceAdmin() .isResourceStillAlive(Version.composeKafkaTopic(storeName, 1))); }); - MultiStoreTopicsResponse childMultiStoreTopicResponse = controllerClient.getDeletableStoreTopics(); - Assert.assertFalse(childMultiStoreTopicResponse.isError()); + MultiStoreTopicsResponse childMultiStoreTopicResponse = assertCommand(controllerClient.getDeletableStoreTopics()); Assert.assertTrue(childMultiStoreTopicResponse.getTopics().contains(Version.composeKafkaTopic(storeName, 1))); Assert.assertFalse(childMultiStoreTopicResponse.getTopics().contains(Version.composeKafkaTopic(storeName, 2))); + Assert.assertFalse( + childMultiStoreTopicResponse.getTopics().contains(Version.composeKafkaTopic(metaSystemStoreName, 1))); + Assert.assertFalse( + childMultiStoreTopicResponse.getTopics().contains(Version.composeRealTimeTopic(metaSystemStoreName))); } finally { deleteStore(parentControllerClient, storeName); parentControllerClient.close(); @@ -955,35 +1027,46 @@ public void controllerClientReturns404ForNonexistentStoreQuery() { @Test(timeOut = TEST_TIMEOUT) public void testDeleteKafkaTopic() { - String clusterName = cluster.getClusterName(); + String clusterName = venice.getClusterNames()[0]; String storeName = Utils.getUniqueString("controllerClientCanDeleteKafkaTopic"); - VeniceHelixAdmin childControllerAdmin = cluster.getRandomVeniceController().getVeniceHelixAdmin(); - childControllerAdmin.createStore(clusterName, storeName, "test", "\"string\"", "\"string\""); - childControllerAdmin.updateStore( - clusterName, - storeName, - new UpdateStoreQueryParams().setHybridRewindSeconds(1000).setHybridOffsetLagThreshold(1)); - childControllerAdmin.incrementVersionIdempotent(clusterName, storeName, "test", 1, 1); + assertCommand(parentControllerClient.createNewStore(storeName, "owner", "\"string\"", "\"string\"")); + assertCommand( + parentControllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setHybridRewindSeconds(1000).setHybridOffsetLagThreshold(1))); + assertCommand(parentControllerClient.emptyPush(storeName, Utils.getUniqueString(storeName), 1)); String topicToDelete = Version.composeKafkaTopic(storeName, 1); + + VeniceHelixAdmin childControllerAdmin = + venice.getChildRegions().get(0).getLeaderController(clusterName).getVeniceHelixAdmin(); TestUtils.waitForNonDeterministicAssertion(30, TimeUnit.SECONDS, () -> { + LogManager.getLogger(TestAdminSparkServer.class) + .info( + "childControllerAdmin.getTopicManager().listTopics(): {}", + childControllerAdmin.getTopicManager().listTopics()); Assert.assertTrue( childControllerAdmin.getTopicManager() - .containsTopic(cluster.getPubSubTopicRepository().getTopic(topicToDelete))); + .containsTopic( + venice.getChildRegions() + .get(0) + .getClusters() + .get(clusterName) + .getPubSubTopicRepository() + .getTopic(topicToDelete))); Assert.assertFalse(childControllerAdmin.isTopicTruncated(topicToDelete)); }); - controllerClient.deleteKafkaTopic(topicToDelete); + assertCommand(controllerClient.deleteKafkaTopic(topicToDelete)); Assert.assertTrue(childControllerAdmin.isTopicTruncated(topicToDelete)); } @Test(timeOut = TEST_TIMEOUT) public void testCleanupInstanceCustomizedStates() { - String clusterName = cluster.getClusterName(); + String clusterName = venice.getClusterNames()[0]; String storeName = Utils.getUniqueString("cleanupInstanceCustomizedStatesTest"); - VeniceHelixAdmin childControllerAdmin = cluster.getRandomVeniceController().getVeniceHelixAdmin(); + VeniceHelixAdmin childControllerAdmin = venice.getChildRegions().get(0).getRandomController().getVeniceHelixAdmin(); childControllerAdmin.createStore(clusterName, storeName, "test", "\"string\"", "\"string\""); Version version = childControllerAdmin.incrementVersionIdempotent(clusterName, storeName, "test", 1, 1); - MultiStoreTopicsResponse response = controllerClient.cleanupInstanceCustomizedStates(); - Assert.assertFalse(response.isError()); + MultiStoreTopicsResponse response = assertCommand(controllerClient.cleanupInstanceCustomizedStates()); Assert.assertNotNull(response.getTopics()); for (String topic: response.getTopics()) { Assert.assertFalse(topic.endsWith("/" + version.kafkaTopicName())); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java index 0e8dc96bde..0ee4b30892 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java @@ -85,15 +85,12 @@ public void testListStoreWithConfigFilter() { .updateStore(nativeReplicationEnabledStore, new UpdateStoreQueryParams().setNativeReplicationEnabled(true)); Assert.assertFalse(updateStoreResponse.isError()); - // Add a store with incremental push enabled - String incrementalPushEnabledStore = Utils.getUniqueString("incremental-push-store"); - newStoreResponse = controllerClient.createNewStore(incrementalPushEnabledStore, "test", "\"string\"", "\"string\""); + // Add a store with chunking enabled + String chunkingEnabledStore = Utils.getUniqueString("chunking-store"); + newStoreResponse = controllerClient.createNewStore(chunkingEnabledStore, "test", "\"string\"", "\"string\""); Assert.assertFalse(newStoreResponse.isError()); - updateStoreResponse = controllerClient.updateStore( - incrementalPushEnabledStore, - new UpdateStoreQueryParams().setIncrementalPushEnabled(true) - .setHybridOffsetLagThreshold(10L) - .setHybridRewindSeconds(1L)); + updateStoreResponse = + controllerClient.updateStore(chunkingEnabledStore, new UpdateStoreQueryParams().setChunkingEnabled(true)); Assert.assertFalse(updateStoreResponse.isError()); // List stores that have native replication enabled @@ -103,12 +100,11 @@ public void testListStoreWithConfigFilter() { Assert.assertEquals(multiStoreResponse.getStores().length, 1); Assert.assertEquals(multiStoreResponse.getStores()[0], nativeReplicationEnabledStore); - // List stores that have incremental push enabled - multiStoreResponse = - controllerClient.queryStoreList(false, Optional.of("incrementalPushEnabled"), Optional.of("true")); + // List stores that have chunking enabled + multiStoreResponse = controllerClient.queryStoreList(false, Optional.of("chunkingEnabled"), Optional.of("true")); Assert.assertFalse(multiStoreResponse.isError()); Assert.assertEquals(multiStoreResponse.getStores().length, 1); - Assert.assertEquals(multiStoreResponse.getStores()[0], incrementalPushEnabledStore); + Assert.assertEquals(multiStoreResponse.getStores()[0], chunkingEnabledStore); // Add a store with hybrid config enabled and the DataReplicationPolicy is non-aggregate String hybridNonAggregateStore = Utils.getUniqueString("hybrid-non-aggregate"); @@ -140,7 +136,7 @@ public void testListStoreWithConfigFilter() { Assert.assertTrue(hybridStoresSet.contains(hybridAggregateStore)); Assert.assertTrue(hybridStoresSet.contains(hybridNonAggregateStore)); Assert.assertFalse(hybridStoresSet.contains(nativeReplicationEnabledStore)); - Assert.assertTrue(hybridStoresSet.contains(incrementalPushEnabledStore)); + Assert.assertFalse(hybridStoresSet.contains(chunkingEnabledStore)); // List hybrid stores that are on non-aggregate mode multiStoreResponse = @@ -151,7 +147,7 @@ public void testListStoreWithConfigFilter() { Assert.assertFalse(nonAggHybridStoresSet.contains(hybridAggregateStore)); Assert.assertTrue(nonAggHybridStoresSet.contains(hybridNonAggregateStore)); Assert.assertFalse(nonAggHybridStoresSet.contains(nativeReplicationEnabledStore)); - Assert.assertTrue(nonAggHybridStoresSet.contains(incrementalPushEnabledStore)); + Assert.assertFalse(nonAggHybridStoresSet.contains(chunkingEnabledStore)); // List hybrid stores that are on aggregate mode multiStoreResponse = @@ -162,7 +158,7 @@ public void testListStoreWithConfigFilter() { Assert.assertTrue(aggHybridStoresSet.contains(hybridAggregateStore)); Assert.assertFalse(aggHybridStoresSet.contains(hybridNonAggregateStore)); Assert.assertFalse(aggHybridStoresSet.contains(nativeReplicationEnabledStore)); - Assert.assertFalse(aggHybridStoresSet.contains(incrementalPushEnabledStore)); + Assert.assertFalse(aggHybridStoresSet.contains(chunkingEnabledStore)); } @Test(timeOut = TEST_TIMEOUT) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java index 3073d79aaf..c643dab7fb 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java @@ -188,12 +188,10 @@ public void testEnableActiveActiveReplicationForCluster() { String storeName1 = Utils.getUniqueString("test-batch-store"); String storeName2 = Utils.getUniqueString("test-hybrid-agg-store"); String storeName3 = Utils.getUniqueString("test-hybrid-non-agg-store"); - String storeName4 = Utils.getUniqueString("test-incremental-push-store"); try { createAndVerifyStoreInAllRegions(storeName1, parentControllerClient, dcControllerClientList); createAndVerifyStoreInAllRegions(storeName2, parentControllerClient, dcControllerClientList); createAndVerifyStoreInAllRegions(storeName3, parentControllerClient, dcControllerClientList); - createAndVerifyStoreInAllRegions(storeName4, parentControllerClient, dcControllerClientList); assertCommand( parentControllerClient.updateStore( @@ -207,9 +205,6 @@ public void testEnableActiveActiveReplicationForCluster() { storeName3, new UpdateStoreQueryParams().setHybridRewindSeconds(10).setHybridOffsetLagThreshold(2))); - assertCommand( - parentControllerClient.updateStore(storeName4, new UpdateStoreQueryParams().setIncrementalPushEnabled(true))); - // Test batch assertCommand( parentControllerClient.configureActiveActiveReplicationForCluster( @@ -251,16 +246,16 @@ public void testEnableActiveActiveReplicationForCluster() { verifyDCConfigAARepl(dc1Client, storeName3, true, true, false); // Test incremental - assertCommand( - parentControllerClient.configureActiveActiveReplicationForCluster( - true, - VeniceUserStoreType.INCREMENTAL_PUSH.toString(), - Optional.empty())); - verifyDCConfigAARepl(parentControllerClient, storeName4, false, false, true); - verifyDCConfigAARepl(dc0Client, storeName4, false, false, true); - verifyDCConfigAARepl(dc1Client, storeName4, false, false, true); + ControllerResponse response = parentControllerClient.configureActiveActiveReplicationForCluster( + true, + VeniceUserStoreType.INCREMENTAL_PUSH.toString(), + Optional.empty()); + assertTrue(response.isError()); + assertTrue( + response.getError() + .contains("Cannot set cluster-level active-active replication for incremental push stores")); } finally { - deleteStores(storeName1, storeName2, storeName3, storeName4); + deleteStores(storeName1, storeName2, storeName3); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/MetaSystemStoreTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/MetaSystemStoreTest.java index 7c0f5d4194..e361019c56 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/MetaSystemStoreTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/MetaSystemStoreTest.java @@ -2,7 +2,6 @@ import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; -import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS; import static com.linkedin.venice.system.store.MetaStoreWriter.KEY_STRING_CLUSTER_NAME; import static com.linkedin.venice.system.store.MetaStoreWriter.KEY_STRING_SCHEMA_ID; import static com.linkedin.venice.system.store.MetaStoreWriter.KEY_STRING_STORE_NAME; @@ -32,15 +31,13 @@ import com.linkedin.venice.integration.utils.D2TestUtils; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; -import com.linkedin.venice.integration.utils.VeniceControllerCreateOptions; -import com.linkedin.venice.integration.utils.VeniceControllerWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; -import com.linkedin.venice.integration.utils.ZkServerWrapper; +import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.ReadOnlyStore; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.Version; import com.linkedin.venice.meta.VersionStatus; -import com.linkedin.venice.pushmonitor.ExecutionStatus; import com.linkedin.venice.schema.SchemaEntry; import com.linkedin.venice.serialization.avro.AvroProtocolDefinition; import com.linkedin.venice.system.store.MetaStoreDataType; @@ -58,7 +55,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.avro.Schema; @@ -81,43 +77,41 @@ public class MetaSystemStoreTest { + " \"fields\": [\n" + " {\"name\": \"test_field1\", \"type\": \"string\"},\n" + " {\"name\": \"test_field2\", \"type\": \"int\", \"default\": 0}\n" + " ]\n" + "}"; - private VeniceClusterWrapper venice; + private VeniceTwoLayerMultiRegionMultiClusterWrapper venice; + private VeniceClusterWrapper veniceLocalCluster; + private ControllerClient controllerClient; - private VeniceControllerWrapper parentController; - private ZkServerWrapper parentZkServer; + private ControllerClient parentControllerClient; + private String clusterName; @BeforeClass public void setUp() { - Properties testProperties = new Properties(); - testProperties - .put(TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS, Long.toString(TimeUnit.DAYS.toMillis(7))); - venice = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 1000000, false, false); - controllerClient = venice.getControllerClient(); - parentZkServer = ServiceFactory.getZkServer(); - parentController = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder( - venice.getClusterName(), - parentZkServer, - venice.getPubSubBrokerWrapper()) - .childControllers(venice.getVeniceControllers().toArray(new VeniceControllerWrapper[0])) - .extraProperties(testProperties) - .build()); + venice = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .build()); + clusterName = venice.getClusterNames()[0]; + veniceLocalCluster = venice.getChildRegions().get(0).getClusters().get(clusterName); + + controllerClient = new ControllerClient(clusterName, veniceLocalCluster.getAllControllersURLs()); + parentControllerClient = new ControllerClient(clusterName, venice.getControllerConnectString()); } @AfterClass public void cleanUp() { controllerClient.close(); - parentController.close(); venice.close(); - parentZkServer.close(); } @Test(timeOut = 60 * Time.MS_PER_SECOND) public void bootstrapMetaSystemStore() throws ExecutionException, InterruptedException { // Create a new regular store. String regularVeniceStoreName = Utils.getUniqueString("venice_store"); - ControllerClient parentControllerClient = - new ControllerClient(venice.getClusterName(), parentController.getControllerUrl()); NewStoreResponse newStoreResponse = parentControllerClient.createNewStore(regularVeniceStoreName, "test_owner", INT_KEY_SCHEMA, VALUE_SCHEMA_1); assertFalse( @@ -126,20 +120,20 @@ public void bootstrapMetaSystemStore() throws ExecutionException, InterruptedExc + newStoreResponse.getError()); // Do an empty push VersionCreationResponse versionCreationResponse = - controllerClient.emptyPush(regularVeniceStoreName, "test_push_id_1", 100000); + parentControllerClient.emptyPush(regularVeniceStoreName, "test_push_id_1", 100000); assertFalse( versionCreationResponse.isError(), "New version creation should success, but got error: " + versionCreationResponse.getError()); TestUtils.waitForNonDeterministicPushCompletion( versionCreationResponse.getKafkaTopic(), - controllerClient, + parentControllerClient, 10, TimeUnit.SECONDS); String metaSystemStoreName = VeniceSystemStoreType.META_STORE.getSystemStoreName(regularVeniceStoreName); // Check meta system store property Store metaSystemStore = - venice.getLeaderVeniceController().getVeniceAdmin().getStore(venice.getClusterName(), metaSystemStoreName); + veniceLocalCluster.getLeaderVeniceController().getVeniceAdmin().getStore(clusterName, metaSystemStoreName); assertNotNull(metaSystemStore, "Meta System Store shouldn't be null"); long currentLatestVersionPromoteToCurrentTimestampForMetaSystemStore = metaSystemStore.getLatestVersionPromoteToCurrentTimestamp(); @@ -149,7 +143,8 @@ public void bootstrapMetaSystemStore() throws ExecutionException, InterruptedExc + currentLatestVersionPromoteToCurrentTimestampForMetaSystemStore); // Do an empty push against the meta system store - versionCreationResponse = controllerClient.emptyPush(metaSystemStoreName, "test_meta_system_store_push_id", 100000); + versionCreationResponse = + parentControllerClient.emptyPush(metaSystemStoreName, "test_meta_system_store_push_id", 100000); assertFalse( versionCreationResponse.isError(), "New version creation should success, but got error: " + versionCreationResponse.getError()); @@ -160,7 +155,7 @@ public void bootstrapMetaSystemStore() throws ExecutionException, InterruptedExc TimeUnit.SECONDS); // Check meta system stsore property again metaSystemStore = - venice.getLeaderVeniceController().getVeniceAdmin().getStore(venice.getClusterName(), metaSystemStoreName); + veniceLocalCluster.getLeaderVeniceController().getVeniceAdmin().getStore(clusterName, metaSystemStoreName); assertNotNull(metaSystemStore, "Meta System Store shouldn't be null"); assertTrue( metaSystemStore @@ -170,22 +165,23 @@ public void bootstrapMetaSystemStore() throws ExecutionException, InterruptedExc // Query meta system store AvroSpecificStoreClient storeClient = ClientFactory.getAndStartSpecificAvroClient( ClientConfig.defaultSpecificClientConfig(metaSystemStoreName, StoreMetaValue.class) - .setVeniceURL(venice.getRandomRouterURL()) + .setVeniceURL(veniceLocalCluster.getRandomRouterURL()) .setSslFactory(SslUtils.getVeniceLocalSslFactory())); // Query store properties StoreMetaKey storePropertiesKey = MetaStoreDataType.STORE_PROPERTIES.getStoreMetaKey(new HashMap() { { put(KEY_STRING_STORE_NAME, regularVeniceStoreName); - put(KEY_STRING_CLUSTER_NAME, venice.getClusterName()); + put(KEY_STRING_CLUSTER_NAME, clusterName); } }); StoreMetaValue storeProperties = storeClient.get(storePropertiesKey).get(); - assertTrue(storeProperties != null && storeProperties.storeProperties != null); + assertNotNull(storeProperties); + assertNotNull(storeProperties.storeProperties); // Query key schema StoreMetaKey keySchemaKey = MetaStoreDataType.STORE_KEY_SCHEMAS.getStoreMetaKey(new HashMap() { { put(KEY_STRING_STORE_NAME, regularVeniceStoreName); - put(KEY_STRING_CLUSTER_NAME, venice.getClusterName()); + put(KEY_STRING_CLUSTER_NAME, clusterName); } }); StoreMetaValue storeKeySchema = storeClient.get(keySchemaKey).get(); @@ -197,7 +193,7 @@ public void bootstrapMetaSystemStore() throws ExecutionException, InterruptedExc StoreMetaKey valueSchemasKey = MetaStoreDataType.STORE_VALUE_SCHEMAS.getStoreMetaKey(new HashMap() { { put(KEY_STRING_STORE_NAME, regularVeniceStoreName); - put(KEY_STRING_CLUSTER_NAME, venice.getClusterName()); + put(KEY_STRING_CLUSTER_NAME, clusterName); } }); StoreMetaValue storeValueSchemas = storeClient.get(valueSchemasKey).get(); @@ -236,7 +232,7 @@ public void bootstrapMetaSystemStore() throws ExecutionException, InterruptedExc storeDeletionResponse.isError(), "Store deletion should success, but got error: " + storeDeletionResponse.getError()); assertNull( - venice.getVeniceControllers() + veniceLocalCluster.getVeniceControllers() .get(0) .getVeniceAdmin() .getMetaStoreWriter() @@ -261,7 +257,7 @@ public void testThinClientMetaStoreBasedRepository() throws InterruptedException D2Client d2Client = null; NativeMetadataRepository nativeMetadataRepository = null; try { - d2Client = D2TestUtils.getAndStartD2Client(venice.getZk().getAddress()); + d2Client = D2TestUtils.getAndStartD2Client(veniceLocalCluster.getZk().getAddress()); ClientConfig clientConfig = getClientConfig(regularVeniceStoreName, d2Client); // Not providing a CLIENT_META_SYSTEM_STORE_VERSION_MAP, should use the default value of 1 for system store // current version. @@ -300,7 +296,7 @@ public void testThinClientMetaStoreBasedRepositoryWithLargeValueSchemas() throws D2Client d2Client = null; NativeMetadataRepository nativeMetadataRepository = null; try { - d2Client = D2TestUtils.getAndStartD2Client(venice.getZk().getAddress()); + d2Client = D2TestUtils.getAndStartD2Client(veniceLocalCluster.getZk().getAddress()); ClientConfig clientConfig = getClientConfig(regularVeniceStoreName, d2Client); // Not providing a CLIENT_META_SYSTEM_STORE_VERSION_MAP, should use the default value of 1 for system store // current version. @@ -316,9 +312,9 @@ public void testThinClientMetaStoreBasedRepositoryWithLargeValueSchemas() throws Collection metaStoreSchemaEntries = nativeMetadataRepository.getValueSchemas(regularVeniceStoreName); assertEquals( metaStoreSchemaEntries.size(), - venice.getLeaderVeniceController() + veniceLocalCluster.getLeaderVeniceController() .getVeniceAdmin() - .getValueSchemas(venice.getClusterName(), regularVeniceStoreName) + .getValueSchemas(clusterName, regularVeniceStoreName) .size(), "Number of value schemas should be the same between meta system store and controller"); for (int i = 2; i < numberOfLargeSchemaVersions; i++) { @@ -335,9 +331,9 @@ public void testThinClientMetaStoreBasedRepositoryWithLargeValueSchemas() throws SchemaEntry latestValueSchema = nativeMetadataRepository.getSupersetOrLatestValueSchema(regularVeniceStoreName); assertEquals( latestValueSchema, - venice.getLeaderVeniceController() + veniceLocalCluster.getLeaderVeniceController() .getVeniceAdmin() - .getValueSchema(venice.getClusterName(), regularVeniceStoreName, latestValueSchema.getId()), + .getValueSchema(clusterName, regularVeniceStoreName, latestValueSchema.getId()), "NativeMetadataRepository is not returning the right schema id and/or schema pair"); } finally { if (d2Client != null) { @@ -356,40 +352,37 @@ public void testThinClientMetaStoreBasedRepositoryWithLargeValueSchemas() throws @Test(timeOut = 60 * Time.MS_PER_SECOND) public void testParentControllerAutoMaterializeMetaSystemStore() { - try (ControllerClient parentControllerClient = - new ControllerClient(venice.getClusterName(), parentController.getControllerUrl())) { - String zkSharedMetaSystemSchemaStoreName = - AvroProtocolDefinition.METADATA_SYSTEM_SCHEMA_STORE.getSystemStoreName(); - TestUtils.waitForNonDeterministicAssertion(30, TimeUnit.SECONDS, true, () -> { - Store readOnlyStore = parentController.getVeniceAdmin() - .getReadOnlyZKSharedSystemStoreRepository() - .getStore(zkSharedMetaSystemSchemaStoreName); - Assert.assertNotNull( - readOnlyStore, - "Store: " + zkSharedMetaSystemSchemaStoreName + " should be initialized by " - + ClusterLeaderInitializationRoutine.class.getSimpleName()); - Assert.assertTrue( - readOnlyStore.isHybrid(), - "Store: " + zkSharedMetaSystemSchemaStoreName + " should be configured to hybrid"); - }); - String storeName = Utils.getUniqueString("new-user-store"); - assertFalse( - parentControllerClient.createNewStore(storeName, "venice-test", INT_KEY_SCHEMA, VALUE_SCHEMA_1).isError(), - "Unexpected new store creation failure"); - String metaSystemStoreName = VeniceSystemStoreType.META_STORE.getSystemStoreName(storeName); - TestUtils.waitForNonDeterministicPushCompletion( - Version.composeKafkaTopic(metaSystemStoreName, 1), - parentControllerClient, - 30, - TimeUnit.SECONDS); - } + String zkSharedMetaSystemSchemaStoreName = AvroProtocolDefinition.METADATA_SYSTEM_SCHEMA_STORE.getSystemStoreName(); + TestUtils.waitForNonDeterministicAssertion(30, TimeUnit.SECONDS, true, () -> { + Store readOnlyStore = venice.getLeaderParentControllerWithRetries(clusterName) + .getVeniceAdmin() + .getReadOnlyZKSharedSystemStoreRepository() + .getStore(zkSharedMetaSystemSchemaStoreName); + Assert.assertNotNull( + readOnlyStore, + "Store: " + zkSharedMetaSystemSchemaStoreName + " should be initialized by " + + ClusterLeaderInitializationRoutine.class.getSimpleName()); + Assert.assertTrue( + readOnlyStore.isHybrid(), + "Store: " + zkSharedMetaSystemSchemaStoreName + " should be configured to hybrid"); + }); + String storeName = Utils.getUniqueString("new-user-store"); + assertFalse( + parentControllerClient.createNewStore(storeName, "venice-test", INT_KEY_SCHEMA, VALUE_SCHEMA_1).isError(), + "Unexpected new store creation failure"); + String metaSystemStoreName = VeniceSystemStoreType.META_STORE.getSystemStoreName(storeName); + TestUtils.waitForNonDeterministicPushCompletion( + Version.composeKafkaTopic(metaSystemStoreName, 1), + parentControllerClient, + 30, + TimeUnit.SECONDS); } private ClientConfig getClientConfig(String storeName, D2Client d2Client) { return ClientConfig.defaultSpecificClientConfig(storeName, StoreMetaValue.class) .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME) .setD2Client(d2Client) - .setVeniceURL(venice.getZk().getAddress()); + .setVeniceURL(veniceLocalCluster.getZk().getAddress()); } private void verifyRepository(NativeMetadataRepository nativeMetadataRepository, String regularVeniceStoreName) @@ -401,17 +394,17 @@ private void verifyRepository(NativeMetadataRepository nativeMetadataRepository, nativeMetadataRepository.subscribe(regularVeniceStoreName); Store store = nativeMetadataRepository.getStore(regularVeniceStoreName); Store controllerStore = new ReadOnlyStore( - venice.getLeaderVeniceController().getVeniceAdmin().getStore(venice.getClusterName(), regularVeniceStoreName)); + veniceLocalCluster.getLeaderVeniceController().getVeniceAdmin().getStore(clusterName, regularVeniceStoreName)); assertEquals(store, controllerStore); SchemaEntry keySchema = nativeMetadataRepository.getKeySchema(regularVeniceStoreName); - SchemaEntry controllerKeySchema = venice.getLeaderVeniceController() + SchemaEntry controllerKeySchema = veniceLocalCluster.getLeaderVeniceController() .getVeniceAdmin() - .getKeySchema(venice.getClusterName(), regularVeniceStoreName); + .getKeySchema(clusterName, regularVeniceStoreName); assertEquals(keySchema, controllerKeySchema); Collection valueSchemas = nativeMetadataRepository.getValueSchemas(regularVeniceStoreName); - Collection controllerValueSchemas = venice.getLeaderVeniceController() + Collection controllerValueSchemas = veniceLocalCluster.getLeaderVeniceController() .getVeniceAdmin() - .getValueSchemas(venice.getClusterName(), regularVeniceStoreName); + .getValueSchemas(clusterName, regularVeniceStoreName); assertEquals(valueSchemas, controllerValueSchemas); long storageQuota = 123456789; int partitionCount = 3; @@ -430,12 +423,12 @@ private void verifyRepository(NativeMetadataRepository nativeMetadataRepository, TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, () -> { assertEquals( nativeMetadataRepository.getValueSchemas(regularVeniceStoreName), - venice.getLeaderVeniceController() + veniceLocalCluster.getLeaderVeniceController() .getVeniceAdmin() - .getValueSchemas(venice.getClusterName(), regularVeniceStoreName)); + .getValueSchemas(clusterName, regularVeniceStoreName)); }); VersionCreationResponse versionCreationResponse = - controllerClient.emptyPush(regularVeniceStoreName, "new_push", 10000); + parentControllerClient.emptyPush(regularVeniceStoreName, "new_push", 10000); assertFalse(versionCreationResponse.isError()); TestUtils.waitForNonDeterministicPushCompletion( versionCreationResponse.getKafkaTopic(), @@ -458,28 +451,16 @@ private void createStoreAndMaterializeMetaSystemStore(String storeName) { private void createStoreAndMaterializeMetaSystemStore(String storeName, String valueSchema) { // Verify and create Venice regular store if it doesn't exist. - if (controllerClient.getStore(storeName).getStore() == null) { - assertFalse(controllerClient.createNewStore(storeName, "test_owner", INT_KEY_SCHEMA, valueSchema).isError()); + if (parentControllerClient.getStore(storeName).getStore() == null) { + assertFalse( + parentControllerClient.createNewStore(storeName, "test_owner", INT_KEY_SCHEMA, valueSchema).isError()); } String metaSystemStoreName = VeniceSystemStoreType.META_STORE.getSystemStoreName(storeName); - // Ignore transient failures on job status when the cluster is still starting. - TestUtils.waitForNonDeterministicAssertion( - 10, - TimeUnit.SECONDS, - () -> assertNotNull( - controllerClient.queryJobStatus(Version.composeKafkaTopic(metaSystemStoreName, 1)).getStatus())); - String metaSystemStoreStatus = - controllerClient.queryJobStatus(Version.composeKafkaTopic(metaSystemStoreName, 1)).getStatus(); - if (ExecutionStatus.NOT_CREATED.toString().equals(metaSystemStoreStatus)) { - assertFalse(controllerClient.emptyPush(metaSystemStoreName, "test_meta_system_store_push", 10000).isError()); - TestUtils.waitForNonDeterministicPushCompletion( - Version.composeKafkaTopic(metaSystemStoreName, 1), - controllerClient, - 30, - TimeUnit.SECONDS); - } else if (!ExecutionStatus.COMPLETED.toString().equals(metaSystemStoreStatus)) { - fail("Unexpected meta system store status: " + metaSystemStoreStatus); - } + TestUtils.waitForNonDeterministicPushCompletion( + Version.composeKafkaTopic(metaSystemStoreName, 1), + controllerClient, + 30, + TimeUnit.SECONDS); } private List generateLargeValueSchemas(int baseNumberOfFields, int numberOfVersions) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java index 90f3c6f74c..7f3edb616e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java @@ -69,6 +69,7 @@ import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; +import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.ReadOnlySchemaRepository; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.Version; @@ -309,7 +310,7 @@ public void testIncrementalPushPartialUpdateClassicFormat() throws IOException { .setCompressionStrategy(CompressionStrategy.NO_OP) .setWriteComputationEnabled(true) .setChunkingEnabled(true) - .setIncrementalPushEnabled(true) + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) .setHybridRewindSeconds(10L) .setHybridOffsetLagThreshold(2L); ControllerResponse updateStoreResponse = @@ -376,7 +377,7 @@ public void testIncrementalPushPartialUpdateNewFormat(boolean useSparkCompute) t .setCompressionStrategy(CompressionStrategy.NO_OP) .setWriteComputationEnabled(true) .setChunkingEnabled(true) - .setIncrementalPushEnabled(true) + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) .setHybridRewindSeconds(10L) .setHybridOffsetLagThreshold(2L); ControllerResponse updateStoreResponse = diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ParticipantStoreTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ParticipantStoreTest.java index 6f24f27c77..1690c4dddf 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ParticipantStoreTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ParticipantStoreTest.java @@ -1,9 +1,5 @@ package com.linkedin.venice.endToEnd; -import static com.linkedin.venice.ConfigKeys.ADMIN_HELIX_MESSAGING_CHANNEL_ENABLED; -import static com.linkedin.venice.ConfigKeys.PARTICIPANT_MESSAGE_CONSUMPTION_DELAY_MS; -import static com.linkedin.venice.ConfigKeys.PARTICIPANT_MESSAGE_STORE_ENABLED; -import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -11,8 +7,6 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.linkedin.d2.balancer.D2Client; -import com.linkedin.venice.D2.D2ClientUtils; import com.linkedin.venice.client.store.AvroSpecificStoreClient; import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; @@ -21,14 +15,12 @@ import com.linkedin.venice.controllerapi.ControllerResponse; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; -import com.linkedin.venice.integration.utils.D2TestUtils; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; -import com.linkedin.venice.integration.utils.VeniceControllerCreateOptions; -import com.linkedin.venice.integration.utils.VeniceControllerWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; -import com.linkedin.venice.integration.utils.ZkServerWrapper; +import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.StoreDataChangedListener; import com.linkedin.venice.meta.StoreInfo; @@ -44,7 +36,6 @@ import io.tehuti.Metric; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.IOUtils; @@ -58,48 +49,33 @@ public class ParticipantStoreTest { private static final Logger LOGGER = LogManager.getLogger(ParticipantStoreTest.class); - private VeniceClusterWrapper veniceClusterWrapper; - private VeniceControllerWrapper parentController; - private ZkServerWrapper parentZk; + private VeniceTwoLayerMultiRegionMultiClusterWrapper venice; + private VeniceClusterWrapper veniceLocalCluster; + private VeniceServerWrapper veniceServerWrapper; + private ControllerClient controllerClient; private ControllerClient parentControllerClient; private String participantMessageStoreName; - private VeniceServerWrapper veniceServerWrapper; - private D2Client d2Client; + private String clusterName; @BeforeClass public void setUp() { - Properties controllerConfig = new Properties(); - Properties serverFeatureProperties = new Properties(); - Properties serverProperties = new Properties(); - controllerConfig.setProperty(PARTICIPANT_MESSAGE_STORE_ENABLED, "true"); - controllerConfig.setProperty(ADMIN_HELIX_MESSAGING_CHANNEL_ENABLED, "false"); - // Disable topic cleanup since parent and child are sharing the same kafka cluster. - controllerConfig - .setProperty(TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS, String.valueOf(Long.MAX_VALUE)); - veniceClusterWrapper = ServiceFactory.getVeniceCluster(1, 0, 1, 1, 100000, false, false, controllerConfig); - d2Client = D2TestUtils.getAndStartD2Client(veniceClusterWrapper.getZk().getAddress()); - serverFeatureProperties.put( - VeniceServerWrapper.CLIENT_CONFIG_FOR_CONSUMER, - ClientConfig.defaultGenericClientConfig("") - .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME) - .setD2Client(d2Client)); - serverProperties.setProperty(PARTICIPANT_MESSAGE_CONSUMPTION_DELAY_MS, Long.toString(100)); - veniceServerWrapper = veniceClusterWrapper.addVeniceServer(serverFeatureProperties, serverProperties); - parentZk = ServiceFactory.getZkServer(); - parentController = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder( - veniceClusterWrapper.getClusterName(), - parentZk, - veniceClusterWrapper.getPubSubBrokerWrapper()) - .childControllers(veniceClusterWrapper.getVeniceControllers().toArray(new VeniceControllerWrapper[0])) - .extraProperties(controllerConfig) - .build()); - participantMessageStoreName = - VeniceSystemStoreUtils.getParticipantStoreNameForCluster(veniceClusterWrapper.getClusterName()); - controllerClient = veniceClusterWrapper.getControllerClient(); - parentControllerClient = - new ControllerClient(veniceClusterWrapper.getClusterName(), parentController.getControllerUrl()); + venice = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .build()); + clusterName = venice.getClusterNames()[0]; + participantMessageStoreName = VeniceSystemStoreUtils.getParticipantStoreNameForCluster(clusterName); + veniceLocalCluster = venice.getChildRegions().get(0).getClusters().get(clusterName); + veniceServerWrapper = veniceLocalCluster.getVeniceServers().get(0); + + controllerClient = new ControllerClient(clusterName, veniceLocalCluster.getAllControllersURLs()); + parentControllerClient = new ControllerClient(clusterName, venice.getControllerConnectString()); TestUtils.waitForNonDeterministicPushCompletion( Version.composeKafkaTopic(participantMessageStoreName, 1), controllerClient, @@ -111,12 +87,7 @@ public void setUp() { public void cleanUp() { Utils.closeQuietlyWithErrorLogged(controllerClient); Utils.closeQuietlyWithErrorLogged(parentControllerClient); - Utils.closeQuietlyWithErrorLogged(parentController); - if (d2Client != null) { - D2ClientUtils.shutdownClient(d2Client); - } - IOUtils.closeQuietly(veniceClusterWrapper); - IOUtils.closeQuietly(parentZk); + IOUtils.closeQuietly(venice); } // @Test(timeOut = 60 * Time.MS_PER_SECOND) @@ -129,8 +100,8 @@ public void testParticipantStoreKill() { // Verify the push job is STARTED. assertEquals(controllerClient.queryJobStatus(topicName).getStatus(), ExecutionStatus.STARTED.toString()); }); - String metricPrefix = "." + veniceClusterWrapper.getClusterName() + "-participant_store_consumption_task"; - double killedPushJobCount = veniceClusterWrapper.getVeniceServers() + String metricPrefix = "." + clusterName + "-participant_store_consumption_task"; + double killedPushJobCount = veniceLocalCluster.getVeniceServers() .iterator() .next() .getMetricsRepository() @@ -147,10 +118,9 @@ public void testParticipantStoreKill() { }); // Verify participant store consumption stats String requestMetricExample = - VeniceSystemStoreUtils.getParticipantStoreNameForCluster(veniceClusterWrapper.getClusterName()) - + "--success_request_key_count.Avg"; + VeniceSystemStoreUtils.getParticipantStoreNameForCluster(clusterName) + "--success_request_key_count.Avg"; Map metrics = - veniceClusterWrapper.getVeniceServers().iterator().next().getMetricsRepository().metrics(); + veniceLocalCluster.getVeniceServers().iterator().next().getMetricsRepository().metrics(); assertEquals(metrics.get(metricPrefix + "--killed_push_jobs.Count").value(), 1.0); assertTrue(metrics.get(metricPrefix + "--kill_push_job_latency.Avg").value() > 0); // One from the server stats and the other from the client stats. @@ -178,9 +148,9 @@ public void testParticipantStoreThrottlerRestartRouter() { }); // restart routers to discard in-memory throttler info - for (VeniceRouterWrapper router: veniceClusterWrapper.getVeniceRouters()) { - veniceClusterWrapper.stopVeniceRouter(router.getPort()); - veniceClusterWrapper.restartVeniceRouter(router.getPort()); + for (VeniceRouterWrapper router: veniceLocalCluster.getVeniceRouters()) { + veniceLocalCluster.stopVeniceRouter(router.getPort()); + veniceLocalCluster.restartVeniceRouter(router.getPort()); } // Verify still can read from participant stores. ParticipantMessageKey key = new ParticipantMessageKey(); @@ -189,7 +159,7 @@ public void testParticipantStoreThrottlerRestartRouter() { try (AvroSpecificStoreClient client = ClientFactory.getAndStartSpecificAvroClient( ClientConfig.defaultSpecificClientConfig(participantMessageStoreName, ParticipantMessageValue.class) - .setVeniceURL(veniceClusterWrapper.getRandomRouterURL()))) { + .setVeniceURL(veniceLocalCluster.getRandomRouterURL()))) { try { client.get(key).get(); } catch (Exception e) { @@ -241,21 +211,21 @@ public void testKillWhenVersionIsOnline() { // Then we could verify whether the previous version receives a kill-job or not. verifyKillMessageInParticipantStore(topicNameForOnlineVersion, false); - veniceClusterWrapper.stopVeniceServer(veniceServerWrapper.getPort()); + veniceLocalCluster.stopVeniceServer(veniceServerWrapper.getPort()); // Ensure the partition assignment is 0 before restarting the server TestUtils.waitForNonDeterministicAssertion(30, TimeUnit.SECONDS, true, true, () -> { - VeniceRouterWrapper routerWrapper = veniceClusterWrapper.getRandomVeniceRouter(); + VeniceRouterWrapper routerWrapper = veniceLocalCluster.getRandomVeniceRouter(); assertFalse(routerWrapper.getRoutingDataRepository().containsKafkaTopic(topicNameForOnlineVersion)); }); - veniceClusterWrapper.restartVeniceServer(veniceServerWrapper.getPort()); + veniceLocalCluster.restartVeniceServer(veniceServerWrapper.getPort()); int expectedOnlineReplicaCount = versionCreationResponseForOnlineVersion.getReplicas(); TestUtils.waitForNonDeterministicAssertion(30, TimeUnit.SECONDS, () -> { for (int p = 0; p < versionCreationResponseForOnlineVersion.getPartitions(); p++) { try { assertEquals( - veniceClusterWrapper.getRandomVeniceRouter() + veniceLocalCluster.getRandomVeniceRouter() .getRoutingDataRepository() .getReadyToServeInstances(topicNameForOnlineVersion, p) .size(), @@ -326,7 +296,7 @@ private void verifyKillMessageInParticipantStore(String topic, boolean shouldPre try (AvroSpecificStoreClient client = ClientFactory.getAndStartSpecificAvroClient( ClientConfig.defaultSpecificClientConfig(participantMessageStoreName, ParticipantMessageValue.class) - .setVeniceURL(veniceClusterWrapper.getRandomRouterURL()))) { + .setVeniceURL(veniceLocalCluster.getRandomRouterURL()))) { TestUtils.waitForNonDeterministicAssertion(30, TimeUnit.SECONDS, true, () -> { try { if (shouldPresent) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java index ce291319a2..8402874159 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java @@ -220,7 +220,7 @@ public void testPushJobDetailsFailureTags() throws ExecutionException, Interrupt recordSchema.getField(DEFAULT_KEY_FIELD_PROP).schema().toString(), recordSchema.getField(DEFAULT_VALUE_FIELD_PROP).schema().toString()); // hadoop job client cannot fetch counters properly and should fail the job - parentControllerClient.updateStore(testStoreName, new UpdateStoreQueryParams().setStorageQuotaInByte(0)); + parentControllerClient.updateStore(testStoreName, new UpdateStoreQueryParams().setStorageQuotaInByte(1)); Properties pushJobProps = defaultVPJProps(multiRegionMultiClusterWrapper, inputDirPath, testStoreName); pushJobProps.setProperty(PUSH_JOB_STATUS_UPLOAD_ENABLE, String.valueOf(true)); try (VenicePushJob testPushJob = new VenicePushJob("test-push-job-details-job", pushJobProps)) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java index 22b2d801b8..d7cacdda1e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java @@ -27,11 +27,13 @@ import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; +import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pushstatushelper.PushStatusStoreReader; import com.linkedin.venice.serialization.avro.AvroProtocolDefinition; import com.linkedin.venice.utils.TestUtils; +import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; import java.util.List; import java.util.Optional; @@ -121,7 +123,9 @@ public void setUpStore() { storeName, new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) .setPartitionCount(PARTITION_COUNT) - .setIncrementalPushEnabled(true))); + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) + .setHybridRewindSeconds(Time.SECONDS_PER_DAY) + .setHybridOffsetLagThreshold(1000))); String daVinciPushStatusSystemStoreName = VeniceSystemStoreType.DAVINCI_PUSH_STATUS_STORE.getSystemStoreName(storeName); VersionCreationResponse versionCreationResponseForDaVinciPushStatusSystemStore = parentControllerClient diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java index 760e8f2119..1512fbfa7d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreTest.java @@ -39,6 +39,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; +import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubTopicRepository; @@ -53,6 +54,7 @@ import com.linkedin.venice.utils.DataProviderUtils; import com.linkedin.venice.utils.PropertyBuilder; import com.linkedin.venice.utils.TestUtils; +import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import java.io.File; @@ -128,7 +130,9 @@ private String createStoreAndSystemStores() { storeName, new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) .setPartitionCount(PARTITION_COUNT) - .setIncrementalPushEnabled(true))); + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) + .setHybridRewindSeconds(Time.SECONDS_PER_DAY) + .setHybridOffsetLagThreshold(1000))); return storeName; } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoragePersonaTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoragePersonaTest.java index 172ee9bf4e..6825a3c35f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoragePersonaTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoragePersonaTest.java @@ -15,10 +15,8 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; -import com.linkedin.venice.integration.utils.VeniceControllerCreateOptions; -import com.linkedin.venice.integration.utils.VeniceControllerWrapper; -import com.linkedin.venice.integration.utils.ZkServerWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.persona.StoragePersona; import com.linkedin.venice.utils.TestUtils; @@ -26,7 +24,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Properties; import java.util.concurrent.TimeUnit; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -36,8 +33,6 @@ public class StoragePersonaTest { private VeniceClusterWrapper venice; - private ZkServerWrapper parentZk; - private VeniceControllerWrapper parentController; private ControllerClient controllerClient; /** @@ -47,21 +42,21 @@ public class StoragePersonaTest { */ @BeforeClass(alwaysRun = true) public void setUp() { - Properties extraProperties = new Properties(); - venice = ServiceFactory.getVeniceCluster(1, 1, 1, 2, 1000000, false, false, extraProperties); - parentZk = ServiceFactory.getZkServer(); - parentController = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder(venice.getClusterName(), parentZk, venice.getPubSubBrokerWrapper()) - .childControllers(new VeniceControllerWrapper[] { venice.getLeaderVeniceController() }) + venice = ServiceFactory.getVeniceCluster( + new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) .build()); - controllerClient = new ControllerClient(venice.getClusterName(), parentController.getControllerUrl()); + controllerClient = new ControllerClient(venice.getClusterName(), venice.getAllControllersURLs()); } @AfterClass(alwaysRun = true) public void cleanUp() { Utils.closeQuietlyWithErrorLogged(controllerClient); - Utils.closeQuietlyWithErrorLogged(parentController); - Utils.closeQuietlyWithErrorLogged(parentZk); Utils.closeQuietlyWithErrorLogged(venice); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java index 75309baedb..0029a2a35e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java @@ -3,7 +3,6 @@ import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; import static com.linkedin.venice.ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE; import static com.linkedin.venice.ConfigKeys.ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE; -import static com.linkedin.venice.ConfigKeys.ENABLE_INCREMENTAL_PUSH_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES; import static com.linkedin.venice.ConfigKeys.SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED; import static com.linkedin.venice.ConfigKeys.SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE; import static com.linkedin.venice.ConfigKeys.SERVER_DEDICATED_CONSUMER_POOL_FOR_AA_WC_LEADER_ENABLED; @@ -87,7 +86,6 @@ public void setUp() { Properties controllerProps = new Properties(); controllerProps.put(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, "true"); controllerProps.put(ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE, true); - controllerProps.put(ENABLE_INCREMENTAL_PUSH_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES, true); multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( NUMBER_OF_CHILD_DATACENTERS, @@ -207,9 +205,12 @@ public void testAAReplicationForIncrementalPushToRT(boolean overrideDataReplicat dc1ControllerClient, dc2ControllerClient); + // Incremental push is not allowed for NON_AGGREGATE DataReplicationPolicy + boolean incrementalPushAllowed = !overrideDataReplicationPolicy; + verifyHybridAndIncPushConfig( storeName, - true, + incrementalPushAllowed, true, parentControllerClient, dc0ControllerClient, @@ -221,10 +222,18 @@ public void testAAReplicationForIncrementalPushToRT(boolean overrideDataReplicat job.run(); Assert.assertEquals(job.getKafkaUrl(), childDatacenters.get(2).getKafkaBrokerWrapper().getAddress()); } + // Run inc push with source fabric preference taking effect. try (VenicePushJob job = new VenicePushJob("Test push job incremental with NR + A/A from dc-2", propsInc1)) { job.run(); + if (!incrementalPushAllowed) { + Assert.fail("Incremental push should throw an exception for NON_AGGREGATE data replication policy"); + } Assert.assertEquals(job.getKafkaUrl(), childDatacenters.get(2).getKafkaBrokerWrapper().getAddress()); + } catch (Exception e) { + if (incrementalPushAllowed) { + throw e; + } } // Verify @@ -235,14 +244,27 @@ public void testAAReplicationForIncrementalPushToRT(boolean overrideDataReplicat childDataCenter.getRandomController().getVeniceAdmin().getStore(clusterName, storeName).getVersion(1); Assert.assertNotNull(version, "Version 1 is not present for DC: " + dcNames[i]); } - NativeReplicationTestUtils.verifyIncrementalPushData(childDatacenters, clusterName, storeName, 150, 2); + + if (incrementalPushAllowed) { + NativeReplicationTestUtils.verifyIncrementalPushData(childDatacenters, clusterName, storeName, 150, 2); + } // Run another inc push with a different source fabric preference taking effect. try (VenicePushJob job = new VenicePushJob("Test push job incremental with NR + A/A from dc-1", propsInc2)) { job.run(); + if (!incrementalPushAllowed) { + Assert.fail("Incremental push should throw an exception for NON_AGGREGATE data replication policy"); + } Assert.assertEquals(job.getKafkaUrl(), childDatacenters.get(1).getKafkaBrokerWrapper().getAddress()); + } catch (Exception e) { + if (incrementalPushAllowed) { + throw e; + } + } + + if (incrementalPushAllowed) { + NativeReplicationTestUtils.verifyIncrementalPushData(childDatacenters, clusterName, storeName, 200, 3); } - NativeReplicationTestUtils.verifyIncrementalPushData(childDatacenters, clusterName, storeName, 200, 3); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java index 8601691a59..af8a6a79e1 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatch.java @@ -63,6 +63,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.BackupStrategy; +import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.Version; import com.linkedin.venice.read.RequestType; import com.linkedin.venice.stats.AbstractVeniceStats; @@ -111,8 +112,6 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -//TODO: write a VPJWrapper that can handle the whole flow - @Test(singleThreaded = true) public abstract class TestBatch { @@ -452,7 +451,9 @@ public void testIncrementalPush() throws Exception { Assert.assertEquals(avroClient.get(Integer.toString(i)).get().toString(), "test_name_" + i); } }, - new UpdateStoreQueryParams().setIncrementalPushEnabled(true)); + new UpdateStoreQueryParams().setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) + .setHybridRewindSeconds(Time.SECONDS_PER_DAY) + .setHybridOffsetLagThreshold(1000)); testBatchStore( inputDir -> new KeyAndValueSchemas(writeSimpleAvroFileWithStringToStringSchema2(inputDir)), @@ -468,7 +469,7 @@ public void testIncrementalPush() throws Exception { } }, storeName, - new UpdateStoreQueryParams().setIncrementalPushEnabled(true)); + null); testBatchStore( inputDir -> new KeyAndValueSchemas(writeSimpleAvroFileWithStringToStringSchema(inputDir)), @@ -480,7 +481,7 @@ public void testIncrementalPush() throws Exception { } }, storeName, - new UpdateStoreQueryParams().setIncrementalPushEnabled(true)); + null); } @Test(timeOut = TEST_TIMEOUT, dataProvider = "Two-True-and-False", dataProviderClass = DataProviderUtils.class) @@ -496,7 +497,7 @@ public void testIncrementalPushWithCompression( }, getSimpleFileWithUserSchemaValidatorForZstd(), new UpdateStoreQueryParams().setCompressionStrategy(CompressionStrategy.ZSTD_WITH_DICT) - .setIncrementalPushEnabled(true) + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) .setHybridOffsetLagThreshold(10) .setHybridRewindSeconds(0)); @@ -522,59 +523,6 @@ public void testIncrementalPushWithCompression( null); } - @Test(timeOut = TEST_TIMEOUT) - public void testIncrementalPushWritesToRealTimeTopicWithPolicy() throws Exception { - double randomNumber = Math.random(); - String classAndFunctionName = getClass().getSimpleName() + ".testIncrementalPushWritesToRealTimeTopicWithPolicy()"; - String uniqueTestId = "attempt [" + randomNumber + "] of " + classAndFunctionName; - LOGGER.info("Start of {}", uniqueTestId); - try { - String storeName = testBatchStore( - inputDir -> new KeyAndValueSchemas(writeSimpleAvroFileWithStringToStringSchema(inputDir)), - properties -> {}, - (avroClient, vsonClient, metricsRepository) -> { - for (int i = 1; i <= 100; i++) { - Assert.assertEquals(avroClient.get(Integer.toString(i)).get().toString(), "test_name_" + i); - } - }, - new UpdateStoreQueryParams().setIncrementalPushEnabled(true) - .setChunkingEnabled(true) - .setHybridOffsetLagThreshold(10) - .setHybridRewindSeconds(0)); - - testBatchStore( - inputDir -> new KeyAndValueSchemas(writeSimpleAvroFileWithStringToStringSchema2(inputDir)), - properties -> properties.setProperty(INCREMENTAL_PUSH, "true"), - (avroClient, vsonClient, metricsRepository) -> { - for (int i = 51; i <= 150; i++) { - Assert.assertEquals(avroClient.get(Integer.toString(i)).get().toString(), "test_name_" + (i * 2)); - } - }, - storeName, - null); - - testBatchStore( - inputDir -> new KeyAndValueSchemas(writeSimpleAvroFileWithStringToStringSchema(inputDir)), - properties -> {}, - (avroClient, vsonClient, metricsRepository) -> { - TestUtils.waitForNonDeterministicAssertion(30, TimeUnit.SECONDS, true, () -> { - for (int i = 1; i <= 100; i++) { - Assert.assertEquals(avroClient.get(Integer.toString(i)).get().toString(), "test_name_" + i); - } - for (int i = 101; i <= 150; i++) { - Assert.assertNull(avroClient.get(Integer.toString(i)).get()); - } - }); - }, - storeName, - null); - LOGGER.info("Successful end of {}", uniqueTestId); - } catch (Throwable e) { - LOGGER.error("Caught throwable in {}", uniqueTestId, e); - throw e; - } - } - @Test(timeOut = TEST_TIMEOUT) public void testMetaStoreSchemaValidation() throws Exception { String storeName = testBatchStore( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java index ac4ef2f71f..0cae1e1e68 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java @@ -2,7 +2,6 @@ import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; import static com.linkedin.venice.ConfigKeys.DEFAULT_MAX_NUMBER_OF_PARTITIONS; -import static com.linkedin.venice.ConfigKeys.INSTANCE_ID; import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; import static com.linkedin.venice.ConfigKeys.PERSISTENCE_TYPE; import static com.linkedin.venice.ConfigKeys.SERVER_CONSUMER_POOL_SIZE_PER_KAFKA_CLUSTER; @@ -30,7 +29,6 @@ import static com.linkedin.venice.utils.TestWriteUtils.getTempDataDirectory; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @@ -47,22 +45,16 @@ import com.linkedin.venice.controller.Admin; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.ControllerResponse; -import com.linkedin.venice.controllerapi.JobStatusQueryResponse; -import com.linkedin.venice.controllerapi.MultiStoreStatusResponse; import com.linkedin.venice.controllerapi.StoreResponse; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; -import com.linkedin.venice.controllerapi.VersionResponse; import com.linkedin.venice.exceptions.RecordTooLargeException; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.helix.HelixBaseRoutingRepository; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; -import com.linkedin.venice.integration.utils.VeniceControllerCreateOptions; -import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; -import com.linkedin.venice.integration.utils.ZkServerWrapper; import com.linkedin.venice.kafka.protocol.GUID; import com.linkedin.venice.kafka.protocol.KafkaMessageEnvelope; import com.linkedin.venice.kafka.protocol.LeaderMetadata; @@ -71,9 +63,6 @@ import com.linkedin.venice.kafka.protocol.enums.MessageType; import com.linkedin.venice.message.KafkaKey; import com.linkedin.venice.meta.BufferReplayPolicy; -import com.linkedin.venice.meta.DataReplicationPolicy; -import com.linkedin.venice.meta.HybridStoreConfig; -import com.linkedin.venice.meta.HybridStoreConfigImpl; import com.linkedin.venice.meta.Instance; import com.linkedin.venice.meta.InstanceStatus; import com.linkedin.venice.meta.PersistenceType; @@ -81,7 +70,6 @@ import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.StoreStatus; import com.linkedin.venice.meta.Version; -import com.linkedin.venice.meta.ZKStore; import com.linkedin.venice.producer.VeniceProducer; import com.linkedin.venice.producer.online.OnlineProducerFactory; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -92,13 +80,10 @@ import com.linkedin.venice.samza.VeniceSystemProducer; import com.linkedin.venice.serializer.AvroGenericDeserializer; import com.linkedin.venice.serializer.AvroSerializer; -import com.linkedin.venice.systemstore.schemas.StoreProperties; -import com.linkedin.venice.utils.AvroRecordUtils; import com.linkedin.venice.utils.ByteUtils; import com.linkedin.venice.utils.DataProviderUtils; import com.linkedin.venice.utils.IntegrationTestPushUtils; import com.linkedin.venice.utils.Pair; -import com.linkedin.venice.utils.StoreUtils; import com.linkedin.venice.utils.TestMockTime; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.TestWriteUtils; @@ -106,7 +91,6 @@ import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import com.linkedin.venice.writer.CompletableFutureCallback; -import com.linkedin.venice.writer.LeaderMetadataWrapper; import com.linkedin.venice.writer.VeniceWriter; import com.linkedin.venice.writer.VeniceWriterOptions; import java.io.File; @@ -172,262 +156,6 @@ public void cleanUp() { Utils.closeQuietlyWithErrorLogged(sharedVenice); } - @Test(timeOut = 180 * Time.MS_PER_SECOND) - public void testHybridInitializationOnMultiColo() throws IOException { - Properties extraProperties = new Properties(); - extraProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(3L)); - extraProperties.setProperty(ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED, "false"); - extraProperties.setProperty(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, "true"); - extraProperties.setProperty(SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE, "300"); - try ( - VeniceClusterWrapper venice = - ServiceFactory.getVeniceCluster(1, 2, 1, 1, 1000000, false, false, extraProperties); - ZkServerWrapper parentZk = ServiceFactory.getZkServer(); - VeniceControllerWrapper parentController = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder( - venice.getClusterName(), - parentZk, - venice.getPubSubBrokerWrapper()) - .childControllers(new VeniceControllerWrapper[] { venice.getLeaderVeniceController() }) - .build()); - ControllerClient controllerClient = - new ControllerClient(venice.getClusterName(), parentController.getControllerUrl()); - TopicManager topicManager = - IntegrationTestPushUtils - .getTopicManagerRepo( - PUBSUB_OPERATION_TIMEOUT_MS_DEFAULT_VALUE, - 100, - 0l, - venice.getPubSubBrokerWrapper(), - venice.getPubSubTopicRepository()) - .getLocalTopicManager()) { - long streamingRewindSeconds = 25L; - long streamingMessageLag = 2L; - final String storeName = Utils.getUniqueString("multi-colo-hybrid-store"); - - // Create store at parent, make it a hybrid store - controllerClient.createNewStore(storeName, "owner", STRING_SCHEMA.toString(), STRING_SCHEMA.toString()); - controllerClient.updateStore( - storeName, - new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) - .setHybridRewindSeconds(streamingRewindSeconds) - .setHybridOffsetLagThreshold(streamingMessageLag)); - - HybridStoreConfig hybridStoreConfig = new HybridStoreConfigImpl( - streamingRewindSeconds, - streamingMessageLag, - HybridStoreConfigImpl.DEFAULT_HYBRID_TIME_LAG_THRESHOLD, - DataReplicationPolicy.NON_AGGREGATE, - REWIND_FROM_EOP); - // There should be no version on the store yet - assertEquals( - controllerClient.getStore(storeName).getStore().getCurrentVersion(), - 0, - "The newly created store must have a current version of 0"); - - // Create a new version, and do an empty push for that version - VersionCreationResponse vcr = - controllerClient.emptyPush(storeName, Utils.getUniqueString("empty-hybrid-push"), 1L); - int versionNumber = vcr.getVersion(); - assertNotEquals(versionNumber, 0, "requesting a topic for a push should provide a non zero version number"); - - TestUtils.waitForNonDeterministicAssertion(100, TimeUnit.SECONDS, true, () -> { - // Now the store should have version 1 - JobStatusQueryResponse jobStatus = - controllerClient.queryJobStatus(Version.composeKafkaTopic(storeName, versionNumber)); - Assert.assertFalse(jobStatus.isError(), "Error in getting JobStatusResponse: " + jobStatus.getError()); - assertEquals(jobStatus.getStatus(), "COMPLETED"); - }); - vcr = controllerClient.emptyPush(storeName, Utils.getUniqueString("empty-hybrid-push1"), 1L); - VersionCreationResponse finalVcr = vcr; - TestUtils.waitForNonDeterministicAssertion(100, TimeUnit.SECONDS, true, () -> { - // Now the store should have version 2 - JobStatusQueryResponse jobStatus = - controllerClient.queryJobStatus(Version.composeKafkaTopic(storeName, finalVcr.getVersion())); - Assert.assertFalse(jobStatus.isError(), "Error in getting JobStatusResponse: " + jobStatus.getError()); - assertEquals(jobStatus.getStatus(), "COMPLETED"); - }); - MultiStoreStatusResponse response = controllerClient.getBackupVersions(venice.getClusterName(), storeName); - Assert.assertEquals(response.getStoreStatusMap().get("dc-0"), "1"); - - // And real-time topic should exist now. - assertTrue( - topicManager.containsTopicAndAllPartitionsAreOnline( - sharedVenice.getPubSubTopicRepository().getTopic(Version.composeRealTimeTopic(storeName)))); - // Creating a store object with default values since we're not updating bootstrap to online timeout - StoreProperties storeProperties = AvroRecordUtils.prefillAvroRecordWithDefaultValue(new StoreProperties()); - storeProperties.name = storeName; - storeProperties.owner = "owner"; - storeProperties.createdTime = System.currentTimeMillis(); - Store store = new ZKStore(storeProperties); - assertEquals( - topicManager.getTopicRetention( - sharedVenice.getPubSubTopicRepository().getTopic(Version.composeRealTimeTopic(storeName))), - StoreUtils.getExpectedRetentionTimeInMs(store, hybridStoreConfig), - "RT retention not configured properly"); - // Make sure RT retention is updated when the rewind time is updated - long newStreamingRewindSeconds = 600; - hybridStoreConfig.setRewindTimeInSeconds(newStreamingRewindSeconds); - controllerClient - .updateStore(storeName, new UpdateStoreQueryParams().setHybridRewindSeconds(newStreamingRewindSeconds)); - assertEquals( - topicManager.getTopicRetention( - sharedVenice.getPubSubTopicRepository().getTopic(Version.composeRealTimeTopic(storeName))), - StoreUtils.getExpectedRetentionTimeInMs(store, hybridStoreConfig), - "RT retention not updated properly"); - } - } - - @Test(timeOut = 180 * Time.MS_PER_SECOND) - public void testHybridSplitBrainIssue() { - Properties extraProperties = new Properties(); - extraProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(3L)); - extraProperties.setProperty(ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED, "false"); - extraProperties.setProperty(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, "true"); - extraProperties.setProperty(SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE, "300"); - try ( - VeniceClusterWrapper venice = - ServiceFactory.getVeniceCluster(1, 2, 1, 1, 1000000, false, false, extraProperties); - ZkServerWrapper parentZk = ServiceFactory.getZkServer(); - VeniceControllerWrapper parentController = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder( - venice.getClusterName(), - parentZk, - venice.getPubSubBrokerWrapper()) - .childControllers(new VeniceControllerWrapper[] { venice.getLeaderVeniceController() }) - .build()); - ControllerClient controllerClient = - new ControllerClient(venice.getClusterName(), parentController.getControllerUrl())) { - long streamingRewindSeconds = 25L; - long streamingMessageLag = 2L; - final String storeName = Utils.getUniqueString("hybrid-store"); - - // Create store at parent, make it a hybrid store - controllerClient.createNewStore(storeName, "owner", STRING_SCHEMA.toString(), STRING_SCHEMA.toString()); - controllerClient.updateStore( - storeName, - new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) - .setHybridRewindSeconds(streamingRewindSeconds) - .setHybridOffsetLagThreshold(streamingMessageLag)); - - // There should be no version on the store yet - assertEquals( - controllerClient.getStore(storeName).getStore().getCurrentVersion(), - 0, - "The newly created store must have a current version of 0"); - - VersionResponse versionResponse = controllerClient.addVersionAndStartIngestion( - storeName, - Utils.getUniqueString("test-hybrid-push"), - 1, - 3, - Version.PushType.BATCH, - null, - -1, - 1); - assertFalse( - versionResponse.isError(), - "Version creation shouldn't return error, but received: " + versionResponse.getError()); - String versionTopicName = Version.composeKafkaTopic(storeName, 1); - - String writer1 = "writer_1_hostname"; - String writer2 = "writer_2_hostname"; - Properties veniceWriterProperties1 = new Properties(); - veniceWriterProperties1.put(KAFKA_BOOTSTRAP_SERVERS, venice.getPubSubBrokerWrapper().getAddress()); - veniceWriterProperties1.putAll( - PubSubBrokerWrapper.getBrokerDetailsForClients(Collections.singletonList(venice.getPubSubBrokerWrapper()))); - veniceWriterProperties1.put(INSTANCE_ID, writer1); - - AvroSerializer stringSerializer = new AvroSerializer(STRING_SCHEMA); - PubSubProducerAdapterFactory pubSubProducerAdapterFactory = - venice.getPubSubBrokerWrapper().getPubSubClientsFactory().getProducerAdapterFactory(); - - Properties veniceWriterProperties2 = new Properties(); - veniceWriterProperties2.put(KAFKA_BOOTSTRAP_SERVERS, venice.getPubSubBrokerWrapper().getAddress()); - veniceWriterProperties2.putAll( - PubSubBrokerWrapper.getBrokerDetailsForClients(Collections.singletonList(venice.getPubSubBrokerWrapper()))); - veniceWriterProperties2.put(INSTANCE_ID, writer2); - - try ( - VeniceWriter veniceWriter1 = - TestUtils.getVeniceWriterFactory(veniceWriterProperties1, pubSubProducerAdapterFactory) - .createVeniceWriter(new VeniceWriterOptions.Builder(versionTopicName).build()); - VeniceWriter veniceWriter2 = - TestUtils.getVeniceWriterFactory(veniceWriterProperties2, pubSubProducerAdapterFactory) - .createVeniceWriter(new VeniceWriterOptions.Builder(versionTopicName).build())) { - veniceWriter1.broadcastStartOfPush(false, Collections.emptyMap()); - - /** - * Explicitly simulate split-brain issue. - * Writer1: - * - * key_0: value_0 with upstream offset: 5 - * key_1: value_1 with upstream offset: 6 - * key_2: value_2 with upstream offset: 7 - * key_3: value_3 with upstream offset: 8 - * key_4: value_4 with upstream offset: 9 - * Writer2: - * key_0: value_x with upstream offset: 3 - * key_5: value_5 with upstream offset: 10 - * key_6: value_6 with upstream offset: 11 - * key_7: value_7 with upstream offset: 12 - * key_8: value_8 with upstream offset: 13 - * key_9: value_9 with upstream offset: 14 - */ - - // Sending out dummy records first to push out SOS messages first. - veniceWriter1.put( - stringSerializer.serialize("key_writer_1"), - stringSerializer.serialize("value_writer_1"), - 1, - null, - new LeaderMetadataWrapper(0, 0)); - veniceWriter1.flush(); - veniceWriter2.put( - stringSerializer.serialize("key_writer_2"), - stringSerializer.serialize("value_writer_2"), - 1, - null, - new LeaderMetadataWrapper(1, 0)); - veniceWriter2.flush(); - - for (int i = 0; i < 5; ++i) { - veniceWriter1.put( - stringSerializer.serialize("key_" + i), - stringSerializer.serialize("value_" + i), - 1, - null, - new LeaderMetadataWrapper(i + 5, 0)); - } - veniceWriter1.flush(); - veniceWriter2.put( - stringSerializer.serialize("key_" + 0), - stringSerializer.serialize("value_x"), - 1, - null, - new LeaderMetadataWrapper(3, 0)); - for (int i = 5; i < 10; ++i) { - veniceWriter2.put( - stringSerializer.serialize("key_" + i), - stringSerializer.serialize("value_" + i), - 1, - null, - new LeaderMetadataWrapper(i + 5, 0)); - } - veniceWriter2.flush(); - veniceWriter1.broadcastEndOfPush(Collections.emptyMap()); - veniceWriter1.flush(); - } - - TestUtils.waitForNonDeterministicAssertion(100, TimeUnit.SECONDS, true, () -> { - // Now the store should have version 1 - JobStatusQueryResponse jobStatus = controllerClient.queryJobStatus(Version.composeKafkaTopic(storeName, 1)); - Assert.assertFalse(jobStatus.isError(), "Error in getting JobStatusResponse: " + jobStatus.getError()); - assertEquals(jobStatus.getStatus(), "ERROR"); - }); - } - } - /** * N.B.: Non-L/F does not support chunking, so this permutation is skipped. */ diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java new file mode 100644 index 0000000000..a86cc70987 --- /dev/null +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java @@ -0,0 +1,355 @@ +package com.linkedin.venice.endToEnd; + +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; +import static com.linkedin.venice.ConfigKeys.DEFAULT_MAX_NUMBER_OF_PARTITIONS; +import static com.linkedin.venice.ConfigKeys.INSTANCE_ID; +import static com.linkedin.venice.ConfigKeys.KAFKA_BOOTSTRAP_SERVERS; +import static com.linkedin.venice.ConfigKeys.PERSISTENCE_TYPE; +import static com.linkedin.venice.ConfigKeys.SERVER_CONSUMER_POOL_SIZE_PER_KAFKA_CLUSTER; +import static com.linkedin.venice.ConfigKeys.SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED; +import static com.linkedin.venice.ConfigKeys.SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE; +import static com.linkedin.venice.ConfigKeys.SERVER_DEDICATED_DRAINER_FOR_SORTED_INPUT_ENABLED; +import static com.linkedin.venice.ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS; +import static com.linkedin.venice.ConfigKeys.SSL_TO_KAFKA_LEGACY; +import static com.linkedin.venice.meta.BufferReplayPolicy.REWIND_FROM_EOP; +import static com.linkedin.venice.meta.BufferReplayPolicy.REWIND_FROM_SOP; +import static com.linkedin.venice.pubsub.PubSubConstants.PUBSUB_OPERATION_TIMEOUT_MS_DEFAULT_VALUE; +import static com.linkedin.venice.utils.TestWriteUtils.STRING_SCHEMA; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; + +import com.linkedin.venice.controllerapi.ControllerClient; +import com.linkedin.venice.controllerapi.JobStatusQueryResponse; +import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; +import com.linkedin.venice.controllerapi.VersionCreationResponse; +import com.linkedin.venice.controllerapi.VersionResponse; +import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; +import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; +import com.linkedin.venice.meta.DataReplicationPolicy; +import com.linkedin.venice.meta.HybridStoreConfig; +import com.linkedin.venice.meta.HybridStoreConfigImpl; +import com.linkedin.venice.meta.PersistenceType; +import com.linkedin.venice.meta.Store; +import com.linkedin.venice.meta.Version; +import com.linkedin.venice.meta.ZKStore; +import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; +import com.linkedin.venice.pubsub.manager.TopicManager; +import com.linkedin.venice.serializer.AvroSerializer; +import com.linkedin.venice.systemstore.schemas.StoreProperties; +import com.linkedin.venice.utils.AvroRecordUtils; +import com.linkedin.venice.utils.IntegrationTestPushUtils; +import com.linkedin.venice.utils.StoreUtils; +import com.linkedin.venice.utils.TestUtils; +import com.linkedin.venice.utils.Time; +import com.linkedin.venice.utils.Utils; +import com.linkedin.venice.writer.LeaderMetadataWrapper; +import com.linkedin.venice.writer.VeniceWriter; +import com.linkedin.venice.writer.VeniceWriterOptions; +import java.io.IOException; +import java.util.Collections; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + + +public class TestHybridMultiRegion { + /** + * IMPORTANT NOTE: if you use this sharedVenice cluster, please do not close it. The {@link #cleanUp()} function + * will take care of it. Besides, if any backend component of the shared cluster is stopped in + * the middle of the test, please restart them at the end of your test. + */ + private VeniceTwoLayerMultiRegionMultiClusterWrapper sharedVenice; + + /** + * This cluster is re-used by some of the tests, in order to speed up the suite. Some other tests require + * certain specific characteristics which makes it awkward to re-use, though not necessarily impossible. + * Further reuse of this shared cluster can be attempted later. + */ + @BeforeClass(alwaysRun = true) + public void setUp() { + sharedVenice = setUpCluster(); + } + + @AfterClass(alwaysRun = true) + public void cleanUp() { + Utils.closeQuietlyWithErrorLogged(sharedVenice); + } + + @Test(timeOut = 180 * Time.MS_PER_SECOND) + public void testHybridInitializationOnMultiColo() throws IOException { + String clusterName = sharedVenice.getClusterNames()[0]; + VeniceClusterWrapper sharedVeniceClusterWrapper = + sharedVenice.getChildRegions().get(0).getClusters().get(clusterName); + try ( + ControllerClient controllerClient = + new ControllerClient(clusterName, sharedVenice.getControllerConnectString()); + TopicManager topicManager = + IntegrationTestPushUtils + .getTopicManagerRepo( + PUBSUB_OPERATION_TIMEOUT_MS_DEFAULT_VALUE, + 100, + 0l, + sharedVeniceClusterWrapper.getPubSubBrokerWrapper(), + sharedVeniceClusterWrapper.getPubSubTopicRepository()) + .getLocalTopicManager()) { + long streamingRewindSeconds = 25L; + long streamingMessageLag = 2L; + final String storeName = Utils.getUniqueString("multi-colo-hybrid-store"); + + // Create store at parent, make it a hybrid store + controllerClient.createNewStore(storeName, "owner", STRING_SCHEMA.toString(), STRING_SCHEMA.toString()); + controllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) + .setHybridRewindSeconds(streamingRewindSeconds) + .setHybridOffsetLagThreshold(streamingMessageLag)); + + HybridStoreConfig hybridStoreConfig = new HybridStoreConfigImpl( + streamingRewindSeconds, + streamingMessageLag, + HybridStoreConfigImpl.DEFAULT_HYBRID_TIME_LAG_THRESHOLD, + DataReplicationPolicy.NON_AGGREGATE, + REWIND_FROM_EOP); + // There should be no version on the store yet + assertEquals( + controllerClient.getStore(storeName).getStore().getCurrentVersion(), + 0, + "The newly created store must have a current version of 0"); + + // Create a new version, and do an empty push for that version + VersionCreationResponse vcr = + controllerClient.emptyPush(storeName, Utils.getUniqueString("empty-hybrid-push"), 1L); + int versionNumber = vcr.getVersion(); + assertNotEquals(versionNumber, 0, "requesting a topic for a push should provide a non zero version number"); + + TestUtils.waitForNonDeterministicAssertion(100, TimeUnit.SECONDS, true, () -> { + // Now the store should have version 1 + JobStatusQueryResponse jobStatus = + controllerClient.queryJobStatus(Version.composeKafkaTopic(storeName, versionNumber)); + Assert.assertFalse(jobStatus.isError(), "Error in getting JobStatusResponse: " + jobStatus.getError()); + assertEquals(jobStatus.getStatus(), "COMPLETED"); + }); + + // And real-time topic should exist now. + assertTrue( + topicManager.containsTopicAndAllPartitionsAreOnline( + sharedVeniceClusterWrapper.getPubSubTopicRepository().getTopic(Version.composeRealTimeTopic(storeName)))); + // Creating a store object with default values since we're not updating bootstrap to online timeout + StoreProperties storeProperties = AvroRecordUtils.prefillAvroRecordWithDefaultValue(new StoreProperties()); + storeProperties.name = storeName; + storeProperties.owner = "owner"; + storeProperties.createdTime = System.currentTimeMillis(); + Store store = new ZKStore(storeProperties); + assertEquals( + topicManager.getTopicRetention( + sharedVeniceClusterWrapper.getPubSubTopicRepository().getTopic(Version.composeRealTimeTopic(storeName))), + StoreUtils.getExpectedRetentionTimeInMs(store, hybridStoreConfig), + "RT retention not configured properly"); + // Make sure RT retention is updated when the rewind time is updated + long newStreamingRewindSeconds = 600; + hybridStoreConfig.setRewindTimeInSeconds(newStreamingRewindSeconds); + controllerClient + .updateStore(storeName, new UpdateStoreQueryParams().setHybridRewindSeconds(newStreamingRewindSeconds)); + assertEquals( + topicManager.getTopicRetention( + sharedVeniceClusterWrapper.getPubSubTopicRepository().getTopic(Version.composeRealTimeTopic(storeName))), + StoreUtils.getExpectedRetentionTimeInMs(store, hybridStoreConfig), + "RT retention not updated properly"); + } + } + + @Test(timeOut = 180 * Time.MS_PER_SECOND) + public void testHybridSplitBrainIssue() { + String clusterName = sharedVenice.getClusterNames()[0]; + VeniceClusterWrapper sharedVeniceClusterWrapper = + sharedVenice.getChildRegions().get(0).getClusters().get(clusterName); + try (ControllerClient controllerClient = + new ControllerClient(clusterName, sharedVenice.getControllerConnectString())) { + long streamingRewindSeconds = 25L; + long streamingMessageLag = 2L; + final String storeName = Utils.getUniqueString("hybrid-store"); + + // Create store at parent, make it a hybrid store + controllerClient.createNewStore(storeName, "owner", STRING_SCHEMA.toString(), STRING_SCHEMA.toString()); + controllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) + .setHybridRewindSeconds(streamingRewindSeconds) + .setHybridOffsetLagThreshold(streamingMessageLag)); + + // There should be no version on the store yet + assertEquals( + controllerClient.getStore(storeName).getStore().getCurrentVersion(), + 0, + "The newly created store must have a current version of 0"); + + VersionResponse versionResponse = controllerClient.addVersionAndStartIngestion( + storeName, + Utils.getUniqueString("test-hybrid-push"), + 1, + 3, + Version.PushType.BATCH, + null, + -1, + 1); + assertFalse( + versionResponse.isError(), + "Version creation shouldn't return error, but received: " + versionResponse.getError()); + String versionTopicName = Version.composeKafkaTopic(storeName, 1); + + String writer1 = "writer_1_hostname"; + String writer2 = "writer_2_hostname"; + Properties veniceWriterProperties1 = new Properties(); + veniceWriterProperties1 + .put(KAFKA_BOOTSTRAP_SERVERS, sharedVeniceClusterWrapper.getPubSubBrokerWrapper().getAddress()); + veniceWriterProperties1.putAll( + PubSubBrokerWrapper.getBrokerDetailsForClients( + Collections.singletonList(sharedVeniceClusterWrapper.getPubSubBrokerWrapper()))); + veniceWriterProperties1.put(INSTANCE_ID, writer1); + + AvroSerializer stringSerializer = new AvroSerializer(STRING_SCHEMA); + PubSubProducerAdapterFactory pubSubProducerAdapterFactory = + sharedVeniceClusterWrapper.getPubSubBrokerWrapper().getPubSubClientsFactory().getProducerAdapterFactory(); + + Properties veniceWriterProperties2 = new Properties(); + veniceWriterProperties2 + .put(KAFKA_BOOTSTRAP_SERVERS, sharedVeniceClusterWrapper.getPubSubBrokerWrapper().getAddress()); + veniceWriterProperties2.putAll( + PubSubBrokerWrapper.getBrokerDetailsForClients( + Collections.singletonList(sharedVeniceClusterWrapper.getPubSubBrokerWrapper()))); + veniceWriterProperties2.put(INSTANCE_ID, writer2); + + try ( + VeniceWriter veniceWriter1 = + TestUtils.getVeniceWriterFactory(veniceWriterProperties1, pubSubProducerAdapterFactory) + .createVeniceWriter(new VeniceWriterOptions.Builder(versionTopicName).build()); + VeniceWriter veniceWriter2 = + TestUtils.getVeniceWriterFactory(veniceWriterProperties2, pubSubProducerAdapterFactory) + .createVeniceWriter(new VeniceWriterOptions.Builder(versionTopicName).build())) { + veniceWriter1.broadcastStartOfPush(false, Collections.emptyMap()); + + /** + * Explicitly simulate split-brain issue. + * Writer1: + * + * key_0: value_0 with upstream offset: 5 + * key_1: value_1 with upstream offset: 6 + * key_2: value_2 with upstream offset: 7 + * key_3: value_3 with upstream offset: 8 + * key_4: value_4 with upstream offset: 9 + * Writer2: + * key_0: value_x with upstream offset: 3 + * key_5: value_5 with upstream offset: 10 + * key_6: value_6 with upstream offset: 11 + * key_7: value_7 with upstream offset: 12 + * key_8: value_8 with upstream offset: 13 + * key_9: value_9 with upstream offset: 14 + */ + + // Sending out dummy records first to push out SOS messages first. + veniceWriter1.put( + stringSerializer.serialize("key_writer_1"), + stringSerializer.serialize("value_writer_1"), + 1, + null, + new LeaderMetadataWrapper(0, 0)); + veniceWriter1.flush(); + veniceWriter2.put( + stringSerializer.serialize("key_writer_2"), + stringSerializer.serialize("value_writer_2"), + 1, + null, + new LeaderMetadataWrapper(1, 0)); + veniceWriter2.flush(); + + for (int i = 0; i < 5; ++i) { + veniceWriter1.put( + stringSerializer.serialize("key_" + i), + stringSerializer.serialize("value_" + i), + 1, + null, + new LeaderMetadataWrapper(i + 5, 0)); + } + veniceWriter1.flush(); + veniceWriter2.put( + stringSerializer.serialize("key_" + 0), + stringSerializer.serialize("value_x"), + 1, + null, + new LeaderMetadataWrapper(3, 0)); + for (int i = 5; i < 10; ++i) { + veniceWriter2.put( + stringSerializer.serialize("key_" + i), + stringSerializer.serialize("value_" + i), + 1, + null, + new LeaderMetadataWrapper(i + 5, 0)); + } + veniceWriter2.flush(); + veniceWriter1.broadcastEndOfPush(Collections.emptyMap()); + veniceWriter1.flush(); + } + + TestUtils.waitForNonDeterministicAssertion(100, TimeUnit.SECONDS, true, () -> { + // Now the store should have version 1 + JobStatusQueryResponse jobStatus = controllerClient.queryJobStatus(Version.composeKafkaTopic(storeName, 1)); + Assert.assertFalse(jobStatus.isError(), "Error in getting JobStatusResponse: " + jobStatus.getError()); + assertEquals(jobStatus.getStatus(), "ERROR"); + }); + } + } + + /** + * N.B.: Non-L/F does not support chunking, so this permutation is skipped. + */ + @DataProvider(name = "testPermutations", parallel = false) + public static Object[][] testPermutations() { + return new Object[][] { { false, false, REWIND_FROM_EOP }, { false, true, REWIND_FROM_EOP }, + { true, false, REWIND_FROM_EOP }, { true, true, REWIND_FROM_EOP }, { false, false, REWIND_FROM_SOP }, + { false, true, REWIND_FROM_SOP }, { true, false, REWIND_FROM_SOP }, { true, true, REWIND_FROM_SOP } }; + } + + private static VeniceTwoLayerMultiRegionMultiClusterWrapper setUpCluster() { + Properties parentControllerProps = new Properties(); + parentControllerProps.setProperty(DEFAULT_MAX_NUMBER_OF_PARTITIONS, "5"); + + Properties childControllerProperties = new Properties(); + childControllerProperties.setProperty(DEFAULT_MAX_NUMBER_OF_PARTITIONS, "5"); + + Properties serverProperties = new Properties(); + serverProperties.setProperty(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB.name()); + serverProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(3L)); + serverProperties.setProperty(ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED, "false"); + serverProperties.setProperty(SERVER_DATABASE_CHECKSUM_VERIFICATION_ENABLED, "true"); + serverProperties.setProperty(SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE, "300"); + + serverProperties.setProperty(SSL_TO_KAFKA_LEGACY, "false"); + serverProperties.setProperty(SERVER_CONSUMER_POOL_SIZE_PER_KAFKA_CLUSTER, "3"); + serverProperties.setProperty(SERVER_DEDICATED_DRAINER_FOR_SORTED_INPUT_ENABLED, "true"); + + VeniceTwoLayerMultiRegionMultiClusterWrapper cluster = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + 1, + 1, + 1, + 1, + 2, + 1, + 1, + Optional.of(parentControllerProps), + Optional.of(childControllerProperties), + Optional.of(serverProperties), + false); + + return cluster; + } +} diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridStoreDeletion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridStoreDeletion.java index 360f20d286..e939f4b3d9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridStoreDeletion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridStoreDeletion.java @@ -11,29 +11,24 @@ import static com.linkedin.venice.ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS; import static com.linkedin.venice.ConfigKeys.SERVER_SHARED_CONSUMER_ASSIGNMENT_STRATEGY; import static com.linkedin.venice.ConfigKeys.SSL_TO_KAFKA_LEGACY; +import static com.linkedin.venice.pubsub.PubSubConstants.PUBSUB_OPERATION_TIMEOUT_MS_DEFAULT_VALUE; import static com.linkedin.venice.utils.IntegrationTestPushUtils.getSamzaProducer; import static com.linkedin.venice.utils.IntegrationTestPushUtils.sendCustomSizeStreamingRecord; import static com.linkedin.venice.utils.TestWriteUtils.STRING_SCHEMA; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; import com.linkedin.davinci.kafka.consumer.KafkaConsumerService; import com.linkedin.venice.client.store.AvroGenericStoreClient; import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; -import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; -import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; -import com.linkedin.venice.integration.utils.VeniceControllerCreateOptions; -import com.linkedin.venice.integration.utils.VeniceControllerWrapper; -import com.linkedin.venice.integration.utils.ZkServerWrapper; import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.Version; -import com.linkedin.venice.pubsub.PubSubConstants; import com.linkedin.venice.pubsub.api.PubSubTopic; import com.linkedin.venice.pubsub.manager.TopicManager; import com.linkedin.venice.utils.IntegrationTestPushUtils; @@ -55,11 +50,8 @@ public class TestHybridStoreDeletion { private static final Logger LOGGER = LogManager.getLogger(TestHybridStoreDeletion.class); public static final int STREAMING_RECORD_SIZE = 1024; - public static final int NUMBER_OF_SERVERS = 1; private VeniceClusterWrapper veniceCluster; - ZkServerWrapper parentZk = null; - VeniceControllerWrapper parentController = null; @BeforeClass(alwaysRun = true) public void setUp() { @@ -68,21 +60,10 @@ public void setUp() { @AfterClass(alwaysRun = true) public void cleanUp() { - parentController.close(); - parentZk.close(); Utils.closeQuietlyWithErrorLogged(veniceCluster); } private static VeniceClusterWrapper setUpCluster() { - Properties extraProperties = new Properties(); - extraProperties.setProperty(DEFAULT_MAX_NUMBER_OF_PARTITIONS, "5"); - VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 1, 1, 1000000, false, false, extraProperties); - - // Add Venice Router - Properties routerProperties = new Properties(); - cluster.addVeniceRouter(routerProperties); - - // Add Venice Server Properties serverProperties = new Properties(); serverProperties.setProperty(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB.name()); serverProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1L)); @@ -103,17 +84,19 @@ private static VeniceClusterWrapper setUpCluster() { SERVER_SHARED_CONSUMER_ASSIGNMENT_STRATEGY, KafkaConsumerService.ConsumerAssignmentStrategy.PARTITION_WISE_SHARED_CONSUMER_ASSIGNMENT_STRATEGY.name()); - for (int i = 0; i < NUMBER_OF_SERVERS; i++) { - cluster.addVeniceServer(new Properties(), serverProperties); - } - - return cluster; + return ServiceFactory.getVeniceCluster( + new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .extraProperties(serverProperties) + .build()); } /** * testHybridStoreRTDeletionWhileIngesting does the following: * - * 1. Set up a Venice cluster with 1 controller, 1 router, and 1 server. + * 1. Set up a Venice cluster with 1 parent controller, 1 child controller, 1 router, and 1 server. * 2. Limit the shared consumer thread pool size to 1 on the server. * 3. Create two hybrid stores. * 4. Produce to the rt topic of the first store and allow the thread to produce some amount of data. @@ -123,43 +106,37 @@ private static VeniceClusterWrapper setUpCluster() { */ @Test(timeOut = 120 * Time.MS_PER_SECOND) public void testHybridStoreRTDeletionWhileIngesting() { - parentZk = ServiceFactory.getZkServer(); - parentController = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder( - veniceCluster.getClusterName(), - parentZk, - veniceCluster.getPubSubBrokerWrapper()) - .childControllers(new VeniceControllerWrapper[] { veniceCluster.getLeaderVeniceController() }) - .build()); - long streamingRewindSeconds = 25; long streamingMessageLag = 2; final String storeNameFirst = Utils.getUniqueString("hybrid-store-test-first"); final String storeNameSecond = Utils.getUniqueString("hybrid-store-test-second"); final String[] storeNames = new String[] { storeNameFirst, storeNameSecond }; - try ( - ControllerClient controllerClient = - new ControllerClient(veniceCluster.getClusterName(), parentController.getControllerUrl()); - AvroGenericStoreClient clientToSecondStore = ClientFactory.getAndStartGenericAvroClient( - ClientConfig.defaultGenericClientConfig(storeNameSecond).setVeniceURL(veniceCluster.getRandomRouterURL())); - TopicManager topicManager = - IntegrationTestPushUtils - .getTopicManagerRepo( - PubSubConstants.PUBSUB_OPERATION_TIMEOUT_MS_DEFAULT_VALUE, - 100, - 0l, - veniceCluster.getPubSubBrokerWrapper(), - veniceCluster.getPubSubTopicRepository()) - .getLocalTopicManager()) { - - createStoresAndVersions(controllerClient, storeNames, streamingRewindSeconds, streamingMessageLag); + try (TopicManager topicManager = + IntegrationTestPushUtils + .getTopicManagerRepo( + PUBSUB_OPERATION_TIMEOUT_MS_DEFAULT_VALUE, + 100, + 0l, + veniceCluster.getPubSubBrokerWrapper(), + veniceCluster.getPubSubTopicRepository()) + .getLocalTopicManager()) { + + createStoresAndVersions(storeNames, streamingRewindSeconds, streamingMessageLag); + + // Wait until the rt topic of the first store is fully deleted. + PubSubTopic rtTopicFirst1 = + veniceCluster.getPubSubTopicRepository().getTopic(Version.composeRealTimeTopic(storeNameFirst)); + TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, true, true, () -> { + Assert.assertTrue(topicManager.containsTopic(rtTopicFirst1)); + }); // Produce to the rt topic of the first store and allow the thread to produce some amount of data. produceToStoreRTTopic(storeNameFirst, 200); // Delete the rt topic of the first store. - controllerClient.deleteKafkaTopic(Version.composeRealTimeTopic(storeNameFirst)); + topicManager.ensureTopicIsDeletedAndBlock( + veniceCluster.getPubSubTopicRepository().getTopic(Version.composeRealTimeTopic(storeNameFirst))); // Wait until the rt topic of the first store is fully deleted. PubSubTopic rtTopicFirst = @@ -171,17 +148,20 @@ public void testHybridStoreRTDeletionWhileIngesting() { // Produce to the rt topic of the second store with 10 key-value pairs. produceToStoreRTTopic(storeNameSecond, 10); - // Check that the second store has all the records. - TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, true, true, () -> { - try { - for (int i = 1; i <= 10; i++) { - checkLargeRecord(clientToSecondStore, i); - LOGGER.info("Checked record {}", i); + try (AvroGenericStoreClient clientToSecondStore = ClientFactory.getAndStartGenericAvroClient( + ClientConfig.defaultGenericClientConfig(storeNameSecond).setVeniceURL(veniceCluster.getRandomRouterURL()))) { + // Check that the second store has all the records. + TestUtils.waitForNonDeterministicAssertion(10, TimeUnit.SECONDS, true, true, () -> { + try { + for (int i = 1; i <= 10; i++) { + checkLargeRecord(clientToSecondStore, i); + LOGGER.info("Checked record {}", i); + } + } catch (Exception e) { + throw new VeniceException(e); } - } catch (Exception e) { - throw new VeniceException(e); - } - }); + }); + } } } @@ -211,32 +191,28 @@ private void checkLargeRecord(AvroGenericStoreClient client, int index) } } - private void createStoresAndVersions( - ControllerClient controllerClient, - String[] storeNames, - long streamingRewindSeconds, - long streamingMessageLag) { - for (String storeName: storeNames) { - // Create store at parent, make it a hybrid store. - controllerClient.createNewStore(storeName, "owner", STRING_SCHEMA.toString(), STRING_SCHEMA.toString()); - controllerClient.updateStore( - storeName, - new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) - .setHybridRewindSeconds(streamingRewindSeconds) - .setPartitionCount(1) - .setHybridOffsetLagThreshold(streamingMessageLag)); - - // There should be no version on the store yet. - assertEquals( - controllerClient.getStore(storeName).getStore().getCurrentVersion(), - 0, - "The newly created store must have a current version of 0"); - - // Create a new version, and do an empty push for that version. - VersionCreationResponse vcr = - controllerClient.emptyPush(storeName, Utils.getUniqueString("empty-hybrid-push"), 1L); - int versionNumber = vcr.getVersion(); - assertNotEquals(versionNumber, 0, "requesting a topic for a push should provide a non zero version number"); - } + private void createStoresAndVersions(String[] storeNames, long streamingRewindSeconds, long streamingMessageLag) { + veniceCluster.useControllerClient(controllerClient -> { + for (String storeName: storeNames) { + // Create store at parent, make it a hybrid store. + controllerClient.createNewStore(storeName, "owner", STRING_SCHEMA.toString(), STRING_SCHEMA.toString()); + controllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) + .setHybridRewindSeconds(streamingRewindSeconds) + .setPartitionCount(1) + .setHybridOffsetLagThreshold(streamingMessageLag)); + + // There should be no version on the store yet. + assertEquals( + controllerClient.getStore(storeName).getStore().getCurrentVersion(), + 0, + "The newly created store must have a current version of 0"); + + // Create a new version, and do an empty push for that version. + controllerClient + .sendEmptyPushAndWait(storeName, Utils.getUniqueString("empty-hybrid-push"), 1L, 60L * Time.MS_PER_SECOND); + } + }); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java index 435a34461e..1171355bc9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java @@ -77,9 +77,7 @@ public class TestPartialUpdateWithActiveActiveReplication { public static final String NULLABLE_MAP_FIELD = "nullableMapField"; private static final Map MAP_FIELD_DEFAULT_VALUE = Collections.emptyMap(); - private static final Map NULLABLE_MAP_FIELD_DEFAULT_VALUE = null; private static final List LIST_FIELD_DEFAULT_VALUE = Collections.emptyList(); - private static final List NULLABLE_LIST_FIELD_DEFAULT_VALUE = null; private static final String REGULAR_FIELD_DEFAULT_VALUE = "default_venice"; @@ -208,7 +206,6 @@ public void testAAReplicationForPartialUpdateOnFields() throws IOException { .setActiveActiveReplicationEnabled(true) .setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) .setChunkingEnabled(false) - .setIncrementalPushEnabled(true) .setHybridRewindSeconds(25L) .setHybridOffsetLagThreshold(1L) .setWriteComputationEnabled(true); @@ -435,7 +432,6 @@ public void testActiveActivePartialUpdateWithRecordMapField() throws IOException .setActiveActiveReplicationEnabled(true) .setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) .setChunkingEnabled(true) - .setIncrementalPushEnabled(true) .setHybridRewindSeconds(25L) .setHybridOffsetLagThreshold(1L) .setWriteComputationEnabled(true); @@ -490,7 +486,6 @@ public void testAAReplicationForPartialUpdateOnMapField() throws IOException { .setActiveActiveReplicationEnabled(true) .setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) .setChunkingEnabled(true) - .setIncrementalPushEnabled(true) .setHybridRewindSeconds(25L) .setHybridOffsetLagThreshold(1L) .setWriteComputationEnabled(true); @@ -933,7 +928,6 @@ public void testAAReplicationForPartialUpdateOnListField() throws IOException { .setActiveActiveReplicationEnabled(true) .setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) .setChunkingEnabled(false) - .setIncrementalPushEnabled(true) .setHybridRewindSeconds(25L) .setHybridOffsetLagThreshold(1L) .setWriteComputationEnabled(true); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java index f581b2c8ed..5de7f6370d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java @@ -61,7 +61,6 @@ import com.linkedin.venice.common.VeniceSystemStoreUtils; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.MultiSchemaResponse; -import com.linkedin.venice.controllerapi.NewStoreResponse; import com.linkedin.venice.controllerapi.StoreResponse; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; @@ -82,7 +81,6 @@ import com.linkedin.venice.meta.Instance; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.StoreInfo; -import com.linkedin.venice.meta.VeniceUserStoreType; import com.linkedin.venice.meta.Version; import com.linkedin.venice.offsets.OffsetRecord; import com.linkedin.venice.pubsub.PubSubTopicPartitionImpl; @@ -482,7 +480,7 @@ public void testNativeReplicationForIncrementalPush() throws Exception { updateStoreQueryParams -> updateStoreQueryParams.setPartitionCount(1) .setHybridOffsetLagThreshold(TEST_TIMEOUT) .setHybridRewindSeconds(2L) - .setIncrementalPushEnabled(true), + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE), 100, (parentControllerClient, clusterName, storeName, props, inputDir) -> { try (VenicePushJob job = new VenicePushJob("Batch Push", props)) { @@ -510,7 +508,9 @@ public void testActiveActiveForHeartbeatSystemStores() throws Exception { motherOfAllTests( "testActiveActiveForHeartbeatSystemStores", updateStoreQueryParams -> updateStoreQueryParams.setPartitionCount(partitionCount) - .setIncrementalPushEnabled(true), + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) + .setHybridRewindSeconds(Time.SECONDS_PER_DAY) + .setHybridOffsetLagThreshold(1000), recordCount, (parentControllerClient, clusterName, storeName, props, inputDir) -> { try ( @@ -576,172 +576,6 @@ public void testActiveActiveForHeartbeatSystemStores() throws Exception { }); } - @Test(timeOut = TEST_TIMEOUT) - public void testClusterLevelAdminCommandForNativeReplication() throws Exception { - motherOfAllTests( - "testClusterLevelAdminCommandForNativeReplication", - updateStoreQueryParams -> updateStoreQueryParams.setPartitionCount(1), - 10, - (parentControllerClient, clusterName, batchOnlyStoreName, props, inputDir) -> { - // Create a hybrid store - String hybridStoreName = Utils.getUniqueString("hybrid-store"); - NewStoreResponse newStoreResponse = parentControllerClient - .createNewStore(hybridStoreName, "", STRING_SCHEMA.toString(), STRING_SCHEMA.toString()); - Assert.assertFalse(newStoreResponse.isError()); - UpdateStoreQueryParams updateStoreParams = - new UpdateStoreQueryParams().setHybridRewindSeconds(10).setHybridOffsetLagThreshold(2); - assertCommand(parentControllerClient.updateStore(hybridStoreName, updateStoreParams)); - - /** - * Create an incremental push enabled store - */ - String incrementPushStoreName = Utils.getUniqueString("incremental-push-store"); - newStoreResponse = parentControllerClient - .createNewStore(incrementPushStoreName, "", STRING_SCHEMA.toString(), STRING_SCHEMA.toString()); - Assert.assertFalse(newStoreResponse.isError()); - updateStoreParams = new UpdateStoreQueryParams().setIncrementalPushEnabled(true); - assertCommand(parentControllerClient.updateStore(incrementPushStoreName, updateStoreParams)); - - final Optional defaultNativeReplicationSource = Optional.of(DEFAULT_NATIVE_REPLICATION_SOURCE); - final Optional newNativeReplicationSource = Optional.of("new-nr-source"); - /** - * Run admin command to disable native replication for all batch-only stores in the cluster - */ - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - false, - VeniceUserStoreType.BATCH_ONLY.toString(), - Optional.empty(), - Optional.empty())); - - childDatacenters.get(0) - .getClusters() - .get(clusterName) - .useControllerClient( - dc0Client -> childDatacenters.get(1).getClusters().get(clusterName).useControllerClient(dc1Client -> { - List allControllerClients = - Arrays.asList(parentControllerClient, dc0Client, dc1Client); - - /** - * Batch-only stores should have native replication enabled; hybrid stores or incremental push stores - * have native replication enabled with dc-0 as source. - */ - NativeReplicationTestUtils - .verifyDCConfigNativeRepl(allControllerClients, batchOnlyStoreName, false); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - hybridStoreName, - true, - defaultNativeReplicationSource); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - incrementPushStoreName, - true, - defaultNativeReplicationSource); - - /** - * Second test: - * 1. Revert the cluster to previous state - * 2. Test the cluster level command that converts all hybrid stores to native replication - */ - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - true, - VeniceUserStoreType.BATCH_ONLY.toString(), - newNativeReplicationSource, - Optional.empty())); - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - false, - VeniceUserStoreType.HYBRID_ONLY.toString(), - Optional.empty(), - Optional.empty())); - - /** - * Hybrid stores shouldn't have native replication enabled; batch-only stores should have native replication - * enabled with the new source fabric and incremental push stores should have native replication enabled - * with original source fabric. - */ - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - batchOnlyStoreName, - true, - newNativeReplicationSource); - NativeReplicationTestUtils.verifyDCConfigNativeRepl(allControllerClients, hybridStoreName, false); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - incrementPushStoreName, - true, - defaultNativeReplicationSource); - - /** - * Third test: - * 1. Revert the cluster to previous state - * 2. Test the cluster level command that disables native replication for all incremental push stores - */ - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - true, - VeniceUserStoreType.HYBRID_ONLY.toString(), - newNativeReplicationSource, - Optional.empty())); - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - false, - VeniceUserStoreType.INCREMENTAL_PUSH.toString(), - Optional.empty(), - Optional.empty())); - - /** - * Incremental push stores shouldn't have native replication enabled; batch-only stores and hybrid stores - * should have native replication enabled with the new source fabric. - */ - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - batchOnlyStoreName, - true, - newNativeReplicationSource); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - hybridStoreName, - true, - newNativeReplicationSource); - NativeReplicationTestUtils - .verifyDCConfigNativeRepl(allControllerClients, incrementPushStoreName, false); - - /** - * Fourth test: - * Test the cluster level command that enables native replication for all incremental push stores - */ - assertCommand( - parentControllerClient.configureNativeReplicationForCluster( - true, - VeniceUserStoreType.INCREMENTAL_PUSH.toString(), - newNativeReplicationSource, - Optional.empty())); - - /** - * All stores should have native replication enabled with the new source fabric - */ - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - batchOnlyStoreName, - true, - newNativeReplicationSource); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - hybridStoreName, - true, - newNativeReplicationSource); - NativeReplicationTestUtils.verifyDCConfigNativeRepl( - allControllerClients, - incrementPushStoreName, - true, - newNativeReplicationSource); - })); - }); - } - @Test(timeOut = TEST_TIMEOUT) public void testMultiDataCenterRePushWithIncrementalPush() throws Exception { motherOfAllTests( @@ -761,7 +595,7 @@ public void testMultiDataCenterRePushWithIncrementalPush() throws Exception { parentControllerClient .updateStore( storeName, - new UpdateStoreQueryParams().setIncrementalPushEnabled(true) + new UpdateStoreQueryParams().setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) .setHybridOffsetLagThreshold(1) .setHybridRewindSeconds(Time.SECONDS_PER_DAY)) .isError()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java index b748b96784..2c177a8e0f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java @@ -102,18 +102,9 @@ public void testPushJobWithSourceGridFabricSelection(int recordCount, int partit String storeName = Utils.getUniqueString("store"); String parentControllerUrls = multiRegionMultiClusterWrapper.getControllerConnectString(); - // Enable NR in all regions and A/A in parent region and 1 child region only. The NR source fabric cluster level - // config is - // dc-0 by default. + // Enable A/A in parent region and 1 child region only. The NR source fabric cluster level + // config is dc-0 by default. try (ControllerClient parentControllerClient = new ControllerClient(clusterName, parentControllerUrls)) { - Assert.assertFalse( - parentControllerClient - .configureNativeReplicationForCluster( - true, - VeniceUserStoreType.BATCH_ONLY.toString(), - Optional.empty(), - Optional.of(String.join(",", parentControllerRegionName, dcNames[0], dcNames[1]))) - .isError()); Assert.assertFalse( parentControllerClient .configureActiveActiveReplicationForCluster( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java index 7116447ecd..eab70f0733 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java @@ -1,6 +1,7 @@ package com.linkedin.venice.endToEnd; import static com.linkedin.venice.utils.IntegrationTestPushUtils.createStoreForJob; +import static com.linkedin.venice.utils.TestUtils.assertCommand; import static com.linkedin.venice.utils.TestWriteUtils.getTempDataDirectory; import com.linkedin.venice.ConfigKeys; @@ -8,11 +9,13 @@ import com.linkedin.venice.controllerapi.ClusterStaleDataAuditResponse; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.StoreHealthAuditResponse; +import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; +import com.linkedin.venice.meta.StoreDataAudit; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.meta.Version; import com.linkedin.venice.utils.IntegrationTestPushUtils; @@ -21,10 +24,8 @@ import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; import java.io.File; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -48,7 +49,6 @@ public class TestStaleDataVisibility { IntStream.range(0, NUMBER_OF_CLUSTERS).mapToObj(i -> "venice-cluster" + i).toArray(String[]::new); private List childClusters; - private List> childControllers; private List parentControllers; private VeniceTwoLayerMultiRegionMultiClusterWrapper multiRegionMultiClusterWrapper; @@ -56,8 +56,6 @@ public class TestStaleDataVisibility { public void setUp() { Properties serverProperties = new Properties(); serverProperties.setProperty(ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1)); - Properties childControllerProperties = new Properties(); - childControllerProperties.setProperty(ConfigKeys.CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD, "true"); multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( NUMBER_OF_CHILD_DATACENTERS, NUMBER_OF_CLUSTERS, @@ -67,14 +65,11 @@ public void setUp() { 1, 1, Optional.empty(), - Optional.of(childControllerProperties), + Optional.empty(), Optional.of(serverProperties), false); childClusters = multiRegionMultiClusterWrapper.getChildRegions(); - childControllers = childClusters.stream() - .map(veniceClusterWrapper -> new ArrayList<>(veniceClusterWrapper.getControllers().values())) - .collect(Collectors.toList()); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); LOGGER.info( @@ -107,6 +102,7 @@ public void testGetClusterStaleStores() throws Exception { String inputDirPath = "file:" + inputDir.getAbsolutePath(); String storeName = Utils.getUniqueString("store"); String parentControllerUrls = multiRegionMultiClusterWrapper.getControllerConnectString(); + String dc0ControllerUrls = multiRegionMultiClusterWrapper.getChildRegions().get(0).getControllerConnectString(); // create a store via parent controller url Properties props = @@ -126,37 +122,38 @@ public void testGetClusterStaleStores() throws Exception { job.run(); } - try (ControllerClient controllerClient = new ControllerClient(clusterName, parentControllerUrls)) { + try (VenicePushJob job = new VenicePushJob("Test push job", props)) { + job.run(); + } + try (ControllerClient parentControllerClient = new ControllerClient(clusterName, parentControllerUrls); + ControllerClient dc0ControllerClient = new ControllerClient(clusterName, dc0ControllerUrls)) { // the store should not be appearing in the stale data audit ClusterStaleDataAuditResponse emptyResponse = - controllerClient.getClusterStaleStores(clusterName, parentControllerUrls); + parentControllerClient.getClusterStaleStores(clusterName, parentControllerUrls); Assert.assertFalse(emptyResponse.isError()); Assert.assertFalse(emptyResponse.getAuditMap().containsKey(storeName)); - // get single child controller, empty push to it - Properties props2 = IntegrationTestPushUtils - .defaultVPJProps(multiRegionMultiClusterWrapper.getChildRegions().get(0), inputDirPath, storeName); - try (VenicePushJob job = new VenicePushJob("Test push job", props2)) { - job.run(); - } + // get single child controller, rollback and delete a version. Revert the largest used version. + assertCommand(dc0ControllerClient.updateStore(storeName, new UpdateStoreQueryParams().setCurrentVersion(1))); + assertCommand(dc0ControllerClient.deleteOldVersion(storeName, 2)); + assertCommand( + dc0ControllerClient.updateStore(storeName, new UpdateStoreQueryParams().setLargestUsedVersionNumber(1))); // store should now appear as stale ClusterStaleDataAuditResponse response = - controllerClient.getClusterStaleStores(clusterName, parentControllerUrls); - Assert.assertFalse(response.isError()); - Assert.assertEquals(response.getAuditMap().get(storeName).getStaleRegions().size(), 1); - Assert.assertEquals(response.getAuditMap().get(storeName).getHealthyRegions().size(), 1); + assertCommand(parentControllerClient.getClusterStaleStores(clusterName, parentControllerUrls)); + Assert.assertTrue(response.getAuditMap().containsKey(storeName)); + StoreDataAudit auditForStore = response.getAuditMap().get(storeName); + Assert.assertEquals(auditForStore.getStaleRegions().size(), 1); + Assert.assertEquals(auditForStore.getHealthyRegions().size(), 1); // test store health check - StoreHealthAuditResponse healthResponse = controllerClient.listStorePushInfo(storeName, true); - Assert.assertTrue(response.getAuditMap().containsKey(healthResponse.getName())); - Map auditMapEntry = response.getAuditMap().get(healthResponse.getName()).getStaleRegions(); + StoreHealthAuditResponse healthResponse = parentControllerClient.listStorePushInfo(storeName, true); + Map auditMapEntry = auditForStore.getStaleRegions(); for (Map.Entry entry: auditMapEntry.entrySet()) { - if (Objects.equals(entry.getValue().getName(), storeName)) { - // verify that the same regions are stale across both responses for the same store. - Assert.assertTrue(healthResponse.getRegionsWithStaleData().contains(entry.getKey())); - } + // verify that the same regions are stale across both responses for the same store. + Assert.assertTrue(healthResponse.getRegionsWithStaleData().contains(entry.getKey())); } } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java index 66be7ae66d..d95f19c29d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java @@ -55,12 +55,12 @@ import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import java.io.File; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; +import java.util.List; import java.util.Optional; import java.util.Properties; -import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import org.apache.avro.Schema; @@ -330,7 +330,7 @@ public void testStoreMigrationWithDaVinciPushStatusSystemStore() throws Exceptio startMigration(parentControllerUrl, storeName); // Store migration status output via closure PrintFunction - Set statusOutput = new HashSet(); + List statusOutput = new ArrayList<>(); PrintFunction printFunction = (message) -> { statusOutput.add(message.trim()); System.err.println(message); @@ -362,18 +362,20 @@ public void testStoreMigrationWithDaVinciPushStatusSystemStore() throws Exceptio .assertEquals(pushStatusStoreReader.getPartitionStatus(storeName, 1, 0, Optional.empty()).size(), 1)); // Verify that store and system store only exist in destination cluster after ending migration - statusOutput.clear(); endMigration(parentControllerUrl, storeName); - checkMigrationStatus(parentControllerUrl, storeName, printFunction); - Assert - .assertFalse(statusOutput.contains(String.format("%s exists in this cluster %s", storeName, srcClusterName))); - Assert - .assertTrue(statusOutput.contains(String.format("%s exists in this cluster %s", storeName, destClusterName))); - Assert.assertFalse( - statusOutput.contains(String.format("%s exists in this cluster %s", systemStoreName, srcClusterName))); - Assert.assertTrue( - statusOutput.contains(String.format("%s exists in this cluster %s", systemStoreName, destClusterName))); + TestUtils.waitForNonDeterministicAssertion(30, TimeUnit.SECONDS, () -> { + statusOutput.clear(); + checkMigrationStatus(parentControllerUrl, storeName, printFunction); + Assert.assertFalse( + statusOutput.contains(String.format("%s exists in this cluster %s", storeName, srcClusterName))); + Assert.assertTrue( + statusOutput.contains(String.format("%s exists in this cluster %s", storeName, destClusterName))); + Assert.assertFalse( + statusOutput.contains(String.format("%s exists in this cluster %s", systemStoreName, srcClusterName))); + Assert.assertTrue( + statusOutput.contains(String.format("%s exists in this cluster %s", systemStoreName, destClusterName))); + }); } finally { Utils.closeQuietlyWithErrorLogged(pushStatusStoreReader); D2ClientUtils.shutdownClient(d2Client); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreUpdateStoragePersona.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreUpdateStoragePersona.java index 1a923386d6..e32074e62b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreUpdateStoragePersona.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreUpdateStoragePersona.java @@ -1,5 +1,6 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.venice.utils.TestUtils.assertCommand; import static com.linkedin.venice.utils.TestWriteUtils.STRING_SCHEMA; import com.linkedin.venice.controllerapi.ControllerClient; @@ -7,10 +8,8 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; -import com.linkedin.venice.integration.utils.VeniceControllerCreateOptions; -import com.linkedin.venice.integration.utils.VeniceControllerWrapper; -import com.linkedin.venice.integration.utils.ZkServerWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.persona.StoragePersona; import com.linkedin.venice.utils.TestStoragePersonaUtils; @@ -18,7 +17,6 @@ import com.linkedin.venice.utils.Utils; import java.util.HashSet; import java.util.Optional; -import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; import org.testng.Assert; @@ -29,27 +27,25 @@ public class TestStoreUpdateStoragePersona { private VeniceClusterWrapper venice; - private ZkServerWrapper parentZk; - private VeniceControllerWrapper parentController; private ControllerClient controllerClient; @BeforeClass(alwaysRun = true) public void setUp() { - Properties extraProperties = new Properties(); - venice = ServiceFactory.getVeniceCluster(1, 1, 1, 2, 1000000, false, false, extraProperties); - parentZk = ServiceFactory.getZkServer(); - parentController = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder(venice.getClusterName(), parentZk, venice.getPubSubBrokerWrapper()) - .childControllers(new VeniceControllerWrapper[] { venice.getLeaderVeniceController() }) + venice = ServiceFactory.getVeniceCluster( + new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) .build()); - controllerClient = new ControllerClient(venice.getClusterName(), parentController.getControllerUrl()); + controllerClient = new ControllerClient(venice.getClusterName(), venice.getAllControllersURLs()); } @AfterClass(alwaysRun = true) public void cleanUp() { Utils.closeQuietlyWithErrorLogged(controllerClient); - Utils.closeQuietlyWithErrorLogged(parentController); - Utils.closeQuietlyWithErrorLogged(parentZk); Utils.closeQuietlyWithErrorLogged(venice); } @@ -134,12 +130,13 @@ void testUpdatePersonaFailedAlreadyHasPersona() { Set expectedStores = new HashSet<>(); Store testStore = TestUtils.createTestStore(Utils.getUniqueString("testStore"), "testStoreOwner", 100); expectedStores.add(testStore.getName()); - controllerClient.createNewStoreWithParameters( - testStore.getName(), - testStore.getOwner(), - STRING_SCHEMA.toString(), - STRING_SCHEMA.toString(), - new UpdateStoreQueryParams().setStoragePersona(persona.getName()).setStorageQuotaInByte(quota)); + assertCommand( + controllerClient.createNewStoreWithParameters( + testStore.getName(), + testStore.getOwner(), + STRING_SCHEMA.toString(), + STRING_SCHEMA.toString(), + new UpdateStoreQueryParams().setStoragePersona(persona.getName()).setStorageQuotaInByte(quota))); ControllerResponse response = controllerClient .updateStore(testStore.getName(), new UpdateStoreQueryParams().setStoragePersona(persona2.getName())); Assert.assertTrue(response.isError()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java index 05b85a6a19..aeeaee7945 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java @@ -1,5 +1,7 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.venice.utils.TestUtils.assertCommand; + import com.linkedin.venice.controller.Admin; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.ControllerResponse; @@ -19,37 +21,81 @@ public class TestWritePathComputation { private static final long GET_LEADER_CONTROLLER_TIMEOUT = 20 * Time.MS_PER_SECOND; + private static final String KEY_SCHEMA_STR = "\"string\""; + private static final String VALUE_FIELD_NAME = "int_field"; + private static final String SECOND_VALUE_FIELD_NAME = "opt_int_field"; + private static final String VALUE_SCHEMA_V2_STR = "{\n" + "\"type\": \"record\",\n" + + "\"name\": \"TestValueSchema\",\n" + "\"namespace\": \"com.linkedin.venice.fastclient.schema\",\n" + + "\"fields\": [\n" + " {\"name\": \"" + VALUE_FIELD_NAME + "\", \"type\": \"int\", \"default\": 10},\n" + + "{\"name\": \"" + SECOND_VALUE_FIELD_NAME + "\", \"type\": [\"null\", \"int\"], \"default\": null}]\n" + "}"; @Test(timeOut = 60 * Time.MS_PER_SECOND) public void testFeatureFlagSingleDC() { - VeniceMultiClusterCreateOptions options = new VeniceMultiClusterCreateOptions.Builder(1).numberOfControllers(1) + VeniceMultiClusterCreateOptions options = new VeniceMultiClusterCreateOptions.Builder().setNumberOfClusters(1) + .numberOfControllers(1) .numberOfServers(1) .numberOfRouters(0) .regionName(VeniceClusterWrapperConstants.STANDALONE_REGION_NAME) .build(); try (VeniceMultiClusterWrapper multiClusterWrapper = ServiceFactory.getVeniceMultiClusterWrapper(options)) { String clusterName = multiClusterWrapper.getClusterNames()[0]; + VeniceControllerWrapper childController = multiClusterWrapper.getLeaderController(clusterName); String storeName = "test-store0"; + String storeName2 = "test-store2"; // Create store - Admin admin = + Admin childAdmin = multiClusterWrapper.getLeaderController(clusterName, GET_LEADER_CONTROLLER_TIMEOUT).getVeniceAdmin(); - admin.createStore(clusterName, storeName, "tester", "\"string\"", "\"string\""); - Assert.assertTrue(admin.hasStore(clusterName, storeName)); - Assert.assertFalse(admin.getStore(clusterName, storeName).isWriteComputationEnabled()); + childAdmin.createStore(clusterName, storeName, "tester", "\"string\"", "\"string\""); + childAdmin.createStore(clusterName, storeName2, "tester", KEY_SCHEMA_STR, VALUE_SCHEMA_V2_STR); + TestUtils.waitForNonDeterministicAssertion(15, TimeUnit.SECONDS, () -> { + Assert.assertTrue(childAdmin.hasStore(clusterName, storeName)); + Assert.assertFalse(childAdmin.getStore(clusterName, storeName).isWriteComputationEnabled()); + }); // Set flag - String controllerUrl = - multiClusterWrapper.getLeaderController(clusterName, GET_LEADER_CONTROLLER_TIMEOUT).getControllerUrl(); - try (ControllerClient controllerClient = new ControllerClient(clusterName, controllerUrl)) { - TestUtils.assertCommand( - controllerClient.updateStore(storeName, new UpdateStoreQueryParams().setWriteComputationEnabled(true)), - "Write Compute should be enabled"); - Assert.assertTrue(admin.getStore(clusterName, storeName).isWriteComputationEnabled()); + String childControllerUrl = childController.getControllerUrl(); + try (ControllerClient childControllerClient = new ControllerClient(clusterName, childControllerUrl)) { + ControllerResponse response = + childControllerClient.updateStore(storeName, new UpdateStoreQueryParams().setWriteComputationEnabled(true)); + Assert.assertTrue(response.isError()); + Assert.assertTrue(response.getError().contains("Write computation is only supported for hybrid stores")); + + ControllerResponse response2 = childControllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setHybridRewindSeconds(1000) + .setHybridOffsetLagThreshold(1000) + .setWriteComputationEnabled(true)); + Assert.assertTrue(response2.isError()); + Assert.assertTrue(response2.getError().contains("top level field probably missing defaults")); + + TestUtils.waitForNonDeterministicAssertion(15, TimeUnit.SECONDS, () -> { + Assert.assertFalse( + childAdmin.getStore(clusterName, storeName).isWriteComputationEnabled(), + "Write Compute should not be enabled before the value schema is not a Record."); + }); + + assertCommand( + childControllerClient.updateStore( + storeName2, + new UpdateStoreQueryParams().setHybridRewindSeconds(1000) + .setHybridOffsetLagThreshold(1000) + .setWriteComputationEnabled(true))); + TestUtils.waitForNonDeterministicAssertion(15, TimeUnit.SECONDS, () -> { + Assert.assertTrue(childAdmin.getStore(clusterName, storeName2).isWriteComputationEnabled()); + }); // Reset flag - controllerClient.updateStore(storeName, new UpdateStoreQueryParams().setWriteComputationEnabled(false)); - Assert.assertFalse(admin.getStore(clusterName, storeName).isWriteComputationEnabled()); + assertCommand( + childControllerClient + .updateStore(storeName, new UpdateStoreQueryParams().setWriteComputationEnabled(false))); + assertCommand( + childControllerClient + .updateStore(storeName2, new UpdateStoreQueryParams().setWriteComputationEnabled(false))); + TestUtils.waitForNonDeterministicAssertion(15, TimeUnit.SECONDS, () -> { + Assert.assertFalse(childAdmin.getStore(clusterName, storeName).isWriteComputationEnabled()); + Assert.assertFalse(childAdmin.getStore(clusterName, storeName2).isWriteComputationEnabled()); + }); } } } @@ -63,12 +109,14 @@ public void testFeatureFlagMultipleDC() { VeniceControllerWrapper parentController = twoLayerMultiRegionMultiClusterWrapper.getParentControllers().get(0); String clusterName = multiCluster.getClusterNames()[0]; String storeName = "test-store0"; + String storeName2 = "test-store2"; // Create store Admin parentAdmin = twoLayerMultiRegionMultiClusterWrapper.getLeaderParentControllerWithRetries(clusterName).getVeniceAdmin(); Admin childAdmin = multiCluster.getLeaderController(clusterName, GET_LEADER_CONTROLLER_TIMEOUT).getVeniceAdmin(); parentAdmin.createStore(clusterName, storeName, "tester", "\"string\"", "\"string\""); + parentAdmin.createStore(clusterName, storeName2, "tester", KEY_SCHEMA_STR, VALUE_SCHEMA_V2_STR); TestUtils.waitForNonDeterministicAssertion(15, TimeUnit.SECONDS, () -> { Assert.assertTrue(parentAdmin.hasStore(clusterName, storeName)); Assert.assertTrue(childAdmin.hasStore(clusterName, storeName)); @@ -82,7 +130,16 @@ public void testFeatureFlagMultipleDC() { ControllerResponse response = parentControllerClient .updateStore(storeName, new UpdateStoreQueryParams().setWriteComputationEnabled(true)); Assert.assertTrue(response.isError()); - Assert.assertTrue(response.getError().contains("top level field probably missing defaults")); + Assert.assertTrue(response.getError().contains("Write computation is only supported for hybrid stores")); + + ControllerResponse response2 = parentControllerClient.updateStore( + storeName, + new UpdateStoreQueryParams().setHybridRewindSeconds(1000) + .setHybridOffsetLagThreshold(1000) + .setWriteComputationEnabled(true)); + Assert.assertTrue(response2.isError()); + Assert.assertTrue(response2.getError().contains("top level field probably missing defaults")); + TestUtils.waitForNonDeterministicAssertion(15, TimeUnit.SECONDS, () -> { Assert.assertFalse( parentAdmin.getStore(clusterName, storeName).isWriteComputationEnabled(), @@ -92,13 +149,29 @@ public void testFeatureFlagMultipleDC() { "Write Compute should not be enabled before the value schema is not a Record."); }); + assertCommand( + parentControllerClient.updateStore( + storeName2, + new UpdateStoreQueryParams().setHybridRewindSeconds(1000) + .setHybridOffsetLagThreshold(1000) + .setWriteComputationEnabled(true))); + TestUtils.waitForNonDeterministicAssertion(15, TimeUnit.SECONDS, () -> { + Assert.assertTrue(parentAdmin.getStore(clusterName, storeName2).isWriteComputationEnabled()); + Assert.assertTrue(childAdmin.getStore(clusterName, storeName2).isWriteComputationEnabled()); + }); + // Reset flag - response = parentControllerClient - .updateStore(storeName, new UpdateStoreQueryParams().setWriteComputationEnabled(false)); - Assert.assertFalse(response.isError(), "No error is expected to disable Write Compute (that was not enabled)"); + assertCommand( + parentControllerClient + .updateStore(storeName, new UpdateStoreQueryParams().setWriteComputationEnabled(false))); + assertCommand( + parentControllerClient + .updateStore(storeName2, new UpdateStoreQueryParams().setWriteComputationEnabled(false))); TestUtils.waitForNonDeterministicAssertion(15, TimeUnit.SECONDS, () -> { Assert.assertFalse(parentAdmin.getStore(clusterName, storeName).isWriteComputationEnabled()); Assert.assertFalse(childAdmin.getStore(clusterName, storeName).isWriteComputationEnabled()); + Assert.assertFalse(parentAdmin.getStore(clusterName, storeName2).isWriteComputationEnabled()); + Assert.assertFalse(childAdmin.getStore(clusterName, storeName2).isWriteComputationEnabled()); }); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientIndividualFeatureConfigurationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientIndividualFeatureConfigurationTest.java index f123db237e..ec1dc2ace7 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientIndividualFeatureConfigurationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientIndividualFeatureConfigurationTest.java @@ -1,5 +1,6 @@ package com.linkedin.venice.fastclient; +import static com.linkedin.venice.utils.TestUtils.assertCommand; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -59,7 +60,7 @@ public class FastClientIndividualFeatureConfigurationTest extends AbstractClient @BeforeMethod public void enableStoreReadQuota() { veniceCluster.useControllerClient(controllerClient -> { - TestUtils.assertCommand( + assertCommand( controllerClient.updateStore(storeName, new UpdateStoreQueryParams().setStorageNodeReadQuotaEnabled(true))); }); } @@ -76,7 +77,7 @@ public void testServerReadQuota() throws Exception { StoreMetadataFetchMode.SERVER_BASED_METADATA); // Update the read quota to 1000 and make 500 requests, all requests should be allowed. veniceCluster.useControllerClient(controllerClient -> { - TestUtils.assertCommand( + assertCommand( controllerClient.updateStore( storeName, new UpdateStoreQueryParams().setReadQuotaInCU(1000).setStorageNodeReadQuotaEnabled(true))); @@ -153,8 +154,7 @@ public void testServerReadQuota() throws Exception { // Update the read quota to 50 and make as many requests needed to trigger quota rejected exception. veniceCluster.useControllerClient(controllerClient -> { - TestUtils - .assertCommand(controllerClient.updateStore(storeName, new UpdateStoreQueryParams().setReadQuotaInCU(50))); + assertCommand(controllerClient.updateStore(storeName, new UpdateStoreQueryParams().setReadQuotaInCU(50))); }); try { // Keep making requests until it gets rejected by read quota, it may take some time for the quota update to be @@ -227,7 +227,7 @@ public void testServerRejectReadComputeRequest() throws Exception { assertTrue(responseMap.isFullResponse()); veniceCluster.useControllerClient(controllerClient -> { - TestUtils.assertCommand( + assertCommand( controllerClient.updateStore(storeName, new UpdateStoreQueryParams().setReadComputationEnabled(false))); }); @@ -259,7 +259,9 @@ public void testServerRejectReadComputeRequest() throws Exception { @Test(timeOut = TIME_OUT) public void testStreamingBatchGetWithMultiValueSchemaVersions() throws Exception { - veniceCluster.useControllerClient(client -> client.addValueSchema(storeName, VALUE_SCHEMA_V2_STR)); + veniceCluster.useControllerClient(client -> { + assertCommand(client.addValueSchema(storeName, VALUE_SCHEMA_V2_STR)); + }); ClientConfig.ClientConfigBuilder clientConfigBuilder = new ClientConfig.ClientConfigBuilder<>().setStoreName(storeName) .setR2Client(r2Client) @@ -317,8 +319,7 @@ public void testStreamingBatchGetWithMultiValueSchemaVersions() throws Exception @Test(timeOut = TIME_OUT) public void testStreamingBatchGetWithRetryAndQuotaRejection() throws Exception { veniceCluster.useControllerClient(controllerClient -> { - TestUtils - .assertCommand(controllerClient.updateStore(storeName, new UpdateStoreQueryParams().setReadQuotaInCU(10))); + assertCommand(controllerClient.updateStore(storeName, new UpdateStoreQueryParams().setReadQuotaInCU(10))); }); ArrayList serverMetrics = new ArrayList<>(); for (int i = 0; i < veniceCluster.getVeniceServers().size(); i++) { @@ -450,7 +451,7 @@ public void testSNQuotaNotEnabled() { .setSpeculativeQueryEnabled(false); // Update store to disable storage node read quota veniceCluster.useControllerClient(controllerClient -> { - TestUtils.assertCommand( + assertCommand( controllerClient.updateStore(storeName, new UpdateStoreQueryParams().setStorageNodeReadQuotaEnabled(false))); }); Assert.assertThrows( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java index 26d8d6cf37..4c63a42851 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/utils/AbstractClientEndToEndSetup.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertTrue; import com.google.common.collect.ImmutableList; +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import com.linkedin.d2.balancer.D2Client; import com.linkedin.davinci.client.DaVinciClient; import com.linkedin.davinci.client.DaVinciConfig; @@ -117,12 +118,12 @@ public abstract class AbstractClientEndToEndSetup { protected static final String VALUE_SCHEMA_STR = "{\n" + "\"type\": \"record\",\n" + "\"name\": \"TestValueSchema\",\n" + "\"namespace\": \"com.linkedin.venice.fastclient.schema\",\n" + "\"fields\": [\n" + " {\"name\": \"" + VALUE_FIELD_NAME + "\", \"type\": \"int\"}]\n" + "}"; - protected static final Schema VALUE_SCHEMA = new Schema.Parser().parse(VALUE_SCHEMA_STR); + protected static final Schema VALUE_SCHEMA = AvroCompatibilityHelper.parse(VALUE_SCHEMA_STR); protected static final String VALUE_SCHEMA_V2_STR = "{\n" + "\"type\": \"record\",\n" + "\"name\": \"TestValueSchema\",\n" + "\"namespace\": \"com.linkedin.venice.fastclient.schema\",\n" + "\"fields\": [\n" + " {\"name\": \"" + VALUE_FIELD_NAME + "\", \"type\": \"int\"},\n" + "{\"name\": \"" + SECOND_VALUE_FIELD_NAME + "\", \"type\": [\"null\", \"int\"], \"default\": null}]\n" + "}"; - protected static final Schema VALUE_SCHEMA_V2 = new Schema.Parser().parse(VALUE_SCHEMA_V2_STR); + protected static final Schema VALUE_SCHEMA_V2 = AvroCompatibilityHelper.parse(VALUE_SCHEMA_V2_STR); protected static final String keyPrefix = "key_"; protected static final int recordCnt = 100; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/TestVenicePushJob.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/TestVenicePushJob.java index 2d1366b5e1..0ce677f5ba 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/TestVenicePushJob.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/hadoop/TestVenicePushJob.java @@ -21,6 +21,7 @@ import static com.linkedin.venice.utils.IntegrationTestPushUtils.createStoreForJob; import static com.linkedin.venice.utils.IntegrationTestPushUtils.defaultVPJProps; import static com.linkedin.venice.utils.TestWriteUtils.getTempDataDirectory; +import static com.linkedin.venice.utils.TestWriteUtils.loadFileAsString; import static com.linkedin.venice.utils.TestWriteUtils.writeSimpleAvroFileWithStringToStringSchema; import static com.linkedin.venice.utils.TestWriteUtils.writeSimpleAvroFileWithStringToStringSchema2; import static com.linkedin.venice.utils.TestWriteUtils.writeSimpleVsonFileWithUserSchema; @@ -39,6 +40,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; +import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.Version; import com.linkedin.venice.utils.DataProviderUtils; @@ -442,8 +444,10 @@ public void testWCJobWithStoreNotWCEnabled() throws Exception { UpdateStoreQueryParams params = new UpdateStoreQueryParams(); // disable WriteCompute in store - params.setWriteComputationEnabled(false); - params.setIncrementalPushEnabled(true); + params.setWriteComputationEnabled(false) + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) + .setHybridRewindSeconds(Time.SECONDS_PER_DAY) + .setHybridOffsetLagThreshold(1000); controllerClient.createNewStoreWithParameters(storeName, "owner", "\"string\"", "\"string\"", params); @@ -469,10 +473,14 @@ public void testWCBatchJob() throws Exception { ControllerClient controllerClient = new ControllerClient(veniceCluster.getClusterName(), routerUrl); UpdateStoreQueryParams params = new UpdateStoreQueryParams(); - params.setWriteComputationEnabled(true); - params.setIncrementalPushEnabled(false); + params.setWriteComputationEnabled(true) + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) + .setHybridRewindSeconds(Time.SECONDS_PER_DAY) + .setHybridOffsetLagThreshold(1000); - controllerClient.createNewStoreWithParameters(storeName, "owner", "\"string\"", "\"string\"", params); + String valueSchemaStr = loadFileAsString("UserValue.avsc"); + + controllerClient.createNewStoreWithParameters(storeName, "owner", "\"string\"", valueSchemaStr, params); String inputDirPath = "file://" + inputDir.getAbsolutePath(); Properties props = defaultVPJProps(veniceCluster, inputDirPath, storeName); @@ -560,7 +568,9 @@ public void testKIFRepushForIncrementalPushStores() throws Exception { storeName, new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) .setPartitionCount(2) - .setIncrementalPushEnabled(true))); + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) + .setHybridRewindSeconds(Time.SECONDS_PER_DAY) + .setHybridOffsetLagThreshold(1000))); Properties props = defaultVPJProps(veniceCluster, inputDirPath, storeName); // create a batch version. @@ -629,8 +639,9 @@ public void testKIFRepushFetch(boolean chunkingEnabled) throws Exception { storeName, new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) .setPartitionCount(2) - .setIncrementalPushEnabled(true) - .setWriteComputationEnabled(true))); + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) + .setHybridRewindSeconds(Time.SECONDS_PER_DAY) + .setHybridOffsetLagThreshold(1000))); Properties props = defaultVPJProps(veniceCluster, inputDirPath, storeName); props.setProperty(SEND_CONTROL_MESSAGES_DIRECTLY, "true"); // create a batch version. diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java index 127666ac74..4944ffa17e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java @@ -155,7 +155,7 @@ public void testIngestionHeartBeat( UpdateStoreQueryParams updateStoreParams = new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) .setCompressionStrategy(CompressionStrategy.NO_OP) - .setIncrementalPushEnabled(isIncrementalPushEnabled) + .setHybridDataReplicationPolicy(DataReplicationPolicy.NONE) .setHybridRewindSeconds(500L) .setHybridOffsetLagThreshold(10L) .setPartitionCount(2) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java index 9292aabf32..ce00d71701 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java @@ -4,7 +4,6 @@ import static com.linkedin.venice.ConfigKeys.D2_ZK_HOSTS_ADDRESS; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_MAX_ATTEMPT; -import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_REPLICATION_FACTOR; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_WAIT_TIME_FOR_CLUSTER_START_S; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.STANDALONE_REGION_NAME; @@ -408,18 +407,14 @@ public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMult int numberOfControllers, int numberOfServers, int numberOfRouters) { - return getService( - VeniceTwoLayerMultiRegionMultiClusterWrapper.SERVICE_NAME, - VeniceTwoLayerMultiRegionMultiClusterWrapper.generateService( - numberOfRegions, - numberOfClustersInEachRegion, - numberOfParentControllers, - numberOfControllers, - numberOfServers, - numberOfRouters, - DEFAULT_REPLICATION_FACTOR, - Optional.empty(), - Optional.empty())); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(numberOfRegions) + .numberOfClusters(numberOfClustersInEachRegion) + .numberOfParentControllers(numberOfParentControllers) + .numberOfChildControllers(numberOfControllers) + .numberOfServers(numberOfServers) + .numberOfRouters(numberOfRouters); + return getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); } public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( @@ -433,20 +428,19 @@ public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMult Optional parentControllerProps, Optional childControllerProperties, Optional serverProps) { - return getService( - VeniceTwoLayerMultiRegionMultiClusterWrapper.SERVICE_NAME, - VeniceTwoLayerMultiRegionMultiClusterWrapper.generateService( - numberOfRegions, - numberOfClustersInEachRegion, - numberOfParentControllers, - numberOfControllers, - numberOfServers, - numberOfRouters, - replicationFactor, - parentControllerProps, - childControllerProperties, - serverProps, - false)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(numberOfRegions) + .numberOfClusters(numberOfClustersInEachRegion) + .numberOfParentControllers(numberOfParentControllers) + .numberOfChildControllers(numberOfControllers) + .numberOfServers(numberOfServers) + .numberOfRouters(numberOfRouters) + .replicationFactor(replicationFactor); + + parentControllerProps.ifPresent(optionsBuilder::parentControllerProperties); + childControllerProperties.ifPresent(optionsBuilder::childControllerProperties); + serverProps.ifPresent(optionsBuilder::serverProperties); + return getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); } public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( @@ -461,20 +455,28 @@ public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMult Optional childControllerProperties, Optional serverProps, boolean forkServer) { + + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(numberOfRegions) + .numberOfClusters(numberOfClustersInEachRegion) + .numberOfParentControllers(numberOfParentControllers) + .numberOfChildControllers(numberOfControllers) + .numberOfServers(numberOfServers) + .numberOfRouters(numberOfRouters) + .replicationFactor(replicationFactor) + .forkServer(forkServer); + + parentControllerProps.ifPresent(optionsBuilder::parentControllerProperties); + childControllerProperties.ifPresent(optionsBuilder::childControllerProperties); + serverProps.ifPresent(optionsBuilder::serverProperties); + return getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); + } + + public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( + VeniceMultiRegionClusterCreateOptions options) { return getService( VeniceTwoLayerMultiRegionMultiClusterWrapper.SERVICE_NAME, - VeniceTwoLayerMultiRegionMultiClusterWrapper.generateService( - numberOfRegions, - numberOfClustersInEachRegion, - numberOfParentControllers, - numberOfControllers, - numberOfServers, - numberOfRouters, - replicationFactor, - parentControllerProps, - childControllerProperties, - serverProps, - forkServer)); + VeniceTwoLayerMultiRegionMultiClusterWrapper.generateService(options)); } public static HelixAsAServiceWrapper getHelixController(String zkAddress) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterCreateOptions.java index aa019c3d18..244bf046e8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterCreateOptions.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterCreateOptions.java @@ -20,6 +20,7 @@ public class VeniceClusterCreateOptions { private final String clusterName; + private final boolean multiRegion; private final String regionName; private final Map clusterToD2; private final Map clusterToServerD2; @@ -46,6 +47,7 @@ public class VeniceClusterCreateOptions { private VeniceClusterCreateOptions(Builder builder) { this.clusterName = builder.clusterName; + this.multiRegion = builder.multiRegion; this.regionName = builder.regionName; this.clusterToD2 = builder.clusterToD2; this.clusterToServerD2 = builder.clusterToServerD2; @@ -75,6 +77,10 @@ public String getClusterName() { return clusterName; } + public boolean isMultiRegion() { + return multiRegion; + } + public String getRegionName() { return regionName; } @@ -246,6 +252,7 @@ public String toString() { public static class Builder { private String clusterName; + private boolean multiRegion; private String regionName; private Map clusterToD2 = null; private Map clusterToServerD2 = null; @@ -276,6 +283,11 @@ public Builder clusterName(String clusterName) { return this; } + public Builder multiRegion(boolean multiRegion) { + this.multiRegion = multiRegion; + return this; + } + public Builder regionName(String regionName) { this.regionName = regionName; return this; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java index 230ab1a136..67a270d6cb 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java @@ -220,6 +220,7 @@ static ServiceProvider generateService(VeniceClusterCreate VeniceControllerWrapper veniceControllerWrapper = ServiceFactory.getVeniceController( new VeniceControllerCreateOptions.Builder(options.getClusterName(), zkServerWrapper, pubSubBrokerWrapper) + .multiRegion(options.isMultiRegion()) .replicationFactor(options.getReplicationFactor()) .partitionSize(options.getPartitionSize()) .numberOfPartitions(options.getNumberOfPartitions()) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerCreateOptions.java index 2de94622c0..fbb1584b07 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerCreateOptions.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerCreateOptions.java @@ -22,6 +22,7 @@ public class VeniceControllerCreateOptions { private final boolean isParent; private final boolean sslToKafka; private final boolean d2Enabled; + private final boolean multiRegion; private final int replicationFactor; private final int partitionSize; private final int numberOfPartitions; @@ -41,6 +42,7 @@ public class VeniceControllerCreateOptions { private VeniceControllerCreateOptions(Builder builder) { sslToKafka = builder.sslToKafka; d2Enabled = builder.d2Enabled; + multiRegion = builder.multiRegion; replicationFactor = builder.replicationFactor; partitionSize = builder.partitionSize; numberOfPartitions = builder.numberOfPartitions; @@ -136,6 +138,10 @@ public boolean isD2Enabled() { return d2Enabled; } + public boolean isMultiRegion() { + return multiRegion; + } + public int getReplicationFactor() { return replicationFactor; } @@ -202,6 +208,7 @@ public static class Builder { private final PubSubBrokerWrapper kafkaBroker; private boolean sslToKafka = false; private boolean d2Enabled = false; + private boolean multiRegion = false; private boolean isMinActiveReplicaSet = false; private int replicationFactor = DEFAULT_REPLICATION_FACTOR; private int partitionSize = DEFAULT_PARTITION_SIZE_BYTES; @@ -237,6 +244,11 @@ public Builder d2Enabled(boolean d2Enabled) { return this; } + public Builder multiRegion(boolean multiRegion) { + this.multiRegion = multiRegion; + return this; + } + public Builder replicationFactor(int replicationFactor) { this.replicationFactor = replicationFactor; return this; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerWrapper.java index 89aca58448..3451be3b2d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceControllerWrapper.java @@ -5,10 +5,10 @@ import static com.linkedin.venice.ConfigKeys.ADMIN_SECURE_PORT; import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_REPLICATION_FACTOR; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_ALLOWLIST; +import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_ALLOWLIST_LEGACY; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_D2_PREFIX; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_D2_SERVICE_NAME; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_URL_PREFIX; -import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_WHITELIST; import static com.linkedin.venice.ConfigKeys.CHILD_DATA_CENTER_KAFKA_URL_PREFIX; import static com.linkedin.venice.ConfigKeys.CLUSTER_DISCOVERY_D2_SERVICE; import static com.linkedin.venice.ConfigKeys.CLUSTER_TO_D2; @@ -35,6 +35,7 @@ import static com.linkedin.venice.ConfigKeys.KAFKA_SECURITY_PROTOCOL; import static com.linkedin.venice.ConfigKeys.LOCAL_REGION_NAME; import static com.linkedin.venice.ConfigKeys.MIN_ACTIVE_REPLICA; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_FABRIC_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_SOURCE_FABRIC; import static com.linkedin.venice.ConfigKeys.OFFLINE_JOB_START_TIMEOUT_MS; @@ -173,6 +174,7 @@ static StatefulServiceProvider generateService(VeniceCo // TODO: Validate that these configs are all still used. // TODO: Centralize default config values in a single place PropertyBuilder builder = new PropertyBuilder().put(clusterProps.toProperties()) + .put(MULTI_REGION, options.isMultiRegion()) .put(KAFKA_REPLICATION_FACTOR, 1) .put(ADMIN_TOPIC_REPLICATION_FACTOR, 1) .put(CONTROLLER_NAME, "venice-controller") // Why is this configurable? @@ -237,8 +239,8 @@ static StatefulServiceProvider generateService(VeniceCo // Parent controller needs config to route per-cluster requests such as job status // This dummy parent controller won't support such requests until we make this config configurable. // go/inclusivecode deferred(Reference will be removed when clients have migrated) - fabricAllowList = - extraProps.getStringWithAlternative(CHILD_CLUSTER_ALLOWLIST, CHILD_CLUSTER_WHITELIST, StringUtils.EMPTY); + fabricAllowList = extraProps + .getStringWithAlternative(CHILD_CLUSTER_ALLOWLIST, CHILD_CLUSTER_ALLOWLIST_LEGACY, StringUtils.EMPTY); } else { // Use A/A fabric list for fabric allow list in case this controller is used in a multi-region test setup String fabricList = options.getExtraProperties().getProperty(ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, ""); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java index 47a3e0ad4e..362ab41633 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java @@ -6,6 +6,7 @@ import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_NUMBER_OF_SERVERS; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_PARTITION_SIZE_BYTES; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_REPLICATION_FACTOR; +import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_SSL_TO_KAFKA; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_SSL_TO_STORAGE_NODES; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.STANDALONE_REGION_NAME; @@ -15,6 +16,7 @@ public class VeniceMultiClusterCreateOptions { + private final boolean multiRegion; private final String regionName; private final int numberOfClusters; private final int numberOfControllers; @@ -27,8 +29,8 @@ public class VeniceMultiClusterCreateOptions { private final boolean enableAllowlist; private final boolean enableAutoJoinAllowlist; private final boolean sslToStorageNodes; + private final boolean sslToKafka; private final boolean randomizeClusterName; - private final boolean multiRegionSetup; private final boolean forkServer; private final Map> kafkaClusterMap; private final ZkServerWrapper zkServerWrapper; @@ -84,12 +86,16 @@ public boolean isSslToStorageNodes() { return sslToStorageNodes; } + public boolean isSslToKafka() { + return sslToKafka; + } + public boolean isRandomizeClusterName() { return randomizeClusterName; } - public boolean isMultiRegionSetup() { - return multiRegionSetup; + public boolean isMultiRegion() { + return multiRegion; } public boolean isForkServer() { @@ -155,11 +161,14 @@ public String toString() { .append("sslToStorageNodes:") .append(sslToStorageNodes) .append(", ") + .append("sslToKafka:") + .append(sslToKafka) + .append(", ") .append("forkServer:") .append(forkServer) .append(", ") - .append("multiRegionSetup:") - .append(multiRegionSetup) + .append("multiRegion:") + .append(multiRegion) .append(", ") .append("randomizeClusterName:") .append(randomizeClusterName) @@ -195,8 +204,9 @@ private VeniceMultiClusterCreateOptions(Builder builder) { rebalanceDelayMs = builder.rebalanceDelayMs; minActiveReplica = builder.minActiveReplica; sslToStorageNodes = builder.sslToStorageNodes; + sslToKafka = builder.sslToKafka; randomizeClusterName = builder.randomizeClusterName; - multiRegionSetup = builder.multiRegionSetup; + multiRegion = builder.multiRegion; zkServerWrapper = builder.zkServerWrapper; pubSubBrokerWrapper = builder.pubSubBrokerWrapper; childControllerProperties = builder.childControllerProperties; @@ -207,7 +217,7 @@ private VeniceMultiClusterCreateOptions(Builder builder) { public static class Builder { private String regionName; - private final int numberOfClusters; + private int numberOfClusters; private int numberOfControllers = DEFAULT_NUMBER_OF_CONTROLLERS; private int numberOfServers = DEFAULT_NUMBER_OF_SERVERS; private int numberOfRouters = DEFAULT_NUMBER_OF_ROUTERS; @@ -218,8 +228,9 @@ public static class Builder { private boolean enableAllowlist = false; private boolean enableAutoJoinAllowlist = false; private boolean sslToStorageNodes = DEFAULT_SSL_TO_STORAGE_NODES; + private boolean sslToKafka = DEFAULT_SSL_TO_KAFKA; private boolean randomizeClusterName = true; - private boolean multiRegionSetup = false; + private boolean multiRegion = false; private boolean forkServer = false; private boolean isMinActiveReplicaSet = false; private Map> kafkaClusterMap; @@ -228,8 +239,9 @@ public static class Builder { private Properties childControllerProperties; private Properties extraProperties; - public Builder(int numberOfClusters) { + public Builder setNumberOfClusters(int numberOfClusters) { this.numberOfClusters = numberOfClusters; + return this; } public Builder regionName(String regionName) { @@ -268,23 +280,13 @@ public Builder minActiveReplica(int minActiveReplica) { return this; } - public Builder rebalanceDelayMs(long rebalanceDelayMs) { - this.rebalanceDelayMs = rebalanceDelayMs; - return this; - } - - public Builder enableAllowlist(boolean enableAllowlist) { - this.enableAllowlist = enableAllowlist; - return this; - } - - public Builder enableAutoJoinAllowlist(boolean enableAutoJoinAllowlist) { - this.enableAutoJoinAllowlist = enableAutoJoinAllowlist; + public Builder sslToStorageNodes(boolean sslToStorageNodes) { + this.sslToStorageNodes = sslToStorageNodes; return this; } - public Builder sslToStorageNodes(boolean sslToStorageNodes) { - this.sslToStorageNodes = sslToStorageNodes; + public Builder sslToKafka(boolean sslToKafka) { + this.sslToKafka = sslToKafka; return this; } @@ -293,8 +295,8 @@ public Builder randomizeClusterName(boolean randomizeClusterName) { return this; } - public Builder multiRegionSetup(boolean multiRegionSetup) { - this.multiRegionSetup = multiRegionSetup; + public Builder multiRegion(boolean multiRegion) { + this.multiRegion = multiRegion; return this; } @@ -329,6 +331,9 @@ public Builder extraProperties(Properties extraProperties) { } private void addDefaults() { + if (numberOfClusters == 0) { + numberOfClusters = 1; + } if (!isMinActiveReplicaSet) { minActiveReplica = replicationFactor - 1; } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java index 6f16f561a4..dade190e18 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java @@ -1,6 +1,5 @@ package com.linkedin.venice.integration.utils; -import static com.linkedin.venice.ConfigKeys.CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD; import static com.linkedin.venice.ConfigKeys.LOCAL_REGION_NAME; import static com.linkedin.venice.ConfigKeys.SYSTEM_SCHEMA_CLUSTER_NAME; @@ -87,11 +86,6 @@ static ServiceProvider generateService(VeniceMultiClu // Create controllers for multi-cluster Properties controllerProperties = options.getChildControllerProperties(); - if (options.isMultiRegionSetup() - && !controllerProperties.containsKey(CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD)) { - // In multi-region setup, we don't allow batch push to each individual child region, but just parent region - controllerProperties.put(CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD, "false"); - } if (options.getRegionName() != null) { controllerProperties.setProperty(LOCAL_REGION_NAME, options.getRegionName()); } @@ -112,6 +106,7 @@ static ServiceProvider generateService(VeniceMultiClu pubBrokerDetails.forEach((key, value) -> controllerProperties.putIfAbsent(key, value)); VeniceControllerCreateOptions controllerCreateOptions = new VeniceControllerCreateOptions.Builder(clusterNames, zkServerWrapper, pubSubBrokerWrapper) + .multiRegion(options.isMultiRegion()) .regionName(options.getRegionName()) .replicationFactor(options.getReplicationFactor()) .partitionSize(options.getPartitionSize()) @@ -133,7 +128,8 @@ static ServiceProvider generateService(VeniceMultiClu extraProperties.putAll(KafkaTestUtils.getLocalCommonKafkaSSLConfig(SslUtils.getTlsConfiguration())); pubBrokerDetails.forEach((key, value) -> extraProperties.putIfAbsent(key, value)); VeniceClusterCreateOptions.Builder vccBuilder = - new VeniceClusterCreateOptions.Builder().regionName(options.getRegionName()) + new VeniceClusterCreateOptions.Builder().multiRegion(options.isMultiRegion()) + .regionName(options.getRegionName()) .standalone(false) .zkServerWrapper(zkServerWrapper) .kafkaBrokerWrapper(pubSubBrokerWrapper) @@ -148,6 +144,7 @@ static ServiceProvider generateService(VeniceMultiClu .enableAutoJoinAllowlist(options.isEnableAutoJoinAllowlist()) .rebalanceDelayMs(options.getRebalanceDelayMs()) .minActiveReplica(options.getMinActiveReplica()) + .sslToKafka(options.isSslToKafka()) .sslToStorageNodes(options.isSslToStorageNodes()) .extraProperties(extraProperties) .forkServer(options.isForkServer()) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java new file mode 100644 index 0000000000..ae8c7872f4 --- /dev/null +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java @@ -0,0 +1,256 @@ +package com.linkedin.venice.integration.utils; + +import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_NUMBER_OF_CONTROLLERS; +import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_NUMBER_OF_ROUTERS; +import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_NUMBER_OF_SERVERS; +import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_REPLICATION_FACTOR; +import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_SSL_TO_KAFKA; +import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_SSL_TO_STORAGE_NODES; + +import com.linkedin.venice.authorization.AuthorizerService; +import java.util.Properties; + + +public class VeniceMultiRegionClusterCreateOptions { + private final int numberOfRegions; + private final int numberOfClusters; + private final int numberOfParentControllers; + private final int numberOfChildControllers; + private final int numberOfServers; + private final int numberOfRouters; + private final int replicationFactor; + private final boolean sslToStorageNodes; + private final boolean sslToKafka; + private final boolean forkServer; + private final Properties parentControllerProperties; + private final Properties childControllerProperties; + private final Properties serverProperties; + private final AuthorizerService parentAuthorizerService; + + public int getNumberOfRegions() { + return numberOfRegions; + } + + public int getNumberOfClusters() { + return numberOfClusters; + } + + public int getNumberOfParentControllers() { + return numberOfParentControllers; + } + + public int getNumberOfChildControllers() { + return numberOfChildControllers; + } + + public int getNumberOfServers() { + return numberOfServers; + } + + public int getNumberOfRouters() { + return numberOfRouters; + } + + public int getReplicationFactor() { + return replicationFactor; + } + + public boolean isSslToStorageNodes() { + return sslToStorageNodes; + } + + public boolean isSslToKafka() { + return sslToKafka; + } + + public boolean isForkServer() { + return forkServer; + } + + public Properties getParentControllerProperties() { + return parentControllerProperties; + } + + public Properties getChildControllerProperties() { + return childControllerProperties; + } + + public Properties getServerProperties() { + return serverProperties; + } + + public AuthorizerService getParentAuthorizerService() { + return parentAuthorizerService; + } + + @Override + public String toString() { + return new StringBuilder().append("VeniceMultiClusterCreateOptions - ") + .append("numberOfRegions:") + .append(numberOfRegions) + .append(", ") + .append("clusters:") + .append(numberOfClusters) + .append(", ") + .append("parent controllers:") + .append(numberOfParentControllers) + .append(", ") + .append("child controllers:") + .append(numberOfChildControllers) + .append(", ") + .append("servers:") + .append(numberOfServers) + .append(", ") + .append("routers:") + .append(numberOfRouters) + .append(", ") + .append("replicationFactor:") + .append(replicationFactor) + .append(", ") + .append("sslToStorageNodes:") + .append(sslToStorageNodes) + .append(", ") + .append("sslToKafka:") + .append(sslToKafka) + .append(", ") + .append("forkServer:") + .append(forkServer) + .append(", ") + .append("childControllerProperties:") + .append(childControllerProperties) + .append(", ") + .append("parentControllerProperties:") + .append(parentControllerProperties) + .append(", ") + .append("serverProperties:") + .append(serverProperties) + .append(", ") + .append("parentAuthorizerService:") + .append(parentAuthorizerService) + .toString(); + } + + private VeniceMultiRegionClusterCreateOptions(Builder builder) { + numberOfRegions = builder.numberOfRegions; + numberOfClusters = builder.numberOfClusters; + numberOfParentControllers = builder.numberOfParentControllers; + numberOfChildControllers = builder.numberOfChildControllers; + numberOfServers = builder.numberOfServers; + numberOfRouters = builder.numberOfRouters; + replicationFactor = builder.replicationFactor; + parentControllerProperties = builder.parentControllerProperties; + childControllerProperties = builder.childControllerProperties; + serverProperties = builder.serverProperties; + sslToStorageNodes = builder.sslToStorageNodes; + sslToKafka = builder.sslToKafka; + forkServer = builder.forkServer; + parentAuthorizerService = builder.parentAuthorizerService; + } + + public static class Builder { + private int numberOfRegions; + private int numberOfClusters; + private int numberOfParentControllers = DEFAULT_NUMBER_OF_CONTROLLERS; + private int numberOfChildControllers = DEFAULT_NUMBER_OF_CONTROLLERS; + private int numberOfServers = DEFAULT_NUMBER_OF_SERVERS; + private int numberOfRouters = DEFAULT_NUMBER_OF_ROUTERS; + private int replicationFactor = DEFAULT_REPLICATION_FACTOR; + private boolean sslToStorageNodes = DEFAULT_SSL_TO_STORAGE_NODES; + private boolean sslToKafka = DEFAULT_SSL_TO_KAFKA; + private boolean forkServer = false; + private Properties parentControllerProperties; + private Properties childControllerProperties; + private Properties serverProperties; + private AuthorizerService parentAuthorizerService; + + public Builder numberOfRegions(int numberOfRegions) { + this.numberOfRegions = numberOfRegions; + return this; + } + + public Builder numberOfClusters(int numberOfClusters) { + this.numberOfClusters = numberOfClusters; + return this; + } + + public Builder numberOfParentControllers(int numberOfParentControllers) { + this.numberOfParentControllers = numberOfParentControllers; + return this; + } + + public Builder numberOfChildControllers(int numberOfChildControllers) { + this.numberOfChildControllers = numberOfChildControllers; + return this; + } + + public Builder numberOfServers(int numberOfServers) { + this.numberOfServers = numberOfServers; + return this; + } + + public Builder numberOfRouters(int numberOfRouters) { + this.numberOfRouters = numberOfRouters; + return this; + } + + public Builder replicationFactor(int replicationFactor) { + this.replicationFactor = replicationFactor; + return this; + } + + public Builder sslToStorageNodes(boolean sslToStorageNodes) { + this.sslToStorageNodes = sslToStorageNodes; + return this; + } + + public Builder sslToKafka(boolean sslToKafka) { + this.sslToKafka = sslToKafka; + return this; + } + + public Builder forkServer(boolean forkServer) { + this.forkServer = forkServer; + return this; + } + + public Builder parentControllerProperties(Properties parentControllerProperties) { + this.parentControllerProperties = parentControllerProperties; + return this; + } + + public Builder childControllerProperties(Properties childControllerProperties) { + this.childControllerProperties = childControllerProperties; + return this; + } + + public Builder serverProperties(Properties serverProperties) { + this.serverProperties = serverProperties; + return this; + } + + public Builder parentAuthorizerService(AuthorizerService parentAuthorizerService) { + this.parentAuthorizerService = parentAuthorizerService; + return this; + } + + private void addDefaults() { + if (numberOfRegions == 0) { + numberOfRegions = 1; + } + if (numberOfClusters == 0) { + numberOfClusters = 1; + } + if (parentControllerProperties == null) { + parentControllerProperties = new Properties(); + } + if (childControllerProperties == null) { + childControllerProperties = new Properties(); + } + } + + public VeniceMultiRegionClusterCreateOptions build() { + addDefaults(); + return new VeniceMultiRegionClusterCreateOptions(this); + } + } +} diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java index a81845078e..75099a395e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java @@ -1,14 +1,9 @@ package com.linkedin.venice.integration.utils; import static com.linkedin.venice.ConfigKeys.ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST; -import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED; import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_SOURCE_REGION; import static com.linkedin.venice.ConfigKeys.AGGREGATE_REAL_TIME_SOURCE_REGION; import static com.linkedin.venice.ConfigKeys.CHILD_DATA_CENTER_KAFKA_URL_PREFIX; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_FOR_BATCH_ONLY; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_FOR_HYBRID; import static com.linkedin.venice.ConfigKeys.KAFKA_CLUSTER_MAP_KEY_NAME; import static com.linkedin.venice.ConfigKeys.KAFKA_CLUSTER_MAP_KEY_OTHER_URLS; import static com.linkedin.venice.ConfigKeys.KAFKA_CLUSTER_MAP_KEY_URL; @@ -73,51 +68,16 @@ public class VeniceTwoLayerMultiRegionMultiClusterWrapper extends ProcessWrapper } static ServiceProvider generateService( - int numberOfRegions, - int numberOfClustersInEachRegion, - int numberOfParentControllers, - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - Optional parentControllerProperties, - Optional serverProperties) { - return generateService( - numberOfRegions, - numberOfClustersInEachRegion, - numberOfParentControllers, - numberOfControllers, - numberOfServers, - numberOfRouters, - replicationFactor, - parentControllerProperties, - Optional.empty(), - serverProperties, - false); - } - - static ServiceProvider generateService( - int numberOfRegions, - int numberOfClustersInEachRegion, - int numberOfParentControllers, - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - Optional parentControllerPropertiesOverride, - Optional childControllerPropertiesOverride, - Optional serverProperties, - boolean forkServer) { + VeniceMultiRegionClusterCreateOptions options) { String parentRegionName = DEFAULT_PARENT_DATA_CENTER_REGION_NAME; - final List parentControllers = new ArrayList<>(numberOfParentControllers); - final List multiClusters = new ArrayList<>(numberOfRegions); + final List parentControllers = new ArrayList<>(options.getNumberOfParentControllers()); + final List multiClusters = new ArrayList<>(options.getNumberOfRegions()); /** * Enable participant system store by default in a two-layer multi-region set-up */ Properties defaultParentControllerProps = new Properties(); defaultParentControllerProps.setProperty(PARTICIPANT_MESSAGE_STORE_ENABLED, "true"); - defaultParentControllerProps.setProperty(ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED, "false"); ZkServerWrapper zkServer = null; PubSubBrokerWrapper parentPubSubBrokerWrapper = null; @@ -133,8 +93,8 @@ static ServiceProvider generateSer Map clusterToD2 = new HashMap<>(); Map clusterToServerD2 = new HashMap<>(); - String[] clusterNames = new String[numberOfClustersInEachRegion]; - for (int i = 0; i < numberOfClustersInEachRegion; i++) { + String[] clusterNames = new String[options.getNumberOfClusters()]; + for (int i = 0; i < options.getNumberOfClusters(); i++) { String clusterName = "venice-cluster" + i; clusterNames[i] = clusterName; String routerD2ServiceName = "venice-" + i; @@ -142,9 +102,9 @@ static ServiceProvider generateSer String serverD2ServiceName = Utils.getUniqueString(clusterName + "_d2"); clusterToServerD2.put(clusterName, serverD2ServiceName); } - List childRegionName = new ArrayList<>(numberOfRegions); + List childRegionName = new ArrayList<>(options.getNumberOfRegions()); - for (int i = 0; i < numberOfRegions; i++) { + for (int i = 0; i < options.getNumberOfRegions(); i++) { childRegionName.add(CHILD_REGION_NAME_PREFIX + i); } @@ -158,10 +118,6 @@ static ServiceProvider generateSer Map zkServerByRegionName = new HashMap<>(childRegionName.size()); Map pubSubBrokerByRegionName = new HashMap<>(childRegionName.size()); - defaultParentControllerProps.put(ENABLE_NATIVE_REPLICATION_FOR_BATCH_ONLY, true); - defaultParentControllerProps.put(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY, true); - defaultParentControllerProps.put(ENABLE_NATIVE_REPLICATION_FOR_HYBRID, true); - defaultParentControllerProps.put(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID, true); defaultParentControllerProps .put(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_BATCH_ONLY_STORES, childRegionName.get(0)); defaultParentControllerProps @@ -171,7 +127,10 @@ static ServiceProvider generateSer final Properties finalParentControllerProperties = new Properties(); finalParentControllerProperties.putAll(defaultParentControllerProps); - parentControllerPropertiesOverride.ifPresent(finalParentControllerProperties::putAll); + Properties parentControllerPropsOverride = options.getParentControllerProperties(); + if (parentControllerPropsOverride != null) { + finalParentControllerProperties.putAll(parentControllerPropsOverride); + } Properties nativeReplicationRequiredChildControllerProps = new Properties(); nativeReplicationRequiredChildControllerProps.put(ADMIN_TOPIC_SOURCE_REGION, parentRegionName); @@ -196,14 +155,18 @@ static ServiceProvider generateSer defaultChildControllerProps.putAll(nativeReplicationRequiredChildControllerProps); defaultChildControllerProps.putAll(activeActiveRequiredChildControllerProps); defaultChildControllerProps.setProperty(PARTICIPANT_MESSAGE_STORE_ENABLED, "true"); - defaultChildControllerProps.setProperty(ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED, "true"); final Properties finalChildControllerProperties = new Properties(); finalChildControllerProperties.putAll(defaultChildControllerProps); - childControllerPropertiesOverride.ifPresent(finalChildControllerProperties::putAll); + Properties childControllerPropsOverride = options.getChildControllerProperties(); + if (childControllerPropsOverride != null) { + finalChildControllerProperties.putAll(childControllerPropsOverride); + } - Map> kafkaClusterMap = - addKafkaClusterIDMappingToServerConfigs(serverProperties, childRegionName, allPubSubBrokerWrappers); + Map> kafkaClusterMap = addKafkaClusterIDMappingToServerConfigs( + Optional.ofNullable(options.getServerProperties()), + childRegionName, + allPubSubBrokerWrappers); Map pubSubBrokerProps = PubSubBrokerWrapper.getBrokerDetailsForClients(allPubSubBrokerWrappers); LOGGER.info("### PubSub broker configs: {}", pubSubBrokerProps); @@ -211,24 +174,28 @@ static ServiceProvider generateSer finalChildControllerProperties.putAll(pubSubBrokerProps); // child controllers Properties additionalServerProps = new Properties(); - serverProperties.ifPresent(additionalServerProps::putAll); + Properties serverPropsOverride = options.getServerProperties(); + if (serverPropsOverride != null) { + additionalServerProps.putAll(serverPropsOverride); + } additionalServerProps.putAll(pubSubBrokerProps); - serverProperties = Optional.of(additionalServerProps); VeniceMultiClusterCreateOptions.Builder builder = - new VeniceMultiClusterCreateOptions.Builder(numberOfClustersInEachRegion) - .numberOfControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) + new VeniceMultiClusterCreateOptions.Builder().setNumberOfClusters(options.getNumberOfClusters()) + .numberOfControllers(options.getNumberOfChildControllers()) + .numberOfServers(options.getNumberOfServers()) + .numberOfRouters(options.getNumberOfRouters()) + .replicationFactor(options.getReplicationFactor()) .randomizeClusterName(false) - .multiRegionSetup(true) + .multiRegion(true) .childControllerProperties(finalChildControllerProperties) - .extraProperties(serverProperties.orElse(null)) - .forkServer(forkServer) + .extraProperties(additionalServerProps) + .sslToStorageNodes(options.isSslToStorageNodes()) + .sslToKafka(options.isSslToKafka()) + .forkServer(options.isForkServer()) .kafkaClusterMap(kafkaClusterMap); // Create multi-clusters - for (int i = 0; i < numberOfRegions; i++) { + for (int i = 0; i < options.getNumberOfRegions(); i++) { String regionName = childRegionName.get(i); builder.regionName(regionName) .kafkaBrokerWrapper(pubSubBrokerByRegionName.get(regionName)) @@ -249,16 +216,17 @@ static ServiceProvider generateSer VeniceControllerWrapper.PARENT_D2_CLUSTER_NAME, VeniceControllerWrapper.PARENT_D2_SERVICE_NAME); VeniceControllerCreateOptions parentControllerCreateOptions = - new VeniceControllerCreateOptions.Builder(clusterNames, zkServer, parentPubSubBrokerWrapper) - .replicationFactor(replicationFactor) + new VeniceControllerCreateOptions.Builder(clusterNames, zkServer, parentPubSubBrokerWrapper).multiRegion(true) + .replicationFactor(options.getReplicationFactor()) .childControllers(childControllers) .extraProperties(finalParentControllerProperties) .clusterToD2(clusterToD2) .clusterToServerD2(clusterToServerD2) .regionName(parentRegionName) + .authorizerService(options.getParentAuthorizerService()) .build(); // Create parentControllers for multi-cluster - for (int i = 0; i < numberOfParentControllers; i++) { + for (int i = 0; i < options.getNumberOfParentControllers(); i++) { VeniceControllerWrapper parentController = ServiceFactory.getVeniceController(parentControllerCreateOptions); parentControllers.add(parentController); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/kafka/ssl/AdminChannelWithSSLTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/kafka/ssl/AdminChannelWithSSLTest.java index 8400f27128..2613762e46 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/kafka/ssl/AdminChannelWithSSLTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/kafka/ssl/AdminChannelWithSSLTest.java @@ -1,16 +1,11 @@ package com.linkedin.venice.kafka.ssl; -import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.CHILD_REGION_NAME_PREFIX; -import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_PARENT_DATA_CENTER_REGION_NAME; - import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.MultiStoreResponse; -import com.linkedin.venice.integration.utils.PubSubBrokerConfigs; -import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; -import com.linkedin.venice.integration.utils.VeniceControllerCreateOptions; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; -import com.linkedin.venice.integration.utils.ZkServerWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; +import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.utils.SslUtils; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; @@ -28,36 +23,33 @@ public class AdminChannelWithSSLTest { @Test(timeOut = 180 * Time.MS_PER_SECOND) public void testEnd2EndWithKafkaSSLEnabled() { Utils.thisIsLocalhost(); - String clusterName = "test-cluster"; - try (ZkServerWrapper zkServer = ServiceFactory.getZkServer(); - PubSubBrokerWrapper pubSubBrokerWrapper = ServiceFactory.getPubSubBroker( - new PubSubBrokerConfigs.Builder().setZkWrapper(zkServer) - .setRegionName(DEFAULT_PARENT_DATA_CENTER_REGION_NAME) - .build()); - VeniceControllerWrapper childControllerWrapper = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder(clusterName, zkServer, pubSubBrokerWrapper).replicationFactor(1) - .partitionSize(10) - .rebalanceDelayMs(0) - .minActiveReplica(1) - .sslToKafka(true) - .regionName(CHILD_REGION_NAME_PREFIX + "0") - .build()); - ZkServerWrapper parentZk = ServiceFactory.getZkServer(); - VeniceControllerWrapper controllerWrapper = ServiceFactory.getVeniceController( - new VeniceControllerCreateOptions.Builder(clusterName, parentZk, pubSubBrokerWrapper) - .childControllers(new VeniceControllerWrapper[] { childControllerWrapper }) + try (VeniceTwoLayerMultiRegionMultiClusterWrapper venice = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) .sslToKafka(true) .build())) { - String secureControllerUrl = controllerWrapper.getSecureControllerUrl(); + + String clusterName = venice.getClusterNames()[0]; + VeniceControllerWrapper childControllerWrapper = venice.getChildRegions().get(0).getLeaderController(clusterName); + + String parentSecureControllerUrl = venice.getParentControllers().get(0).getSecureControllerUrl(); // Adding store String storeName = "test_store"; String owner = "test_owner"; String keySchemaStr = "\"long\""; String valueSchemaStr = "\"string\""; - try (ControllerClient controllerClient = - new ControllerClient(clusterName, secureControllerUrl, Optional.of(SslUtils.getVeniceLocalSslFactory()))) { + try (ControllerClient controllerClient = new ControllerClient( + clusterName, + parentSecureControllerUrl, + Optional.of(SslUtils.getVeniceLocalSslFactory()))) { controllerClient.createNewStore(storeName, owner, keySchemaStr, valueSchemaStr); TestUtils.waitForNonDeterministicAssertion(5, TimeUnit.SECONDS, () -> { MultiStoreResponse response = controllerClient.queryStoreList(false); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/multicluster/TestMetadataOperationInMultiCluster.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/multicluster/TestMetadataOperationInMultiCluster.java index 73b488c262..c4b488ed55 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/multicluster/TestMetadataOperationInMultiCluster.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/multicluster/TestMetadataOperationInMultiCluster.java @@ -44,7 +44,8 @@ public void testCreateStoreAndVersionForMultiCluster() { String keySchema = "\"string\""; String valSchema = "\"string\""; - VeniceMultiClusterCreateOptions options = new VeniceMultiClusterCreateOptions.Builder(2).numberOfControllers(3) + VeniceMultiClusterCreateOptions options = new VeniceMultiClusterCreateOptions.Builder().setNumberOfClusters(2) + .numberOfControllers(3) .numberOfServers(1) .numberOfRouters(1) .regionName(VeniceClusterWrapperConstants.STANDALONE_REGION_NAME) @@ -133,7 +134,8 @@ public void testCreateStoreAndVersionForMultiCluster() { @Test public void testRunVPJInMultiCluster() throws Exception { - VeniceMultiClusterCreateOptions options = new VeniceMultiClusterCreateOptions.Builder(2).numberOfControllers(3) + VeniceMultiClusterCreateOptions options = new VeniceMultiClusterCreateOptions.Builder().setNumberOfClusters(2) + .numberOfControllers(3) .numberOfServers(1) .numberOfRouters(1) .regionName(VeniceClusterWrapperConstants.STANDALONE_REGION_NAME) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java index 6d16863aa1..3322801f43 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java @@ -39,7 +39,6 @@ import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.PersistenceType; -import com.linkedin.venice.meta.VeniceUserStoreType; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; import com.linkedin.venice.pushmonitor.ExecutionStatus; @@ -146,11 +145,6 @@ public void setUp() throws Exception { String parentControllerURLs = parentControllers.stream().map(VeniceControllerWrapper::getControllerUrl).collect(Collectors.joining(",")); parentControllerClient = new ControllerClient(clusterName, parentControllerURLs); - TestUtils.assertCommand( - parentControllerClient.configureActiveActiveReplicationForCluster( - true, - VeniceUserStoreType.INCREMENTAL_PUSH.toString(), - Optional.empty())); // create an active-active enabled store File inputDir = getTempDataDirectory(); Schema recordSchema = TestWriteUtils.writeSimpleAvroFileWithStringToStringSchema(inputDir); @@ -164,7 +158,6 @@ public void setUp() throws Exception { .setHybridOffsetLagThreshold(2) .setNativeReplicationEnabled(true) .setBackupVersionRetentionMs(1) - .setIncrementalPushEnabled(true) .setPartitionCount(NUMBER_OF_PARTITIONS) .setReplicationFactor(NUMBER_OF_REPLICAS); createStoreForJob(clusterName, keySchemaStr, valueSchemaStr, props, storeParms).close(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java index 1990f6a446..0eefce39f1 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/utils/IntegrationTestPushUtils.java @@ -175,9 +175,7 @@ public static ControllerClient createStoreForJob( keySchemaStr, valueSchema, props, - CompressionStrategy.NO_OP, - false, - false); + CompressionStrategy.NO_OP); } public static void makeStoreHybrid( @@ -258,9 +256,7 @@ public static ControllerClient createStoreForJob(String veniceClusterName, Schem recordSchema.getField(props.getProperty(KEY_FIELD_PROP, DEFAULT_KEY_FIELD_PROP)).schema().toString(), recordSchema.getField(props.getProperty(VALUE_FIELD_PROP, DEFAULT_VALUE_FIELD_PROP)).schema().toString(), props, - CompressionStrategy.NO_OP, - false, - false); + CompressionStrategy.NO_OP); } public static ControllerClient createStoreForJob( @@ -268,17 +264,13 @@ public static ControllerClient createStoreForJob( String keySchemaStr, String valueSchemaStr, Properties props, - CompressionStrategy compressionStrategy, - boolean chunkingEnabled, - boolean incrementalPushEnabled) { + CompressionStrategy compressionStrategy) { UpdateStoreQueryParams storeParams = new UpdateStoreQueryParams().setStorageQuotaInByte(Store.UNLIMITED_STORAGE_QUOTA) .setCompressionStrategy(compressionStrategy) .setBatchGetLimit(2000) - .setReadQuotaInCU(DEFAULT_PER_ROUTER_READ_QUOTA) - .setChunkingEnabled(chunkingEnabled) - .setIncrementalPushEnabled(incrementalPushEnabled); + .setReadQuotaInCU(DEFAULT_PER_ROUTER_READ_QUOTA); return createStoreForJob(veniceClusterName, keySchemaStr, valueSchemaStr, props, storeParams); } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/Admin.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/Admin.java index 285daea6b5..a2637f2379 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/Admin.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/Admin.java @@ -3,6 +3,7 @@ import com.linkedin.venice.acl.AclException; import com.linkedin.venice.common.VeniceSystemStoreType; import com.linkedin.venice.controller.kafka.consumer.AdminConsumerService; +import com.linkedin.venice.controller.supersetschema.SupersetSchemaGenerator; import com.linkedin.venice.controllerapi.NodeReplicasReadinessState; import com.linkedin.venice.controllerapi.RepushInfo; import com.linkedin.venice.controllerapi.StoreComparisonInfo; @@ -14,6 +15,7 @@ import com.linkedin.venice.helix.HelixReadOnlyZKSharedSystemStoreRepository; import com.linkedin.venice.helix.Replica; import com.linkedin.venice.meta.Instance; +import com.linkedin.venice.meta.ReadWriteSchemaRepository; import com.linkedin.venice.meta.RegionPushDetails; import com.linkedin.venice.meta.RoutersClusterConfig; import com.linkedin.venice.meta.Store; @@ -25,6 +27,7 @@ import com.linkedin.venice.meta.Version; import com.linkedin.venice.persona.StoragePersona; import com.linkedin.venice.pubsub.PubSubConsumerAdapterFactory; +import com.linkedin.venice.pubsub.PubSubTopicRepository; import com.linkedin.venice.pubsub.manager.TopicManager; import com.linkedin.venice.pushmonitor.ExecutionStatus; import com.linkedin.venice.pushstatushelper.PushStatusStoreReader; @@ -386,7 +389,14 @@ default SchemaEntry addValueSchema(String clusterName, String storeName, String SchemaEntry.DEFAULT_SCHEMA_CREATION_COMPATIBILITY_TYPE); } - SchemaEntry addSupersetSchema( + /** + * Add a new superset schema for the given store with all specified properties. + *

+ * Generate the superset schema off the current schema and latest superset schema (if any. If not, pick the latest + * value schema) existing in the store. + * If the newly generated superset schema is unique add it to the store and update latestSuperSetValueSchemaId of the store. + */ + void addSupersetSchema( String clusterName, String storeName, String valueSchemaStr, @@ -396,15 +406,6 @@ SchemaEntry addSupersetSchema( DerivedSchemaEntry addDerivedSchema(String clusterName, String storeName, int valueSchemaId, String derivedSchemaStr); - Set getInUseValueSchemaIds(String clusterName, String storeName); - - /** - * Deletes a store's values schema with ids `except` the ids passed in the argument inuseValueSchemaIds - */ - void deleteValueSchemas(String clusterName, String storeName, Set inuseValueSchemaIds); - - StoreMetaValue getMetaStoreValue(StoreMetaKey storeMetaKey, String storeName); - /** * This method skips most precondition checks and is intended for only internal use. */ @@ -430,6 +431,15 @@ RmdSchemaEntry addReplicationMetadataSchema( int replicationMetadataVersionId, String replicationMetadataSchemaStr); + StoreMetaValue getMetaStoreValue(StoreMetaKey storeMetaKey, String storeName); + + Set getInUseValueSchemaIds(String clusterName, String storeName); + + /** + * Deletes a store's values schema with ids `except` the ids passed in the argument inuseValueSchemaIds + */ + void deleteValueSchemas(String clusterName, String storeName, Set inuseValueSchemaIds); + void validateAndMaybeRetrySystemStoreAutoCreation( String clusterName, String storeName, @@ -447,8 +457,6 @@ void validateAndMaybeRetrySystemStoreAutoCreation( void rollbackToBackupVersion(String clusterName, String storeName, String regionFilter); - void setStoreLargestUsedVersion(String clusterName, String storeName, int versionNumber); - void setStoreOwner(String clusterName, String storeName, String owner); void setStorePartitionCount(String clusterName, String storeName, int partitionCount); @@ -754,6 +762,13 @@ void updateRoutersClusterConfig( */ boolean isParent(); + /** + * The "Primary Controller" term is used to refer to whichever controller is the main controller in a Venice set-up. + * In a multi-region deployment, the primary controller is the parent controller. + * In a single-region deployment, the primary controller is the only controller. + */ + boolean isPrimary(); + /** * Get child datacenter to child controller url mapping. * @return A map of child datacenter -> child controller url @@ -799,18 +814,6 @@ void updateRoutersClusterConfig( */ List getClustersLeaderOf(); - /** - * Enable/disable native replications for certain stores (batch only, hybrid only, incremental push, hybrid or incremental push, - * all) in a cluster. If storeName is not empty, only the specified store might be updated. - */ - void configureNativeReplication( - String cluster, - VeniceUserStoreType storeType, - Optional storeName, - boolean enableNativeReplicationForCluster, - Optional newSourceFabric, - Optional regionsFilter); - /** * Enable/disable active active replications for certain stores (batch only, hybrid only, incremental push, hybrid or incremental push, * all) in a cluster. If storeName is not empty, only the specified store might be updated. @@ -1001,4 +1004,31 @@ default void clearInstanceMonitor(String clusterName) { * Read the latest heartbeat timestamp from system store. If it failed to read from system store, this method should return -1. */ long getHeartbeatFromSystemStore(String clusterName, String storeName); + + /** + * @return the aggregate resources required by controller to manage a Venice cluster. + */ + HelixVeniceClusterResources getHelixVeniceClusterResources(String cluster); + + /** + * @return the aggregate resources required by controller to manage a Venice cluster. + */ + PubSubTopicRepository getPubSubTopicRepository(); + + default Schema getSupersetOrLatestValueSchema(String clusterName, Store store) { + ReadWriteSchemaRepository schemaRepository = getHelixVeniceClusterResources(clusterName).getSchemaRepository(); + // If already a superset schema exists, try to generate the new superset from that and the input value schema + SchemaEntry existingSchema = schemaRepository.getSupersetOrLatestValueSchema(store.getName()); + return existingSchema == null ? null : existingSchema.getSchema(); + } + + /** + * Return the current superset schema generator for the given cluster. + */ + SupersetSchemaGenerator getSupersetSchemaGenerator(String clusterName); + + /** + * Return the multi-cluster configs for the controller. + */ + VeniceControllerMultiClusterConfig getMultiClusterConfigs(); } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/StoreViewUtils.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/StoreViewUtils.java index 3e363fdd6b..aaf6c462e5 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/StoreViewUtils.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/StoreViewUtils.java @@ -20,28 +20,7 @@ public class StoreViewUtils { private static final VeniceJsonSerializer viewConfigVeniceJsonSerializer = new VeniceJsonSerializer<>(ViewConfig.class); - static Map convertStringMapViewToStoreViewConfigRecordMap( - Map stringMap) throws VeniceException { - Map mergedViewConfigRecords = new HashMap<>(); - if (!stringMap.isEmpty()) { - for (Map.Entry stringViewConfig: stringMap.entrySet()) { - try { - ViewConfig viewConfig = - viewConfigVeniceJsonSerializer.deserialize(stringViewConfig.getValue().getBytes(), ""); - StoreViewConfigRecord newViewConfigRecord = new StoreViewConfigRecord( - viewConfig.getViewClassName(), - CollectionUtils.getStringKeyCharSequenceValueMapFromStringMap(viewConfig.getViewParameters())); - mergedViewConfigRecords.put(stringViewConfig.getKey(), newViewConfigRecord); - } catch (IOException e) { - LOGGER.error("Failed to serialize provided view config: {}", stringViewConfig.getValue()); - throw new VeniceException("Failed to serialize provided view config:" + stringViewConfig.getValue(), e); - } - } - } - return mergedViewConfigRecords; - } - - static Map convertStringMapViewToStoreViewConfigMap(Map stringMap) { + public static Map convertStringMapViewToStoreViewConfigMap(Map stringMap) { Map mergedViewConfigRecords = new HashMap<>(); if (!stringMap.isEmpty()) { for (Map.Entry stringViewConfig: stringMap.entrySet()) { @@ -62,20 +41,20 @@ static Map convertStringMapViewToStoreViewConfigMap(Map return mergedViewConfigRecords; } - static Map convertStringMapViewToViewConfigMap(Map stringMap) { + public static Map convertStringMapViewToViewConfigMap(Map stringMap) { return convertStringMapViewToStoreViewConfigMap(stringMap).entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> new ViewConfigImpl(e.getValue()))); } - static Map convertViewConfigMapToStoreViewRecordMap( + public static Map convertViewConfigMapToStoreViewRecordMap( Map viewConfigMap) { return viewConfigMap.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> convertViewConfigToStoreViewConfigRecord(e.getValue()))); } - static StoreViewConfigRecord convertViewConfigToStoreViewConfigRecord(ViewConfig viewConfig) { + public static StoreViewConfigRecord convertViewConfigToStoreViewConfigRecord(ViewConfig viewConfig) { return new StoreViewConfigRecord(viewConfig.getViewClassName(), viewConfig.dataModel().getViewParameters()); } } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerClusterConfig.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerClusterConfig.java index 31666db60f..020d04862a 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerClusterConfig.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerClusterConfig.java @@ -1,6 +1,7 @@ package com.linkedin.venice.controller; import static com.linkedin.venice.CommonConfigKeys.SSL_FACTORY_CLASS_NAME; +import static com.linkedin.venice.ConfigKeys.ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST; import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_REPLICATION_FACTOR; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.CLUSTER_NAME; @@ -10,6 +11,7 @@ import static com.linkedin.venice.ConfigKeys.CONTROLLER_DISABLE_PARENT_REQUEST_TOPIC_FOR_STREAM_PUSHES; import static com.linkedin.venice.ConfigKeys.CONTROLLER_JETTY_CONFIG_OVERRIDE_PREFIX; import static com.linkedin.venice.ConfigKeys.CONTROLLER_NAME; +import static com.linkedin.venice.ConfigKeys.CONTROLLER_PARENT_MODE; import static com.linkedin.venice.ConfigKeys.CONTROLLER_SCHEMA_VALIDATION_ENABLED; import static com.linkedin.venice.ConfigKeys.CONTROLLER_SSL_ENABLED; import static com.linkedin.venice.ConfigKeys.DEFAULT_MAX_NUMBER_OF_PARTITIONS; @@ -21,15 +23,9 @@ import static com.linkedin.venice.ConfigKeys.DEFAULT_REPLICA_FACTOR; import static com.linkedin.venice.ConfigKeys.DEFAULT_ROUTING_STRATEGY; import static com.linkedin.venice.ConfigKeys.DELAY_TO_REBALANCE_MS; -import static com.linkedin.venice.ConfigKeys.ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY_STORE; import static com.linkedin.venice.ConfigKeys.ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE; import static com.linkedin.venice.ConfigKeys.ENABLE_HYBRID_PUSH_SSL_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.ENABLE_HYBRID_PUSH_SSL_WHITELIST; -import static com.linkedin.venice.ConfigKeys.ENABLE_INCREMENTAL_PUSH_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_FOR_BATCH_ONLY; -import static com.linkedin.venice.ConfigKeys.ENABLE_NATIVE_REPLICATION_FOR_HYBRID; import static com.linkedin.venice.ConfigKeys.ENABLE_OFFLINE_PUSH_SSL_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.ENABLE_OFFLINE_PUSH_SSL_WHITELIST; import static com.linkedin.venice.ConfigKeys.ENABLE_PARTIAL_UPDATE_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES; @@ -43,7 +39,6 @@ import static com.linkedin.venice.ConfigKeys.KAFKA_MIN_IN_SYNC_REPLICAS; import static com.linkedin.venice.ConfigKeys.KAFKA_MIN_IN_SYNC_REPLICAS_ADMIN_TOPICS; import static com.linkedin.venice.ConfigKeys.KAFKA_MIN_IN_SYNC_REPLICAS_RT_TOPICS; -import static com.linkedin.venice.ConfigKeys.KAFKA_MIN_LOG_COMPACTION_LAG_MS; import static com.linkedin.venice.ConfigKeys.KAFKA_OVER_SSL; import static com.linkedin.venice.ConfigKeys.KAFKA_REPLICATION_FACTOR; import static com.linkedin.venice.ConfigKeys.KAFKA_REPLICATION_FACTOR_RT_TOPICS; @@ -51,6 +46,7 @@ import static com.linkedin.venice.ConfigKeys.LEAKED_PUSH_STATUS_CLEAN_UP_SERVICE_SLEEP_INTERVAL_MS; import static com.linkedin.venice.ConfigKeys.LEAKED_RESOURCE_ALLOWED_LINGER_TIME_MS; import static com.linkedin.venice.ConfigKeys.MIN_ACTIVE_REPLICA; +import static com.linkedin.venice.ConfigKeys.MULTI_REGION; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_BATCH_ONLY_STORES; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_HYBRID_STORES; import static com.linkedin.venice.ConfigKeys.OFFLINE_JOB_START_TIMEOUT_MS; @@ -68,7 +64,6 @@ import static com.linkedin.venice.SSLConfig.DEFAULT_CONTROLLER_SSL_ENABLED; import static com.linkedin.venice.VeniceConstants.DEFAULT_PER_ROUTER_READ_QUOTA; import static com.linkedin.venice.VeniceConstants.DEFAULT_SSL_FACTORY_CLASS_NAME; -import static com.linkedin.venice.pubsub.PubSubConstants.DEFAULT_KAFKA_MIN_LOG_COMPACTION_LAG_MS; import static com.linkedin.venice.pubsub.PubSubConstants.DEFAULT_KAFKA_REPLICATION_FACTOR; import com.linkedin.venice.SSLConfig; @@ -81,13 +76,16 @@ import com.linkedin.venice.pushmonitor.LeakedPushStatusCleanUpService; import com.linkedin.venice.pushmonitor.PushMonitorType; import com.linkedin.venice.utils.KafkaSSLUtils; +import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang.StringUtils; import org.apache.helix.controller.rebalancer.strategy.CrushRebalanceStrategy; import org.apache.kafka.common.protocol.SecurityProtocol; import org.apache.logging.log4j.LogManager; @@ -102,6 +100,13 @@ public class VeniceControllerClusterConfig { private final VeniceProperties props; private String clusterName; + + /** + * Specify if the current Venice deployment is deployed in a multi-region setup or not. + */ + private boolean multiRegion; + private boolean parent; + private String zkAddress; private String controllerName; private PersistenceType persistenceType; @@ -133,53 +138,12 @@ public class VeniceControllerClusterConfig { private boolean enableNearlinePushSSLAllowlist; private List pushSSLAllowlist; - /** - * TODO: the follower 3 cluster level configs remains in the code base in case the new cluster level configs are not - * working as expected. Once the new cluster level configs for native replication have been tested in prod, retire - * the following configs. - */ - /** - * When this option is enabled, all new batch-only store versions created will have native replication enabled so long - * as the store has leader follower also enabled. - */ - private boolean nativeReplicationEnabledForBatchOnly; - - /** - * When this option is enabled, all new hybrid store versions created will have native replication enabled so long - * as the store has leader follower also enabled. - */ - private boolean nativeReplicationEnabledForHybrid; - - /** - * When this option is enabled, all new batch-only stores will have native replication enabled in store config so long - * as the store has leader follower also enabled. - */ - private boolean nativeReplicationEnabledAsDefaultForBatchOnly; - - /** - * When this option is enabled, all new hybrid stores will have native replication enabled in store config so long - * as the store has leader follower also enabled. - */ - private boolean nativeReplicationEnabledAsDefaultForHybrid; - private String nativeReplicationSourceFabricAsDefaultForBatchOnly; private String nativeReplicationSourceFabricAsDefaultForHybrid; - /** - * When the following option is enabled, active-active enabled new user hybrid store will automatically - * have incremental push enabled. - */ - private boolean enabledIncrementalPushForHybridActiveActiveUserStores; - private boolean enablePartialUpdateForHybridActiveActiveUserStores; private boolean enablePartialUpdateForHybridNonActiveActiveUserStores; - /** - * When this option is enabled, all new batch-only stores will have active-active replication enabled in store config so long - * as the store has leader follower also enabled. - */ - private boolean activeActiveReplicationEnabledAsDefaultForBatchOnly; - /** * When this option is enabled, all new hybrid stores will have active-active replication enabled in store config so long * as the store has leader follower also enabled. @@ -219,7 +183,6 @@ public class VeniceControllerClusterConfig { private Optional minInSyncReplicasRealTimeTopics; private Optional minInSyncReplicasAdminTopics; private boolean kafkaLogCompactionForHybridStores; - private long kafkaMinLogCompactionLagInMs; /** * Alg used by helix to decide the mapping between replicas and nodes. @@ -270,7 +233,21 @@ public VeniceControllerClusterConfig(VeniceProperties props) { } private void initFieldsWithProperties(VeniceProperties props) { + parent = props.getBoolean(CONTROLLER_PARENT_MODE, false); + + Set activeActiveRealTimeSourceFabrics = Utils + .parseCommaSeparatedStringToSet(props.getString(ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, (String) null)); + /** + * Historically, {@link MULTI_REGION} was not a supported config. It was handled on a case-by-case basis by + * carefully setting feature configs on various components. While this works for ramping new features, it makes it + * hard to remove the feature flags once the feature is fully ramped. Ideally, this should be a mandatory config, + * but that would break backward compatibility and hence, we infer the multi-region setup through the presence of + * a valid {@link ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST}. + */ + boolean multiRegionInferred = parent || !activeActiveRealTimeSourceFabrics.isEmpty(); + clusterName = props.getString(CLUSTER_NAME); + multiRegion = props.getBoolean(MULTI_REGION, multiRegionInferred); zkAddress = props.getString(ZOOKEEPER_ADDRESS); controllerName = props.getString(CONTROLLER_NAME); kafkaReplicationFactor = props.getInt(KAFKA_REPLICATION_FACTOR, DEFAULT_KAFKA_REPLICATION_FACTOR); @@ -279,8 +256,6 @@ private void initFieldsWithProperties(VeniceProperties props) { minInSyncReplicasRealTimeTopics = props.getOptionalInt(KAFKA_MIN_IN_SYNC_REPLICAS_RT_TOPICS); minInSyncReplicasAdminTopics = props.getOptionalInt(KAFKA_MIN_IN_SYNC_REPLICAS_ADMIN_TOPICS); kafkaLogCompactionForHybridStores = props.getBoolean(KAFKA_LOG_COMPACTION_FOR_HYBRID_STORES, true); - kafkaMinLogCompactionLagInMs = - props.getLong(KAFKA_MIN_LOG_COMPACTION_LAG_MS, DEFAULT_KAFKA_MIN_LOG_COMPACTION_LAG_MS); replicationFactor = props.getInt(DEFAULT_REPLICA_FACTOR); minNumberOfPartitions = props.getInt(DEFAULT_NUMBER_OF_PARTITION); minNumberOfPartitionsForHybrid = props.getInt(DEFAULT_NUMBER_OF_PARTITION_FOR_HYBRID, minNumberOfPartitions); @@ -318,23 +293,25 @@ private void initFieldsWithProperties(VeniceProperties props) { routingStrategy = RoutingStrategy.CONSISTENT_HASH; } - nativeReplicationEnabledForBatchOnly = props.getBoolean(ENABLE_NATIVE_REPLICATION_FOR_BATCH_ONLY, false); - nativeReplicationEnabledAsDefaultForBatchOnly = - props.getBoolean(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY, false); - nativeReplicationEnabledForHybrid = props.getBoolean(ENABLE_NATIVE_REPLICATION_FOR_HYBRID, false); - nativeReplicationEnabledAsDefaultForHybrid = - props.getBoolean(ENABLE_NATIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID, false); nativeReplicationSourceFabricAsDefaultForBatchOnly = props.getString(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_BATCH_ONLY_STORES, ""); + + if (StringUtils.isEmpty(nativeReplicationSourceFabricAsDefaultForBatchOnly) && multiRegion) { + throw new ConfigurationException( + "Native replication source fabric as default for batch only stores must be set in multi-region setup."); + } + nativeReplicationSourceFabricAsDefaultForHybrid = props.getString(NATIVE_REPLICATION_SOURCE_FABRIC_AS_DEFAULT_FOR_HYBRID_STORES, ""); - activeActiveReplicationEnabledAsDefaultForBatchOnly = - props.getBoolean(ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_BATCH_ONLY_STORE, false); + + if (StringUtils.isEmpty(nativeReplicationSourceFabricAsDefaultForHybrid) && multiRegion) { + throw new ConfigurationException( + "Native replication source fabric as default for hybrid stores must be set in multi-region setup."); + } + activeActiveReplicationEnabledAsDefaultForHybrid = props.getBoolean(ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE, false); controllerSchemaValidationEnabled = props.getBoolean(CONTROLLER_SCHEMA_VALIDATION_ENABLED, true); - enabledIncrementalPushForHybridActiveActiveUserStores = - props.getBoolean(ENABLE_INCREMENTAL_PUSH_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES, false); enablePartialUpdateForHybridActiveActiveUserStores = props.getBoolean(ENABLE_PARTIAL_UPDATE_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES, false); enablePartialUpdateForHybridNonActiveActiveUserStores = @@ -419,6 +396,14 @@ public String getClusterName() { return clusterName; } + public boolean isMultiRegion() { + return multiRegion; + } + + public boolean isParent() { + return parent; + } + public final String getZkAddress() { return zkAddress; } @@ -583,30 +568,6 @@ public boolean isKafkaLogCompactionForHybridStoresEnabled() { return kafkaLogCompactionForHybridStores; } - public long getKafkaMinLogCompactionLagInMs() { - return kafkaMinLogCompactionLagInMs; - } - - public boolean isNativeReplicationEnabledForBatchOnly() { - return nativeReplicationEnabledForBatchOnly; - } - - public boolean isNativeReplicationEnabledAsDefaultForBatchOnly() { - return nativeReplicationEnabledAsDefaultForBatchOnly; - } - - public boolean isNativeReplicationEnabledForHybrid() { - return nativeReplicationEnabledForHybrid; - } - - public boolean isNativeReplicationEnabledAsDefaultForHybrid() { - return nativeReplicationEnabledAsDefaultForHybrid; - } - - public boolean isActiveActiveReplicationEnabledAsDefaultForBatchOnly() { - return activeActiveReplicationEnabledAsDefaultForBatchOnly; - } - public boolean isActiveActiveReplicationEnabledAsDefaultForHybrid() { return activeActiveReplicationEnabledAsDefaultForHybrid; } @@ -643,10 +604,6 @@ public String getChildDatacenters() { return childDatacenters; } - public boolean enabledIncrementalPushForHybridActiveActiveUserStores() { - return enabledIncrementalPushForHybridActiveActiveUserStores; - } - public boolean isEnablePartialUpdateForHybridActiveActiveUserStores() { return enablePartialUpdateForHybridActiveActiveUserStores; } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerConfig.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerConfig.java index 2e9d612661..0d11572f24 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerConfig.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerConfig.java @@ -1,26 +1,22 @@ package com.linkedin.venice.controller; import static com.linkedin.venice.ConfigConstants.DEFAULT_PUSH_STATUS_STORE_HEARTBEAT_EXPIRATION_TIME_IN_SECONDS; -import static com.linkedin.venice.ConfigKeys.ACTIVE_ACTIVE_ENABLED_ON_CONTROLLER; import static com.linkedin.venice.ConfigKeys.ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST; import static com.linkedin.venice.ConfigKeys.ADMIN_CHECK_READ_METHOD_FOR_KAFKA; import static com.linkedin.venice.ConfigKeys.ADMIN_CONSUMPTION_CYCLE_TIMEOUT_MS; import static com.linkedin.venice.ConfigKeys.ADMIN_CONSUMPTION_MAX_WORKER_THREAD_POOL_SIZE; -import static com.linkedin.venice.ConfigKeys.ADMIN_CONSUMPTION_TIMEOUT_MINUTES; import static com.linkedin.venice.ConfigKeys.ADMIN_HELIX_MESSAGING_CHANNEL_ENABLED; import static com.linkedin.venice.ConfigKeys.ADMIN_HOSTNAME; import static com.linkedin.venice.ConfigKeys.ADMIN_PORT; import static com.linkedin.venice.ConfigKeys.ADMIN_SECURE_PORT; -import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED; import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_SOURCE_REGION; import static com.linkedin.venice.ConfigKeys.AGGREGATE_REAL_TIME_SOURCE_REGION; import static com.linkedin.venice.ConfigKeys.ALLOW_CLUSTER_WIPE; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_ALLOWLIST; +import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_ALLOWLIST_LEGACY; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_D2_PREFIX; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_D2_SERVICE_NAME; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_URL_PREFIX; -import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_WHITELIST; -import static com.linkedin.venice.ConfigKeys.CHILD_CONTROLLER_ADMIN_TOPIC_CONSUMPTION_ENABLED; import static com.linkedin.venice.ConfigKeys.CHILD_DATA_CENTER_KAFKA_URL_PREFIX; import static com.linkedin.venice.ConfigKeys.CLUSTER_DISCOVERY_D2_SERVICE; import static com.linkedin.venice.ConfigKeys.CONCURRENT_INIT_ROUTINES_ENABLED; @@ -40,14 +36,13 @@ import static com.linkedin.venice.ConfigKeys.CONTROLLER_DISABLED_ROUTES; import static com.linkedin.venice.ConfigKeys.CONTROLLER_DISABLE_PARENT_TOPIC_TRUNCATION_UPON_COMPLETION; import static com.linkedin.venice.ConfigKeys.CONTROLLER_EARLY_DELETE_BACKUP_ENABLED; -import static com.linkedin.venice.ConfigKeys.CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD; import static com.linkedin.venice.ConfigKeys.CONTROLLER_ENABLE_DISABLED_REPLICA_ENABLED; import static com.linkedin.venice.ConfigKeys.CONTROLLER_ENFORCE_SSL; +import static com.linkedin.venice.ConfigKeys.CONTROLLER_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED; import static com.linkedin.venice.ConfigKeys.CONTROLLER_HAAS_SUPER_CLUSTER_NAME; import static com.linkedin.venice.ConfigKeys.CONTROLLER_IN_AZURE_FABRIC; import static com.linkedin.venice.ConfigKeys.CONTROLLER_MIN_SCHEMA_COUNT_TO_KEEP; import static com.linkedin.venice.ConfigKeys.CONTROLLER_PARENT_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED; -import static com.linkedin.venice.ConfigKeys.CONTROLLER_PARENT_MODE; import static com.linkedin.venice.ConfigKeys.CONTROLLER_PARENT_SYSTEM_STORE_HEARTBEAT_CHECK_WAIT_TIME_SECONDS; import static com.linkedin.venice.ConfigKeys.CONTROLLER_PARENT_SYSTEM_STORE_REPAIR_CHECK_INTERVAL_SECONDS; import static com.linkedin.venice.ConfigKeys.CONTROLLER_PARENT_SYSTEM_STORE_REPAIR_RETRY_COUNT; @@ -74,16 +69,13 @@ import static com.linkedin.venice.ConfigKeys.ERROR_PARTITION_PROCESSING_CYCLE_DELAY; import static com.linkedin.venice.ConfigKeys.FATAL_DATA_VALIDATION_FAILURE_TOPIC_RETENTION_MS; import static com.linkedin.venice.ConfigKeys.IDENTITY_PARSER_CLASS; -import static com.linkedin.venice.ConfigKeys.KAFKA_ADMIN_CLASS; -import static com.linkedin.venice.ConfigKeys.KAFKA_READ_ONLY_ADMIN_CLASS; -import static com.linkedin.venice.ConfigKeys.KAFKA_WRITE_ONLY_ADMIN_CLASS; import static com.linkedin.venice.ConfigKeys.KME_REGISTRATION_FROM_MESSAGE_HEADER_ENABLED; import static com.linkedin.venice.ConfigKeys.META_STORE_WRITER_CLOSE_CONCURRENCY; import static com.linkedin.venice.ConfigKeys.META_STORE_WRITER_CLOSE_TIMEOUT_MS; import static com.linkedin.venice.ConfigKeys.MIN_NUMBER_OF_STORE_VERSIONS_TO_PRESERVE; import static com.linkedin.venice.ConfigKeys.MIN_NUMBER_OF_UNUSED_KAFKA_TOPICS_TO_PRESERVE; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_FABRIC_ALLOWLIST; -import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_FABRIC_WHITELIST; +import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_FABRIC_ALLOWLIST_LEGACY; import static com.linkedin.venice.ConfigKeys.NATIVE_REPLICATION_SOURCE_FABRIC; import static com.linkedin.venice.ConfigKeys.PARENT_CONTROLLER_MAX_ERRORED_TOPIC_NUM_TO_KEEP; import static com.linkedin.venice.ConfigKeys.PARENT_CONTROLLER_WAITING_TIME_FOR_CONSUMPTION_MS; @@ -98,16 +90,11 @@ import static com.linkedin.venice.ConfigKeys.SYSTEM_SCHEMA_INITIALIZATION_AT_START_TIME_ENABLED; import static com.linkedin.venice.ConfigKeys.TERMINAL_STATE_TOPIC_CHECK_DELAY_MS; import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_DELAY_FACTOR; -import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_SEND_CONCURRENT_DELETES_REQUESTS; import static com.linkedin.venice.ConfigKeys.TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS; -import static com.linkedin.venice.ConfigKeys.TOPIC_CREATION_THROTTLING_TIME_WINDOW_MS; -import static com.linkedin.venice.ConfigKeys.TOPIC_DELETION_STATUS_POLL_INTERVAL_MS; -import static com.linkedin.venice.ConfigKeys.TOPIC_MANAGER_KAFKA_OPERATION_TIMEOUT_MS; import static com.linkedin.venice.ConfigKeys.UNREGISTER_METRIC_FOR_DELETED_STORE_ENABLED; import static com.linkedin.venice.ConfigKeys.USE_DA_VINCI_SPECIFIC_EXECUTION_STATUS_FOR_ERROR; import static com.linkedin.venice.ConfigKeys.USE_PUSH_STATUS_STORE_FOR_INCREMENTAL_PUSH; import static com.linkedin.venice.ConfigKeys.VENICE_STORAGE_CLUSTER_LEADER_HAAS; -import static com.linkedin.venice.pubsub.PubSubConstants.PUBSUB_TOPIC_DELETION_STATUS_POLL_INTERVAL_MS_DEFAULT_VALUE; import static com.linkedin.venice.pubsub.PubSubConstants.PUBSUB_TOPIC_MANAGER_METADATA_FETCHER_CONSUMER_POOL_SIZE_DEFAULT_VALUE; import com.linkedin.venice.authorization.DefaultIdentityParser; @@ -116,14 +103,12 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.pubsub.PubSubAdminAdapterFactory; import com.linkedin.venice.pubsub.PubSubClientsFactory; -import com.linkedin.venice.pubsub.adapter.kafka.admin.ApacheKafkaAdminAdapter; import com.linkedin.venice.status.BatchJobHeartbeatConfigs; import com.linkedin.venice.utils.RegionUtils; import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import java.time.Duration; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -146,17 +131,14 @@ */ public class VeniceControllerConfig extends VeniceControllerClusterConfig { private static final Logger LOGGER = LogManager.getLogger(VeniceControllerConfig.class); - private static final String LIST_SEPARATOR = ",\\s*"; private final int adminPort; - private final String adminHostname; private final int adminSecurePort; private final int controllerClusterReplica; private final String controllerClusterName; private final String controllerClusterZkAddress; - private final boolean parent; - private final List childDataCenterAllowlist; + private final Set childDataCenterAllowlist; private final Map childDataCenterControllerUrlMap; private final String d2ServiceName; private final String clusterDiscoveryD2ServiceName; @@ -168,11 +150,9 @@ public class VeniceControllerConfig extends VeniceControllerClusterConfig { // heartbeat. private final Duration batchJobHeartbeatTimeout; private final Duration batchJobHeartbeatInitialBufferTime; - private final long adminConsumptionTimeoutMinute; private final long adminConsumptionCycleTimeoutMs; private final int adminConsumptionMaxWorkerThreadPoolSize; private final double storageEngineOverheadRatio; - private final long topicCreationThrottlingTimeWindowMs; private final long deprecatedJobTopicRetentionMs; private final long fatalDataValidationFailureRetentionMs; @@ -182,7 +162,6 @@ public class VeniceControllerConfig extends VeniceControllerClusterConfig { private final long disabledReplicaEnablerServiceIntervalMs; private final int topicCleanupDelayFactor; - private final int topicManagerKafkaOperationTimeOutMs; private final int topicManagerMetadataFetcherConsumerPoolSize; private final int topicManagerMetadataFetcherThreadPoolSize; private final int minNumberOfUnusedKafkaTopicsToPreserve; @@ -191,20 +170,13 @@ public class VeniceControllerConfig extends VeniceControllerClusterConfig { private final String pushJobStatusStoreClusterName; private final boolean participantMessageStoreEnabled; private final String systemSchemaClusterName; - private final int topicDeletionStatusPollIntervalMs; private final boolean adminHelixMessagingChannelEnabled; private final boolean isControllerClusterLeaderHAAS; private final boolean isVeniceClusterLeaderHAAS; private final String controllerHAASSuperClusterName; private final boolean earlyDeleteBackUpEnabled; - private final boolean sendConcurrentTopicDeleteRequestsEnabled; - private final boolean enableBatchPushFromAdminInChildController; private final boolean adminCheckReadMethodForKafka; - private final String kafkaAdminClass; - private final String kafkaWriteOnlyClass; - private final String kafkaReadOnlyClass; private final Map childDataCenterKafkaUrlMap; - private final boolean activeActiveEnabledOnController; private final Set activeActiveRealTimeSourceFabrics; private final String nativeReplicationSourceFabric; private final int errorPartitionAutoResetLimit; @@ -289,8 +261,6 @@ public class VeniceControllerConfig extends VeniceControllerClusterConfig { private final boolean allowClusterWipe; - private final boolean childControllerAdminTopicConsumptionEnabled; - private final boolean concurrentInitRoutinesEnabled; private final boolean controllerInAzureFabric; @@ -319,7 +289,7 @@ public class VeniceControllerConfig extends VeniceControllerClusterConfig { private final int parentSystemStoreRepairRetryCount; - private final boolean parentExternalSupersetSchemaGenerationEnabled; + private final boolean externalSupersetSchemaGenerationEnabled; private final boolean systemSchemaInitializationAtStartTimeEnabled; @@ -349,28 +319,24 @@ public VeniceControllerConfig(VeniceProperties props) { this.controllerClusterName = props.getString(CONTROLLER_CLUSTER, "venice-controllers"); this.controllerClusterReplica = props.getInt(CONTROLLER_CLUSTER_REPLICA, 3); this.controllerClusterZkAddress = props.getString(CONTROLLER_CLUSTER_ZK_ADDRESSS, getZkAddress()); - this.topicCreationThrottlingTimeWindowMs = - props.getLong(TOPIC_CREATION_THROTTLING_TIME_WINDOW_MS, 10 * Time.MS_PER_SECOND); - this.parent = props.getBoolean(CONTROLLER_PARENT_MODE, false); - this.activeActiveEnabledOnController = props.getBoolean(ACTIVE_ACTIVE_ENABLED_ON_CONTROLLER, false); - this.activeActiveRealTimeSourceFabrics = - Utils.parseCommaSeparatedStringToSet(props.getString(ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, "")); + this.activeActiveRealTimeSourceFabrics = Utils + .parseCommaSeparatedStringToSet(props.getString(ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, (String) null)); validateActiveActiveConfigs(); - // go/inclusivecode deferred(Will be replaced when clients have migrated) - String dataCenterAllowlist = props.getStringWithAlternative(CHILD_CLUSTER_ALLOWLIST, CHILD_CLUSTER_WHITELIST); + String dataCenterAllowlist = + props.getStringWithAlternative(CHILD_CLUSTER_ALLOWLIST, CHILD_CLUSTER_ALLOWLIST_LEGACY); if (dataCenterAllowlist.isEmpty()) { this.childDataCenterControllerUrlMap = Collections.emptyMap(); this.childDataCenterControllerD2Map = Collections.emptyMap(); - this.childDataCenterAllowlist = Collections.emptyList(); + this.childDataCenterAllowlist = Collections.emptySet(); } else { this.childDataCenterControllerUrlMap = parseClusterMap(props, dataCenterAllowlist); this.childDataCenterControllerD2Map = parseClusterMap(props, dataCenterAllowlist, true); - this.childDataCenterAllowlist = Arrays.asList(dataCenterAllowlist.split(LIST_SEPARATOR)); + this.childDataCenterAllowlist = Utils.parseCommaSeparatedStringToSet(dataCenterAllowlist); } this.d2ServiceName = childDataCenterControllerD2Map.isEmpty() ? null : props.getString(CHILD_CLUSTER_D2_SERVICE_NAME); - if (this.parent) { + if (super.isParent()) { if (childDataCenterControllerUrlMap.isEmpty() && childDataCenterControllerD2Map.isEmpty()) { throw new VeniceException("child controller list can not be empty"); } @@ -378,8 +344,7 @@ public VeniceControllerConfig(VeniceProperties props) { this.parentFabrics = Utils.parseCommaSeparatedStringToSet(parentFabricList); String nativeReplicationSourceFabricAllowlist = props.getStringWithAlternative( NATIVE_REPLICATION_FABRIC_ALLOWLIST, - // go/inclusivecode deferred(will be removed once all configs have migrated) - NATIVE_REPLICATION_FABRIC_WHITELIST, + NATIVE_REPLICATION_FABRIC_ALLOWLIST_LEGACY, dataCenterAllowlist); this.childDataCenterKafkaUrlMap = parseChildDataCenterKafkaUrl(props, nativeReplicationSourceFabricAllowlist); } else { @@ -387,10 +352,9 @@ public VeniceControllerConfig(VeniceProperties props) { String nativeReplicationSourceFabricAllowlist = props.getStringWithAlternative( NATIVE_REPLICATION_FABRIC_ALLOWLIST, - // go/inclusivecode deferred(will be removed once all configs have migrated) - NATIVE_REPLICATION_FABRIC_WHITELIST, + NATIVE_REPLICATION_FABRIC_ALLOWLIST_LEGACY, ""); - if (nativeReplicationSourceFabricAllowlist == null || nativeReplicationSourceFabricAllowlist.length() == 0) { + if (nativeReplicationSourceFabricAllowlist == null || nativeReplicationSourceFabricAllowlist.isEmpty()) { this.childDataCenterKafkaUrlMap = Collections.emptyMap(); } else { this.childDataCenterKafkaUrlMap = parseChildDataCenterKafkaUrl(props, nativeReplicationSourceFabricAllowlist); @@ -413,7 +377,6 @@ public VeniceControllerConfig(VeniceProperties props) { props.getLong( BatchJobHeartbeatConfigs.HEARTBEAT_CONTROLLER_INITIAL_DELAY_CONFIG.getConfigName(), BatchJobHeartbeatConfigs.HEARTBEAT_CONTROLLER_INITIAL_DELAY_CONFIG.getDefaultValue())); - this.adminConsumptionTimeoutMinute = props.getLong(ADMIN_CONSUMPTION_TIMEOUT_MINUTES, TimeUnit.DAYS.toMinutes(5)); this.adminConsumptionCycleTimeoutMs = props.getLong(ADMIN_CONSUMPTION_CYCLE_TIMEOUT_MS, TimeUnit.MINUTES.toMillis(30)); // A value of one will result in a bad message for one store to block the admin message consumption of other stores. @@ -445,8 +408,6 @@ public VeniceControllerConfig(VeniceProperties props) { this.disabledReplicaEnablerServiceIntervalMs = props.getLong(CONTROLLER_DISABLED_REPLICA_ENABLER_INTERVAL_MS, TimeUnit.HOURS.toMillis(16)); - this.topicManagerKafkaOperationTimeOutMs = - props.getInt(TOPIC_MANAGER_KAFKA_OPERATION_TIMEOUT_MS, 30 * Time.MS_PER_SECOND); this.topicManagerMetadataFetcherConsumerPoolSize = props.getInt( PUBSUB_TOPIC_MANAGER_METADATA_FETCHER_CONSUMER_POOL_SIZE, PUBSUB_TOPIC_MANAGER_METADATA_FETCHER_CONSUMER_POOL_SIZE_DEFAULT_VALUE); @@ -472,8 +433,6 @@ public VeniceControllerConfig(VeniceProperties props) { } this.systemSchemaClusterName = props.getString(CONTROLLER_SYSTEM_SCHEMA_CLUSTER_NAME, ""); this.earlyDeleteBackUpEnabled = props.getBoolean(CONTROLLER_EARLY_DELETE_BACKUP_ENABLED, true); - this.topicDeletionStatusPollIntervalMs = props - .getInt(TOPIC_DELETION_STATUS_POLL_INTERVAL_MS, PUBSUB_TOPIC_DELETION_STATUS_POLL_INTERVAL_MS_DEFAULT_VALUE); // 2s this.isControllerClusterLeaderHAAS = props.getBoolean(CONTROLLER_CLUSTER_LEADER_HAAS, false); this.isVeniceClusterLeaderHAAS = props.getBoolean(VENICE_STORAGE_CLUSTER_LEADER_HAAS, false); this.controllerHAASSuperClusterName = props.getString(CONTROLLER_HAAS_SUPER_CLUSTER_NAME, ""); @@ -482,13 +441,6 @@ public VeniceControllerConfig(VeniceProperties props) { CONTROLLER_HAAS_SUPER_CLUSTER_NAME + " is required for " + CONTROLLER_CLUSTER_LEADER_HAAS + " or " + VENICE_STORAGE_CLUSTER_LEADER_HAAS + " to be set to true"); } - this.sendConcurrentTopicDeleteRequestsEnabled = - props.getBoolean(TOPIC_CLEANUP_SEND_CONCURRENT_DELETES_REQUESTS, false); - this.enableBatchPushFromAdminInChildController = - props.getBoolean(CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD, true); - this.kafkaAdminClass = props.getString(KAFKA_ADMIN_CLASS, ApacheKafkaAdminAdapter.class.getName()); - this.kafkaWriteOnlyClass = props.getString(KAFKA_WRITE_ONLY_ADMIN_CLASS, kafkaAdminClass); - this.kafkaReadOnlyClass = props.getString(KAFKA_READ_ONLY_ADMIN_CLASS, kafkaAdminClass); this.errorPartitionAutoResetLimit = props.getInt(ERROR_PARTITION_AUTO_RESET_LIMIT, 0); this.errorPartitionProcessingCycleDelay = props.getLong(ERROR_PARTITION_PROCESSING_CYCLE_DELAY, 5 * Time.MS_PER_MINUTE); @@ -530,14 +482,18 @@ public VeniceControllerConfig(VeniceProperties props) { props.getBoolean(CONTROLLER_ZK_SHARED_DAVINCI_PUSH_STATUS_SYSTEM_SCHEMA_STORE_AUTO_CREATION_ENABLED, true); this.systemStoreAclSynchronizationDelayMs = props.getLong(CONTROLLER_SYSTEM_STORE_ACL_SYNCHRONIZATION_DELAY_MS, TimeUnit.HOURS.toMillis(1)); - this.regionName = RegionUtils.getLocalRegionName(props, parent); + this.regionName = RegionUtils.getLocalRegionName(props, super.isParent()); LOGGER.info("Final region name for this node: {}", this.regionName); this.disabledRoutes = parseControllerRoutes(props, CONTROLLER_DISABLED_ROUTES, Collections.emptyList()); - this.adminTopicRemoteConsumptionEnabled = props.getBoolean(ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED, false); + this.adminTopicRemoteConsumptionEnabled = !super.isParent() && super.isMultiRegion(); if (adminTopicRemoteConsumptionEnabled && childDataCenterKafkaUrlMap.isEmpty()) { throw new VeniceException("Admin topic remote consumption is enabled but Kafka url map is empty"); } - this.adminTopicSourceRegion = props.getString(ADMIN_TOPIC_SOURCE_REGION, ""); + this.adminTopicSourceRegion = props.getString(ADMIN_TOPIC_SOURCE_REGION, (String) null); + if (adminTopicRemoteConsumptionEnabled && adminTopicSourceRegion == null) { + throw new VeniceException("Admin topic remote consumption is enabled but source region is not set"); + } + this.aggregateRealTimeSourceRegion = props.getString(AGGREGATE_REAL_TIME_SOURCE_REGION, ""); this.isAutoMaterializeMetaSystemStoreEnabled = props.getBoolean(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); @@ -548,8 +504,6 @@ public VeniceControllerConfig(VeniceProperties props) { this.metaStoreWriterCloseConcurrency = props.getInt(META_STORE_WRITER_CLOSE_CONCURRENCY, -1); this.emergencySourceRegion = props.getString(EMERGENCY_SOURCE_REGION, ""); this.allowClusterWipe = props.getBoolean(ALLOW_CLUSTER_WIPE, false); - this.childControllerAdminTopicConsumptionEnabled = - props.getBoolean(CHILD_CONTROLLER_ADMIN_TOPIC_CONSUMPTION_ENABLED, true); this.concurrentInitRoutinesEnabled = props.getBoolean(CONCURRENT_INIT_ROUTINES_ENABLED, false); this.controllerInAzureFabric = props.getBoolean(CONTROLLER_IN_AZURE_FABRIC, false); this.unregisterMetricForDeletedStoreEnabled = props.getBoolean(UNREGISTER_METRIC_FOR_DELETED_STORE_ENABLED, false); @@ -567,8 +521,10 @@ public VeniceControllerConfig(VeniceProperties props) { this.parentSystemStoreRepairRetryCount = props.getInt(CONTROLLER_PARENT_SYSTEM_STORE_REPAIR_RETRY_COUNT, 1); this.clusterDiscoveryD2ServiceName = props.getString(CLUSTER_DISCOVERY_D2_SERVICE, ClientConfig.DEFAULT_CLUSTER_DISCOVERY_D2_SERVICE_NAME); - this.parentExternalSupersetSchemaGenerationEnabled = - props.getBoolean(CONTROLLER_PARENT_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED, false); + this.externalSupersetSchemaGenerationEnabled = props.getBooleanWithAlternative( + CONTROLLER_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED, + CONTROLLER_PARENT_EXTERNAL_SUPERSET_SCHEMA_GENERATION_ENABLED, + false); this.systemSchemaInitializationAtStartTimeEnabled = props.getBoolean(SYSTEM_SCHEMA_INITIALIZATION_AT_START_TIME_ENABLED, false); this.isKMERegistrationFromMessageHeaderEnabled = @@ -589,27 +545,17 @@ public VeniceControllerConfig(VeniceProperties props) { } private void validateActiveActiveConfigs() { - if (this.activeActiveEnabledOnController && this.activeActiveRealTimeSourceFabrics.isEmpty()) { + if (super.isMultiRegion() && !super.isParent() && this.activeActiveRealTimeSourceFabrics.isEmpty()) { throw new VeniceException( String.format( - "The config %s cannot be empty when the child controller has A/A enabled " + "(%s == true).", - ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, - ACTIVE_ACTIVE_ENABLED_ON_CONTROLLER)); + "The config %s cannot be empty in child controllers when running in multi region mode.", + ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST)); - } else if (this.activeActiveEnabledOnController) { + } else if (!super.isMultiRegion() && !this.activeActiveRealTimeSourceFabrics.isEmpty()) { LOGGER.info( - "A/A is enabled on a child controller and {} == {}", + "System is not running in multi region mode. But {} is still set to {}.", ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, this.activeActiveRealTimeSourceFabrics); - } else { - LOGGER.info( - "A/A is not enabled on child controller. {}", - !this.activeActiveRealTimeSourceFabrics.isEmpty() - ? String.format( - " But %s is still set to %s.", - ACTIVE_ACTIVE_REAL_TIME_SOURCE_FABRIC_LIST, - this.activeActiveRealTimeSourceFabrics) - : ""); } } @@ -641,14 +587,6 @@ public String getControllerClusterZkAddress() { return controllerClusterZkAddress; } - public boolean isParent() { - return parent; - } - - public long getTopicCreationThrottlingTimeWindowMs() { - return topicCreationThrottlingTimeWindowMs; - } - public long getDeprecatedJobTopicRetentionMs() { return deprecatedJobTopicRetentionMs; } @@ -723,7 +661,7 @@ public Map getChildDataCenterKafkaUrlMap() { return childDataCenterKafkaUrlMap; } - public List getChildDataCenterAllowlist() { + public Set getChildDataCenterAllowlist() { return childDataCenterAllowlist; } @@ -759,10 +697,6 @@ public Duration getBatchJobHeartbeatInitialBufferTime() { return batchJobHeartbeatInitialBufferTime; } - public long getAdminConsumptionTimeoutMinutes() { - return adminConsumptionTimeoutMinute; - } - public boolean isEnableDisabledReplicaEnabled() { return enableDisabledReplicaEnabled; } @@ -783,10 +717,6 @@ public double getStorageEngineOverheadRatio() { return storageEngineOverheadRatio; } - public int getTopicManagerKafkaOperationTimeOutMs() { - return topicManagerKafkaOperationTimeOutMs; - } - public int getTopicManagerMetadataFetcherConsumerPoolSize() { return topicManagerMetadataFetcherConsumerPoolSize; } @@ -823,10 +753,6 @@ public String getSystemSchemaClusterName() { return systemSchemaClusterName; } - public int getTopicDeletionStatusPollIntervalMs() { - return topicDeletionStatusPollIntervalMs; - } - public boolean isAdminHelixMessagingChannelEnabled() { return adminHelixMessagingChannelEnabled; } @@ -847,26 +773,6 @@ public boolean isEarlyDeleteBackUpEnabled() { return earlyDeleteBackUpEnabled; } - public boolean isConcurrentTopicDeleteRequestsEnabled() { - return sendConcurrentTopicDeleteRequestsEnabled; - } - - public boolean isEnableBatchPushFromAdminInChildController() { - return enableBatchPushFromAdminInChildController; - } - - public String getKafkaAdminClass() { - return kafkaAdminClass; - } - - public String getKafkaWriteOnlyClass() { - return kafkaWriteOnlyClass; - } - - public String getKafkaReadOnlyClass() { - return kafkaReadOnlyClass; - } - public int getErrorPartitionAutoResetLimit() { return errorPartitionAutoResetLimit; } @@ -998,10 +904,6 @@ public boolean isClusterWipeAllowed() { return allowClusterWipe; } - public boolean isChildControllerAdminTopicConsumptionEnabled() { - return childControllerAdminTopicConsumptionEnabled; - } - public boolean isConcurrentInitRoutinesEnabled() { return concurrentInitRoutinesEnabled; } @@ -1058,8 +960,8 @@ public int getParentSystemStoreRepairRetryCount() { return parentSystemStoreRepairRetryCount; } - public boolean isParentExternalSupersetSchemaGenerationEnabled() { - return parentExternalSupersetSchemaGenerationEnabled; + public boolean isExternalSupersetSchemaGenerationEnabled() { + return externalSupersetSchemaGenerationEnabled; } public boolean isSystemSchemaInitializationAtStartTimeEnabled() { @@ -1097,18 +999,18 @@ public static Map parseClusterMap( String propsPrefix = D2Routing ? CHILD_CLUSTER_D2_PREFIX : CHILD_CLUSTER_URL_PREFIX; return parseChildDataCenterToValue(propsPrefix, clusterPros, datacenterAllowlist, (m, k, v, errMsg) -> { m.computeIfAbsent(k, key -> { - String[] uriList = v.split(LIST_SEPARATOR); + List uriList = Utils.parseCommaSeparatedStringToList(v); - if (D2Routing && uriList.length != 1) { + if (D2Routing && uriList.size() != 1) { throw new VeniceException(errMsg + ": can only have 1 zookeeper url"); } if (!D2Routing) { - if (uriList.length == 0) { + if (uriList.isEmpty()) { throw new VeniceException(errMsg + ": urls can not be empty"); } - if (Arrays.stream(uriList).anyMatch(uri -> (!uri.startsWith("http://") && !uri.startsWith("https://")))) { + if (uriList.stream().anyMatch(uri -> (!uri.startsWith("http://") && !uri.startsWith("https://")))) { throw new VeniceException(errMsg + ": urls must begin with http:// or https://"); } } @@ -1148,7 +1050,7 @@ private static Map parseChildDataCenterToValue( } Map outputMap = new HashMap<>(); - List allowlist = Arrays.asList(datacenterAllowlist.split(LIST_SEPARATOR)); + Set allowlist = Utils.parseCommaSeparatedStringToSet(datacenterAllowlist); for (Map.Entry uriEntry: childDataCenterKafkaUriProps.entrySet()) { String datacenter = (String) uriEntry.getKey(); diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerMultiClusterConfig.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerMultiClusterConfig.java index 7638a8829a..dc701ccde6 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerMultiClusterConfig.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerMultiClusterConfig.java @@ -59,6 +59,10 @@ public boolean adminCheckReadMethodForKafka() { return getCommonConfig().adminCheckReadMethodForKafka(); } + public boolean isMultiRegion() { + return getCommonConfig().isMultiRegion(); + } + public boolean isParent() { return getCommonConfig().isParent(); } @@ -103,10 +107,6 @@ public String getSslFactoryClassName() { return getCommonConfig().getSslFactoryClassName(); } - public boolean isDisableParentRequestTopicForStreamPushes() { - return getCommonConfig().isDisableParentRequestTopicForStreamPushes(); - } - public long getDeprecatedJobTopicRetentionMs() { return getCommonConfig().getDeprecatedJobTopicRetentionMs(); } @@ -155,10 +155,6 @@ public Duration getBatchJobHeartbeatInitialBufferTime() { return getCommonConfig().getBatchJobHeartbeatInitialBufferTime(); } - public long getTopicCreationThrottlingTimeWindowMs() { - return getCommonConfig().getTopicCreationThrottlingTimeWindowMs(); - } - public Map getClusterToD2Map() { return getCommonConfig().getClusterToD2Map(); } @@ -167,22 +163,6 @@ public Map getClusterToServerD2Map() { return getCommonConfig().getClusterToServerD2Map(); } - public int getTopicManagerKafkaOperationTimeOutMs() { - return getCommonConfig().getTopicManagerKafkaOperationTimeOutMs(); - } - - public int getTopicDeletionStatusPollIntervalMs() { - return getCommonConfig().getTopicDeletionStatusPollIntervalMs(); - } - - public boolean isConcurrentTopicDeleteRequestsEnabled() { - return getCommonConfig().isConcurrentTopicDeleteRequestsEnabled(); - } - - public long getKafkaMinLogCompactionLagInMs() { - return getCommonConfig().getKafkaMinLogCompactionLagInMs(); - } - public int getMinNumberOfUnusedKafkaTopicsToPreserve() { return getCommonConfig().getMinNumberOfUnusedKafkaTopicsToPreserve(); } @@ -223,10 +203,6 @@ public String getSystemSchemaClusterName() { return getCommonConfig().getSystemSchemaClusterName(); } - public boolean isEnableBatchPushFromAdminInChildController() { - return getCommonConfig().isEnableBatchPushFromAdminInChildController(); - } - public long getBackupVersionDefaultRetentionMs() { return getCommonConfig().getBackupVersionDefaultRetentionMs(); } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerService.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerService.java index 2d102226f3..ce9b1141a4 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerService.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceControllerService.java @@ -90,6 +90,7 @@ public VeniceControllerService( sslConfig, accessController, icProvider, + externalSupersetSchemaGenerator, pubSubTopicRepository, pubSubClientsFactory, Arrays.asList(initRoutineForPushJobDetailsSystemStore, initRoutineForHeartbeatSystemStore)); @@ -172,15 +173,19 @@ public VeniceControllerService( new LandFillObjectPool<>(KafkaMessageEnvelope::new), new LandFillObjectPool<>(KafkaMessageEnvelope::new)); for (String cluster: multiClusterConfigs.getClusters()) { - AdminConsumerService adminConsumerService = new AdminConsumerService( - internalAdmin, - multiClusterConfigs.getControllerConfig(cluster), - metricsRepository, - pubSubTopicRepository, - pubSubMessageDeserializer); - this.consumerServicesByClusters.put(cluster, adminConsumerService); + VeniceControllerConfig clusterConfig = multiClusterConfigs.getControllerConfig(cluster); + if (clusterConfig.isMultiRegion()) { + // Enable admin channel consumption only for multi-region setups + AdminConsumerService adminConsumerService = new AdminConsumerService( + internalAdmin, + clusterConfig, + metricsRepository, + pubSubTopicRepository, + pubSubMessageDeserializer); + this.consumerServicesByClusters.put(cluster, adminConsumerService); - this.admin.setAdminConsumerService(cluster, adminConsumerService); + this.admin.setAdminConsumerService(cluster, adminConsumerService); + } } } @@ -209,7 +214,9 @@ private LingeringStoreVersionChecker createLingeringStoreVersionChecker( public boolean startInner() { for (String clusterName: multiClusterConfigs.getClusters()) { admin.initStorageCluster(clusterName); - consumerServicesByClusters.get(clusterName).start(); + if (multiClusterConfigs.isMultiRegion()) { + consumerServicesByClusters.get(clusterName).start(); + } LOGGER.info("started cluster: {}", clusterName); } LOGGER.info("Started Venice controller."); @@ -227,7 +234,9 @@ public void stopInner() { // prevent the partial updates. admin.stop(clusterName); try { - consumerServicesByClusters.get(clusterName).stop(); + if (multiClusterConfigs.isMultiRegion()) { + consumerServicesByClusters.get(clusterName).stop(); + } } catch (Exception e) { LOGGER.error("Got exception when stop AdminConsumerService", e); } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java index eec89ad943..85bcf94e04 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java @@ -7,9 +7,6 @@ import static com.linkedin.venice.ConfigKeys.SSL_KAFKA_BOOTSTRAP_SERVERS; import static com.linkedin.venice.ConfigKeys.SSL_TO_KAFKA_LEGACY; import static com.linkedin.venice.controller.UserSystemStoreLifeCycleHelper.AUTO_META_SYSTEM_STORE_PUSH_ID_PREFIX; -import static com.linkedin.venice.meta.HybridStoreConfigImpl.DEFAULT_HYBRID_OFFSET_LAG_THRESHOLD; -import static com.linkedin.venice.meta.HybridStoreConfigImpl.DEFAULT_HYBRID_TIME_LAG_THRESHOLD; -import static com.linkedin.venice.meta.HybridStoreConfigImpl.DEFAULT_REWIND_TIME_IN_SECONDS; import static com.linkedin.venice.meta.Store.NON_EXISTING_VERSION; import static com.linkedin.venice.meta.Version.PushType; import static com.linkedin.venice.meta.VersionStatus.ERROR; @@ -53,9 +50,12 @@ import com.linkedin.venice.controller.init.SystemSchemaInitializationRoutine; import com.linkedin.venice.controller.kafka.StoreStatusDecider; import com.linkedin.venice.controller.kafka.consumer.AdminConsumerService; -import com.linkedin.venice.controller.kafka.protocol.admin.HybridStoreConfigRecord; -import com.linkedin.venice.controller.kafka.protocol.admin.StoreViewConfigRecord; import com.linkedin.venice.controller.stats.DisabledPartitionStats; +import com.linkedin.venice.controller.supersetschema.DefaultSupersetSchemaGenerator; +import com.linkedin.venice.controller.supersetschema.SupersetSchemaGenerator; +import com.linkedin.venice.controller.util.PrimaryControllerConfigUpdateUtils; +import com.linkedin.venice.controller.util.UpdateStoreUtils; +import com.linkedin.venice.controller.util.UpdateStoreWrapper; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.ControllerResponse; import com.linkedin.venice.controllerapi.ControllerRoute; @@ -70,7 +70,6 @@ import com.linkedin.venice.controllerapi.UpdateStoragePersonaQueryParams; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionResponse; -import com.linkedin.venice.exceptions.ErrorType; import com.linkedin.venice.exceptions.InvalidVeniceSchemaException; import com.linkedin.venice.exceptions.ResourceStillExistsException; import com.linkedin.venice.exceptions.VeniceException; @@ -104,12 +103,8 @@ import com.linkedin.venice.ingestion.control.RealTimeTopicSwitcher; import com.linkedin.venice.kafka.protocol.enums.ControlMessageType; import com.linkedin.venice.meta.BackupStrategy; -import com.linkedin.venice.meta.BufferReplayPolicy; import com.linkedin.venice.meta.DataReplicationPolicy; -import com.linkedin.venice.meta.ETLStoreConfig; -import com.linkedin.venice.meta.ETLStoreConfigImpl; import com.linkedin.venice.meta.HybridStoreConfig; -import com.linkedin.venice.meta.HybridStoreConfigImpl; import com.linkedin.venice.meta.Instance; import com.linkedin.venice.meta.InstanceStatus; import com.linkedin.venice.meta.LiveClusterConfig; @@ -426,6 +421,10 @@ public class VeniceHelixAdmin implements Admin, StoreCleaner { private final Lazy emptyPushZSTDDictionary; + private final Optional externalSupersetSchemaGenerator; + + private final SupersetSchemaGenerator defaultSupersetSchemaGenerator = new DefaultSupersetSchemaGenerator(); + public VeniceHelixAdmin( VeniceControllerMultiClusterConfig multiClusterConfigs, MetricsRepository metricsRepository, @@ -440,6 +439,7 @@ public VeniceHelixAdmin( Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), pubSubTopicRepository, pubSubClientsFactory, Collections.EMPTY_LIST); @@ -454,6 +454,7 @@ public VeniceHelixAdmin( Optional sslConfig, Optional accessController, Optional icProvider, + Optional externalSupersetSchemaGenerator, PubSubTopicRepository pubSubTopicRepository, PubSubClientsFactory pubSubClientsFactory, List additionalInitRoutines) { @@ -473,6 +474,7 @@ public VeniceHelixAdmin( this.minNumberOfStoreVersionsToPreserve = multiClusterConfigs.getMinNumberOfStoreVersionsToPreserve(); this.d2Client = d2Client; + this.externalSupersetSchemaGenerator = externalSupersetSchemaGenerator; this.pubSubTopicRepository = pubSubTopicRepository; if (sslEnabled) { @@ -613,8 +615,7 @@ public VeniceHelixAdmin( multiClusterConfigs, this, Optional.of(AvroProtocolDefinition.METADATA_SYSTEM_SCHEMA_STORE_KEY.getCurrentProtocolVersionSchema()), - Optional.of(VeniceSystemStoreUtils.DEFAULT_USER_SYSTEM_STORE_UPDATE_QUERY_PARAMS), - true)); + Optional.of(VeniceSystemStoreUtils.DEFAULT_USER_SYSTEM_STORE_UPDATE_QUERY_PARAMS))); } if (multiClusterConfigs.isZkSharedDaVinciPushStatusSystemSchemaStoreAutoCreationEnabled()) { // Add routine to create zk shared da vinci push status system store @@ -624,8 +625,7 @@ public VeniceHelixAdmin( multiClusterConfigs, this, Optional.of(AvroProtocolDefinition.PUSH_STATUS_SYSTEM_SCHEMA_STORE_KEY.getCurrentProtocolVersionSchema()), - Optional.of(VeniceSystemStoreUtils.DEFAULT_USER_SYSTEM_STORE_UPDATE_QUERY_PARAMS), - true)); + Optional.of(VeniceSystemStoreUtils.DEFAULT_USER_SYSTEM_STORE_UPDATE_QUERY_PARAMS))); } initRoutines.addAll(additionalInitRoutines); @@ -974,7 +974,11 @@ public void createStore( storeName, largestUsedStoreVersion); } - configureNewStore(newStore, config, largestUsedStoreVersion); + configureNewStore( + newStore, + getMultiClusterConfigs().getControllerConfig(clusterName), + config, + largestUsedStoreVersion); storeRepo.addStore(newStore); // Create global config for that store. @@ -996,9 +1000,12 @@ public void createStore( } } - private void configureNewStore(Store newStore, VeniceControllerClusterConfig config, int largestUsedVersionNumber) { - newStore.setNativeReplicationEnabled(config.isNativeReplicationEnabledAsDefaultForBatchOnly()); - newStore.setActiveActiveReplicationEnabled(config.isActiveActiveReplicationEnabledAsDefaultForBatchOnly()); + private void configureNewStore( + Store newStore, + VeniceControllerConfig controllerConfig, + VeniceControllerClusterConfig config, + int largestUsedVersionNumber) { + newStore.setNativeReplicationEnabled(controllerConfig.isMultiRegion()); /** * Initialize default NR source fabric base on default config for different store types. @@ -1324,7 +1331,7 @@ public void writeEndOfPush(String clusterName, String storeName, int versionNumb /** * Test if a store is allowed for a batch push. * @param storeName name of a store. - * @return ture is the store is a participant system store or {@linkplain ConfigKeys#CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD CONTROLLER_ENABLE_BATCH_PUSH_FROM_ADMIN_IN_CHILD} is enabled. + * @return {@code true} is the store is a participant system store or if Venice is running in multi-region mode */ @Override public boolean whetherEnableBatchPushFromAdmin(String storeName) { @@ -1332,8 +1339,7 @@ public boolean whetherEnableBatchPushFromAdmin(String storeName) { * Allow (empty) push to participant system store from child controller directly since participant stores are * independent in different fabrics (different data). */ - return VeniceSystemStoreUtils.isParticipantStore(storeName) - || multiClusterConfigs.isEnableBatchPushFromAdminInChildController(); + return VeniceSystemStoreUtils.isParticipantStore(storeName) || !multiClusterConfigs.isMultiRegion(); } /** @@ -2082,14 +2088,7 @@ public Version addVersionOnly( // Apply cluster-level native replication configs VeniceControllerClusterConfig clusterConfig = resources.getConfig(); - boolean nativeReplicationEnabled = version.isNativeReplicationEnabled(); - - if (store.isHybrid()) { - nativeReplicationEnabled |= clusterConfig.isNativeReplicationEnabledForHybrid(); - } else { - nativeReplicationEnabled |= clusterConfig.isNativeReplicationEnabledForBatchOnly(); - } - version.setNativeReplicationEnabled(nativeReplicationEnabled); + version.setNativeReplicationEnabled(clusterConfig.isMultiRegion()); if (version.isNativeReplicationEnabled()) { if (remoteKafkaBootstrapServers != null) { @@ -2552,13 +2551,7 @@ private Pair addVersion( } // Apply cluster-level native replication configs - boolean nativeReplicationEnabled = version.isNativeReplicationEnabled(); - if (store.isHybrid()) { - nativeReplicationEnabled |= clusterConfig.isNativeReplicationEnabledForHybrid(); - } else { - nativeReplicationEnabled |= clusterConfig.isNativeReplicationEnabledForBatchOnly(); - } - version.setNativeReplicationEnabled(nativeReplicationEnabled); + version.setNativeReplicationEnabled(store.isNativeReplicationEnabled()); // Check whether native replication is enabled if (version.isNativeReplicationEnabled()) { @@ -2977,7 +2970,7 @@ public String getRealTimeTopic(String clusterName, String storeName) { if (store == null) { throwStoreDoesNotExist(clusterName, storeName); } - if (!store.isHybrid() && !store.isWriteComputationEnabled() && !store.isSystemStore()) { + if (!store.isHybrid() && !store.isSystemStore()) { logAndThrow("Store " + storeName + " is not hybrid, refusing to return a realtime topic"); } Version version = store.getVersion(store.getLargestUsedVersionNumber()); @@ -3866,20 +3859,8 @@ Pair waitVersion(String clusterName, String storeName, int versi */ @Override public void setStoreCurrentVersion(String clusterName, String storeName, int versionNumber) { - this.setStoreCurrentVersion(clusterName, storeName, versionNumber, false); - } - - /** - * In most cases, parent region should not update the current version. This is only allowed via an update-store call - * where the region filter list only contains one region, which is the region of the parent controller - */ - private void setStoreCurrentVersion( - String clusterName, - String storeName, - int versionNumber, - boolean allowedInParent) { - if (isParent() && !allowedInParent) { - // Parent colo should not update the current version of a store unless explicitly asked to do so + if (isParent()) { + // Parent colo should not update the current version of a store LOGGER.info( "Skipping current version update for store: {} in cluster: {} because it is not allowed in the " + "parent region", @@ -3907,6 +3888,7 @@ private void setStoreCurrentVersion( } return store; }); + LOGGER.info("Set store: {} version to {} in cluster: {}", storeName, versionNumber, clusterName); } @Override @@ -3995,17 +3977,6 @@ public int getBackupVersionNumber(List versions, int currentVersion) { return NON_EXISTING_VERSION; } - /** - * Update the largest used version number of a specified store. - */ - @Override - public void setStoreLargestUsedVersion(String clusterName, String storeName, int versionNumber) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setLargestUsedVersionNumber(versionNumber); - return store; - }); - } - /** * Update the owner of a specified store. */ @@ -4023,63 +3994,13 @@ public void setStoreOwner(String clusterName, String storeName, String owner) { */ @Override public void setStorePartitionCount(String clusterName, String storeName, int partitionCount) { - VeniceControllerClusterConfig clusterConfig = getHelixVeniceClusterResources(clusterName).getConfig(); storeMetadataUpdate(clusterName, storeName, store -> { - preCheckStorePartitionCountUpdate(clusterName, store, partitionCount); - // Do not update the partitionCount on the store.version as version config is immutable. The - // version.getPartitionCount() - // is read only in getRealTimeTopic and createInternalStore creation, so modifying currentVersion should not have - // any effect. - if (partitionCount != 0) { - store.setPartitionCount(partitionCount); - } else { - store.setPartitionCount(clusterConfig.getMinNumberOfPartitions()); - } - + UpdateStoreUtils.validateStorePartitionCountUpdate(this, multiClusterConfigs, clusterName, store, partitionCount); + store.setPartitionCount(partitionCount); return store; }); } - void preCheckStorePartitionCountUpdate(String clusterName, Store store, int newPartitionCount) { - String errorMessagePrefix = "Store update error for " + store.getName() + " in cluster: " + clusterName + ": "; - VeniceControllerClusterConfig clusterConfig = getHelixVeniceClusterResources(clusterName).getConfig(); - if (store.isHybrid() && store.getPartitionCount() != newPartitionCount) { - // Allow the update if partition count is not configured and the new partition count matches RT partition count - if (store.getPartitionCount() == 0) { - TopicManager topicManager; - if (isParent()) { - // RT might not exist in parent colo. Get RT partition count from a child colo. - String childDatacenter = Utils.parseCommaSeparatedStringToList(clusterConfig.getChildDatacenters()).get(0); - topicManager = getTopicManager(multiClusterConfigs.getChildDataCenterKafkaUrlMap().get(childDatacenter)); - } else { - topicManager = getTopicManager(); - } - PubSubTopic realTimeTopic = pubSubTopicRepository.getTopic(Version.composeRealTimeTopic(store.getName())); - if (topicManager.containsTopic(realTimeTopic) - && topicManager.getPartitionCount(realTimeTopic) == newPartitionCount) { - LOGGER.info("Allow updating store " + store.getName() + " partition count to " + newPartitionCount); - return; - } - } - String errorMessage = errorMessagePrefix + "Cannot change partition count for this hybrid store"; - LOGGER.error(errorMessage); - throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); - } - - int maxPartitionNum = clusterConfig.getMaxNumberOfPartitions(); - if (newPartitionCount > maxPartitionNum) { - String errorMessage = - errorMessagePrefix + "Partition count: " + newPartitionCount + " should be less than max: " + maxPartitionNum; - LOGGER.error(errorMessage); - throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); - } - if (newPartitionCount < 0) { - String errorMessage = errorMessagePrefix + "Partition count: " + newPartitionCount + " should NOT be negative"; - LOGGER.error(errorMessage); - throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); - } - } - void setStorePartitionerConfig(String clusterName, String storeName, PartitionerConfig partitionerConfig) { storeMetadataUpdate(clusterName, storeName, store -> { // Only amplification factor is allowed to be changed if the store is a hybrid store. @@ -4133,32 +4054,6 @@ public void setStoreReadWriteability(String clusterName, String storeName, boole }); } - /** - * We will not expose this interface to Spark server. Updating quota can only be done by #updateStore - * TODO: remove all store attribute setters. - */ - private void setStoreStorageQuota(String clusterName, String storeName, long storageQuotaInByte) { - storeMetadataUpdate(clusterName, storeName, store -> { - if (storageQuotaInByte < 0 && storageQuotaInByte != Store.UNLIMITED_STORAGE_QUOTA) { - throw new VeniceException("storage quota can not be less than 0"); - } - store.setStorageQuotaInByte(storageQuotaInByte); - - return store; - }); - } - - private void setStoreReadQuota(String clusterName, String storeName, long readQuotaInCU) { - storeMetadataUpdate(clusterName, storeName, store -> { - if (readQuotaInCU < 0) { - throw new VeniceException("read quota can not be less than 0"); - } - store.setReadQuotaInCU(readQuotaInCU); - - return store; - }); - } - void setAccessControl(String clusterName, String storeName, boolean accessControlled) { storeMetadataUpdate(clusterName, storeName, store -> { store.setAccessControlled(accessControlled); @@ -4226,190 +4121,6 @@ public void deleteValueSchemas(String clusterName, String storeName, Set { - store.setCompressionStrategy(compressionStrategy); - - return store; - }); - } - - private void setClientDecompressionEnabled(String clusterName, String storeName, boolean clientDecompressionEnabled) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setClientDecompressionEnabled(clientDecompressionEnabled); - return store; - }); - } - - private void setChunkingEnabled(String clusterName, String storeName, boolean chunkingEnabled) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setChunkingEnabled(chunkingEnabled); - return store; - }); - } - - private void setRmdChunkingEnabled(String clusterName, String storeName, boolean rmdChunkingEnabled) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setRmdChunkingEnabled(rmdChunkingEnabled); - return store; - }); - } - - void setIncrementalPushEnabled(String clusterName, String storeName, boolean incrementalPushEnabled) { - storeMetadataUpdate(clusterName, storeName, store -> { - VeniceControllerClusterConfig config = getHelixVeniceClusterResources(clusterName).getConfig(); - if (incrementalPushEnabled || store.isHybrid()) { - // Enabling incremental push - store.setNativeReplicationEnabled(config.isNativeReplicationEnabledAsDefaultForHybrid()); - store.setNativeReplicationSourceFabric(config.getNativeReplicationSourceFabricAsDefaultForHybrid()); - store.setActiveActiveReplicationEnabled( - store.isActiveActiveReplicationEnabled() - || (config.isActiveActiveReplicationEnabledAsDefaultForHybrid() && !store.isSystemStore())); - } else { - // Disabling incremental push - // This is only possible when hybrid settings are set to null before turning of incremental push for the store. - store.setNativeReplicationEnabled(config.isNativeReplicationEnabledAsDefaultForBatchOnly()); - store.setNativeReplicationSourceFabric(config.getNativeReplicationSourceFabricAsDefaultForBatchOnly()); - store.setActiveActiveReplicationEnabled( - store.isActiveActiveReplicationEnabled() || config.isActiveActiveReplicationEnabledAsDefaultForBatchOnly()); - } - store.setIncrementalPushEnabled(incrementalPushEnabled); - - return store; - }); - } - - private void setReplicationFactor(String clusterName, String storeName, int replicaFactor) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setReplicationFactor(replicaFactor); - - return store; - }); - } - - private void setBatchGetLimit(String clusterName, String storeName, int batchGetLimit) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setBatchGetLimit(batchGetLimit); - - return store; - }); - } - - private void setNumVersionsToPreserve(String clusterName, String storeName, int numVersionsToPreserve) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setNumVersionsToPreserve(numVersionsToPreserve); - - return store; - }); - } - - private void setStoreMigration(String clusterName, String storeName, boolean migrating) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setMigrating(migrating); - return store; - }); - } - - private void setMigrationDuplicateStore(String clusterName, String storeName, boolean migrationDuplicateStore) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setMigrationDuplicateStore(migrationDuplicateStore); - return store; - }); - } - - private void setWriteComputationEnabled(String clusterName, String storeName, boolean writeComputationEnabled) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setWriteComputationEnabled(writeComputationEnabled); - return store; - }); - } - - void setReplicationMetadataVersionID(String clusterName, String storeName, int rmdVersion) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setRmdVersion(rmdVersion); - return store; - }); - } - - private void setReadComputationEnabled(String clusterName, String storeName, boolean computationEnabled) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setReadComputationEnabled(computationEnabled); - return store; - }); - } - - void setBootstrapToOnlineTimeoutInHours(String clusterName, String storeName, int bootstrapToOnlineTimeoutInHours) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setBootstrapToOnlineTimeoutInHours(bootstrapToOnlineTimeoutInHours); - return store; - }); - } - - private void setNativeReplicationEnabled(String clusterName, String storeName, boolean nativeReplicationEnabled) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setNativeReplicationEnabled(nativeReplicationEnabled); - return store; - }); - } - - private void setPushStreamSourceAddress(String clusterName, String storeName, String pushStreamSourceAddress) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setPushStreamSourceAddress(pushStreamSourceAddress); - return store; - }); - } - - private void addStoreViews(String clusterName, String storeName, Map viewConfigMap) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setViewConfigs(StoreViewUtils.convertStringMapViewToViewConfigMap(viewConfigMap)); - return store; - }); - } - - private void setBackupStrategy(String clusterName, String storeName, BackupStrategy backupStrategy) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setBackupStrategy(backupStrategy); - return store; - }); - } - - private void setAutoSchemaRegisterPushJobEnabled( - String clusterName, - String storeName, - boolean autoSchemaRegisterPushJobEnabled) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setSchemaAutoRegisterFromPushJobEnabled(autoSchemaRegisterPushJobEnabled); - return store; - }); - } - - void setHybridStoreDiskQuotaEnabled(String clusterName, String storeName, boolean hybridStoreDiskQuotaEnabled) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setHybridStoreDiskQuotaEnabled(hybridStoreDiskQuotaEnabled); - return store; - }); - } - - private void setBackupVersionRetentionMs(String clusterName, String storeName, long backupVersionRetentionMs) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setBackupVersionRetentionMs(backupVersionRetentionMs); - return store; - }); - } - - private void setNativeReplicationSourceFabric( - String clusterName, - String storeName, - String nativeReplicationSourceFabric) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setNativeReplicationSourceFabric(nativeReplicationSourceFabric); - return store; - }); - } - void setActiveActiveReplicationEnabled(String clusterName, String storeName, boolean activeActiveReplicationEnabled) { storeMetadataUpdate(clusterName, storeName, store -> { store.setActiveActiveReplicationEnabled(activeActiveReplicationEnabled); @@ -4417,40 +4128,6 @@ void setActiveActiveReplicationEnabled(String clusterName, String storeName, boo }); } - private void disableMetaSystemStore(String clusterName, String storeName) { - LOGGER.info("Disabling meta system store for store: {} of cluster: {}", storeName, clusterName); - storeMetadataUpdate(clusterName, storeName, store -> { - store.setStoreMetaSystemStoreEnabled(false); - store.setStoreMetadataSystemStoreEnabled(false); - return store; - }); - } - - private void disableDavinciPushStatusStore(String clusterName, String storeName) { - LOGGER.info("Disabling davinci push status store for store: {} of cluster: {}", storeName, clusterName); - storeMetadataUpdate(clusterName, storeName, store -> { - store.setDaVinciPushStatusStoreEnabled(false); - return store; - }); - } - - private void setLatestSupersetSchemaId(String clusterName, String storeName, int latestSupersetSchemaId) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setLatestSuperSetValueSchemaId(latestSupersetSchemaId); - return store; - }); - } - - private void setStorageNodeReadQuotaEnabled( - String clusterName, - String storeName, - boolean storageNodeReadQuotaEnabled) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setStorageNodeReadQuotaEnabled(storageNodeReadQuotaEnabled); - return store; - }); - } - /** * TODO: some logics are in parent controller {@link VeniceParentHelixAdmin} #updateStore and * some are in the child controller here. Need to unify them in the future. @@ -4464,6 +4141,15 @@ public void updateStore(String clusterName, String storeName, UpdateStoreQueryPa } } + @Override + public SupersetSchemaGenerator getSupersetSchemaGenerator(String clusterName) { + if (externalSupersetSchemaGenerator.isPresent() + && getMultiClusterConfigs().getControllerConfig(clusterName).isExternalSupersetSchemaGenerationEnabled()) { + return externalSupersetSchemaGenerator.get(); + } + return defaultSupersetSchemaGenerator; + } + /** * Update the {@linkplain LiveClusterConfig} at runtime for a specified cluster. * @param clusterName name of the Venice cluster. @@ -4499,373 +4185,54 @@ public void updateClusterConfig(String clusterName, UpdateClusterConfigQueryPara } private void internalUpdateStore(String clusterName, String storeName, UpdateStoreQueryParams params) { - // There are certain configs that are only allowed to be updated in child regions. We might still want the ability - // to update such configs in the parent region via the Admin tool for operational reasons. So, we allow such updates - // if the regions filter only specifies one region, which is the parent region. - boolean onlyParentRegionFilter = false; - - // Check whether the command affects this region. - if (params.getRegionsFilter().isPresent()) { - Set regionsFilter = parseRegionsFilterList(params.getRegionsFilter().get()); - if (!regionsFilter.contains(multiClusterConfigs.getRegionName())) { - LOGGER.info( - "UpdateStore command will be skipped for store: {} in cluster: {}, because the region filter is {}" - + " which doesn't include the current region: {}", - storeName, - clusterName, - regionsFilter, - multiClusterConfigs.getRegionName()); - return; - } - - if (isParent() && regionsFilter.size() == 1) { - onlyParentRegionFilter = true; - } + UpdateStoreWrapper updatedStoreWrapper = UpdateStoreUtils.getStoreUpdate(this, clusterName, storeName, params); + if (updatedStoreWrapper == null) { + return; } - Store originalStore = getStore(clusterName, storeName); - if (originalStore == null) { - throw new VeniceNoStoreException(storeName, clusterName); - } - if (originalStore.isHybrid()) { - // If this is a hybrid store, always try to disable compaction if RT topic exists. - try { - PubSubTopic rtTopic = pubSubTopicRepository.getTopic(Version.composeRealTimeTopic(storeName)); - getTopicManager().updateTopicCompactionPolicy(rtTopic, false); - } catch (PubSubTopicDoesNotExistException e) { - LOGGER.error("Could not find realtime topic for hybrid store {}", storeName); - } - } + Store originalStore = updatedStoreWrapper.originalStore; + Store updatedStore = updatedStoreWrapper.updatedStore; - Optional owner = params.getOwner(); - Optional readability = params.getEnableReads(); - Optional writeability = params.getEnableWrites(); - Optional partitionCount = params.getPartitionCount(); - Optional partitionerClass = params.getPartitionerClass(); - Optional> partitionerParams = params.getPartitionerParams(); - Optional amplificationFactor = params.getAmplificationFactor(); - Optional storageQuotaInByte = params.getStorageQuotaInByte(); - Optional readQuotaInCU = params.getReadQuotaInCU(); - Optional currentVersion = params.getCurrentVersion(); - Optional largestUsedVersionNumber = params.getLargestUsedVersionNumber(); - Optional hybridRewindSeconds = params.getHybridRewindSeconds(); - Optional hybridOffsetLagThreshold = params.getHybridOffsetLagThreshold(); - Optional hybridTimeLagThreshold = params.getHybridTimeLagThreshold(); - Optional hybridDataReplicationPolicy = params.getHybridDataReplicationPolicy(); - Optional hybridBufferReplayPolicy = params.getHybridBufferReplayPolicy(); - Optional accessControlled = params.getAccessControlled(); - Optional compressionStrategy = params.getCompressionStrategy(); - Optional clientDecompressionEnabled = params.getClientDecompressionEnabled(); - Optional chunkingEnabled = params.getChunkingEnabled(); - Optional rmdChunkingEnabled = params.getRmdChunkingEnabled(); - Optional batchGetLimit = params.getBatchGetLimit(); - Optional numVersionsToPreserve = params.getNumVersionsToPreserve(); - Optional incrementalPushEnabled = params.getIncrementalPushEnabled(); - Optional storeMigration = params.getStoreMigration(); - Optional writeComputationEnabled = params.getWriteComputationEnabled(); - Optional replicationMetadataVersionID = params.getReplicationMetadataVersionID(); - Optional readComputationEnabled = params.getReadComputationEnabled(); - Optional bootstrapToOnlineTimeoutInHours = params.getBootstrapToOnlineTimeoutInHours(); - Optional backupStrategy = params.getBackupStrategy(); - Optional autoSchemaRegisterPushJobEnabled = params.getAutoSchemaRegisterPushJobEnabled(); - Optional hybridStoreDiskQuotaEnabled = params.getHybridStoreDiskQuotaEnabled(); - Optional regularVersionETLEnabled = params.getRegularVersionETLEnabled(); - Optional futureVersionETLEnabled = params.getFutureVersionETLEnabled(); - Optional etledUserProxyAccount = params.getETLedProxyUserAccount(); - Optional nativeReplicationEnabled = params.getNativeReplicationEnabled(); - Optional pushStreamSourceAddress = params.getPushStreamSourceAddress(); - Optional backupVersionRetentionMs = params.getBackupVersionRetentionMs(); - Optional replicationFactor = params.getReplicationFactor(); - Optional migrationDuplicateStore = params.getMigrationDuplicateStore(); - Optional nativeReplicationSourceFabric = params.getNativeReplicationSourceFabric(); - Optional activeActiveReplicationEnabled = params.getActiveActiveReplicationEnabled(); - Optional personaName = params.getStoragePersona(); - Optional> storeViews = params.getStoreViews(); - Optional latestSupersetSchemaId = params.getLatestSupersetSchemaId(); - Optional storageNodeReadQuotaEnabled = params.getStorageNodeReadQuotaEnabled(); - Optional minCompactionLagSeconds = params.getMinCompactionLagSeconds(); - Optional maxCompactionLagSeconds = params.getMaxCompactionLagSeconds(); - Optional maxRecordSizeBytes = params.getMaxRecordSizeBytes(); - Optional unusedSchemaDeletionEnabled = params.getUnusedSchemaDeletionEnabled(); - Optional blobTransferEnabled = params.getBlobTransferEnabled(); - - final Optional newHybridStoreConfig; - if (hybridRewindSeconds.isPresent() || hybridOffsetLagThreshold.isPresent() || hybridTimeLagThreshold.isPresent() - || hybridDataReplicationPolicy.isPresent() || hybridBufferReplayPolicy.isPresent()) { - HybridStoreConfig hybridConfig = mergeNewSettingsIntoOldHybridStoreConfig( - originalStore, - hybridRewindSeconds, - hybridOffsetLagThreshold, - hybridTimeLagThreshold, - hybridDataReplicationPolicy, - hybridBufferReplayPolicy); - newHybridStoreConfig = Optional.ofNullable(hybridConfig); - } else { - newHybridStoreConfig = Optional.empty(); + if (updatedStore == originalStore) { + return; } try { - if (owner.isPresent()) { - setStoreOwner(clusterName, storeName, owner.get()); - } - - if (readability.isPresent()) { - setStoreReadability(clusterName, storeName, readability.get()); - } - - if (writeability.isPresent()) { - setStoreWriteability(clusterName, storeName, writeability.get()); - } - - if (partitionCount.isPresent()) { - setStorePartitionCount(clusterName, storeName, partitionCount.get()); - } - - /** - * If either of these three fields is not present, we should use store's original value to construct correct - * updated partitioner config. - */ - if (partitionerClass.isPresent() || partitionerParams.isPresent() || amplificationFactor.isPresent()) { - PartitionerConfig updatedPartitionerConfig = mergeNewSettingsIntoOldPartitionerConfig( - originalStore, - partitionerClass, - partitionerParams, - amplificationFactor); - setStorePartitionerConfig(clusterName, storeName, updatedPartitionerConfig); - } - - if (storageQuotaInByte.isPresent()) { - setStoreStorageQuota(clusterName, storeName, storageQuotaInByte.get()); + if (originalStore.isHybrid()) { + // If this is a hybrid store, always try to disable compaction if RT topic exists. + try { + PubSubTopic rtTopic = pubSubTopicRepository.getTopic(Version.composeRealTimeTopic(storeName)); + getTopicManager().updateTopicCompactionPolicy(rtTopic, false); + } catch (PubSubTopicDoesNotExistException e) { + LOGGER.error("Could not find realtime topic for hybrid store {}", storeName); + } } - if (readQuotaInCU.isPresent()) { - HelixVeniceClusterResources resources = getHelixVeniceClusterResources(clusterName); - ZkRoutersClusterManager routersClusterManager = resources.getRoutersClusterManager(); - int routerCount = routersClusterManager.getLiveRoutersCount(); - VeniceControllerClusterConfig clusterConfig = getHelixVeniceClusterResources(clusterName).getConfig(); - int defaultReadQuotaPerRouter = clusterConfig.getDefaultReadQuotaPerRouter(); - - if (Math.max(defaultReadQuotaPerRouter, routerCount * defaultReadQuotaPerRouter) < readQuotaInCU.get()) { - throw new VeniceException( - "Cannot update read quota for store " + storeName + " in cluster " + clusterName + ". Read quota " - + readQuotaInCU.get() + " requested is more than the cluster quota."); + if (updatedStore.isHybrid()) { + PubSubTopic rtTopic = pubSubTopicRepository.getTopic(Version.composeRealTimeTopic(storeName)); + if (getTopicManager().containsTopicAndAllPartitionsAreOnline(rtTopic)) { + // RT already exists, ensure the retention is correct + getTopicManager().updateTopicRetention( + rtTopic, + StoreUtils.getExpectedRetentionTimeInMs(updatedStore, updatedStore.getHybridStoreConfig())); } - setStoreReadQuota(clusterName, storeName, readQuotaInCU.get()); } - if (currentVersion.isPresent()) { - setStoreCurrentVersion(clusterName, storeName, currentVersion.get(), onlyParentRegionFilter); - } + // All validations are done. We are ready to perform the persist the update on Zk + storeMetadataUpdate(clusterName, storeName, store -> updatedStore); - if (largestUsedVersionNumber.isPresent()) { - setStoreLargestUsedVersion(clusterName, storeName, largestUsedVersionNumber.get()); + Optional personaName = params.getStoragePersona(); + if (personaName.isPresent()) { + StoragePersonaRepository repository = getHelixVeniceClusterResources(clusterName).getStoragePersonaRepository(); + repository.addStoresToPersona(personaName.get(), Collections.singletonList(storeName)); } - if (bootstrapToOnlineTimeoutInHours.isPresent()) { - setBootstrapToOnlineTimeoutInHours(clusterName, storeName, bootstrapToOnlineTimeoutInHours.get()); + // Since we expect the parent controller to emit the actions to the Admin channel where necessary, we need to run + // it within the context of VeniceParentHelixAdmin. So, here, we only run it for the child controller if it is + // running in a single-region mode. + if (!isParent() && isPrimary()) { + UpdateStoreUtils.handlePostUpdateActions(this, clusterName, storeName); } - - VeniceControllerClusterConfig clusterConfig = getHelixVeniceClusterResources(clusterName).getConfig(); - if (newHybridStoreConfig.isPresent()) { - // To fix the final variable problem in the lambda expression - final HybridStoreConfig finalHybridConfig = newHybridStoreConfig.get(); - storeMetadataUpdate(clusterName, storeName, store -> { - if (!isHybrid(finalHybridConfig)) { - /** - * If all the hybrid config values are negative, it indicates that the store is being set back to batch-only store. - * We cannot remove the RT topic immediately because with NR and AA, existing current version is - * still consuming the RT topic. - */ - store.setHybridStoreConfig(null); - store.setIncrementalPushEnabled(false); - // Enable/disable native replication for batch-only stores if the cluster level config for new batch - // stores is on - store.setNativeReplicationEnabled(clusterConfig.isNativeReplicationEnabledAsDefaultForBatchOnly()); - store.setNativeReplicationSourceFabric( - clusterConfig.getNativeReplicationSourceFabricAsDefaultForBatchOnly()); - store.setActiveActiveReplicationEnabled( - store.isActiveActiveReplicationEnabled() - || clusterConfig.isActiveActiveReplicationEnabledAsDefaultForBatchOnly()); - } else { - // Batch-only store is being converted to hybrid store. - if (!store.isHybrid()) { - /* - * Enable/disable native replication for hybrid stores if the cluster level config - * for new hybrid stores is on - */ - store.setNativeReplicationEnabled(clusterConfig.isNativeReplicationEnabledAsDefaultForHybrid()); - store - .setNativeReplicationSourceFabric(clusterConfig.getNativeReplicationSourceFabricAsDefaultForHybrid()); - /* - * Enable/disable active-active replication for user hybrid stores if the cluster level config - * for new hybrid stores is on - */ - store.setActiveActiveReplicationEnabled( - store.isActiveActiveReplicationEnabled() - || (clusterConfig.isActiveActiveReplicationEnabledAsDefaultForHybrid() - && !store.isSystemStore())); - } - store.setHybridStoreConfig(finalHybridConfig); - PubSubTopic rtTopic = pubSubTopicRepository.getTopic(Version.composeRealTimeTopic(storeName)); - if (getTopicManager().containsTopicAndAllPartitionsAreOnline(rtTopic)) { - // RT already exists, ensure the retention is correct - getTopicManager() - .updateTopicRetention(rtTopic, StoreUtils.getExpectedRetentionTimeInMs(store, finalHybridConfig)); - } - } - return store; - }); - } - - if (accessControlled.isPresent()) { - setAccessControl(clusterName, storeName, accessControlled.get()); - } - - if (compressionStrategy.isPresent()) { - setStoreCompressionStrategy(clusterName, storeName, compressionStrategy.get()); - } - - if (clientDecompressionEnabled.isPresent()) { - setClientDecompressionEnabled(clusterName, storeName, clientDecompressionEnabled.get()); - } - - if (chunkingEnabled.isPresent()) { - setChunkingEnabled(clusterName, storeName, chunkingEnabled.get()); - } - - if (rmdChunkingEnabled.isPresent()) { - setRmdChunkingEnabled(clusterName, storeName, rmdChunkingEnabled.get()); - } - - if (batchGetLimit.isPresent()) { - setBatchGetLimit(clusterName, storeName, batchGetLimit.get()); - } - - if (numVersionsToPreserve.isPresent()) { - setNumVersionsToPreserve(clusterName, storeName, numVersionsToPreserve.get()); - } - - if (incrementalPushEnabled.isPresent()) { - if (incrementalPushEnabled.get()) { - enableHybridModeOrUpdateSettings(clusterName, storeName); - } - setIncrementalPushEnabled(clusterName, storeName, incrementalPushEnabled.get()); - } - - if (replicationFactor.isPresent()) { - setReplicationFactor(clusterName, storeName, replicationFactor.get()); - } - - if (storeMigration.isPresent()) { - setStoreMigration(clusterName, storeName, storeMigration.get()); - } - - if (migrationDuplicateStore.isPresent()) { - setMigrationDuplicateStore(clusterName, storeName, migrationDuplicateStore.get()); - } - - if (writeComputationEnabled.isPresent()) { - setWriteComputationEnabled(clusterName, storeName, writeComputationEnabled.get()); - } - - if (replicationMetadataVersionID.isPresent()) { - setReplicationMetadataVersionID(clusterName, storeName, replicationMetadataVersionID.get()); - } - - if (readComputationEnabled.isPresent()) { - setReadComputationEnabled(clusterName, storeName, readComputationEnabled.get()); - } - - if (nativeReplicationEnabled.isPresent()) { - setNativeReplicationEnabled(clusterName, storeName, nativeReplicationEnabled.get()); - } - - if (activeActiveReplicationEnabled.isPresent()) { - setActiveActiveReplicationEnabled(clusterName, storeName, activeActiveReplicationEnabled.get()); - } - - if (pushStreamSourceAddress.isPresent()) { - setPushStreamSourceAddress(clusterName, storeName, pushStreamSourceAddress.get()); - } - - if (backupStrategy.isPresent()) { - setBackupStrategy(clusterName, storeName, backupStrategy.get()); - } - - autoSchemaRegisterPushJobEnabled - .ifPresent(value -> setAutoSchemaRegisterPushJobEnabled(clusterName, storeName, value)); - hybridStoreDiskQuotaEnabled.ifPresent(value -> setHybridStoreDiskQuotaEnabled(clusterName, storeName, value)); - if (regularVersionETLEnabled.isPresent() || futureVersionETLEnabled.isPresent() - || etledUserProxyAccount.isPresent()) { - ETLStoreConfig etlStoreConfig = new ETLStoreConfigImpl( - etledUserProxyAccount.orElse(originalStore.getEtlStoreConfig().getEtledUserProxyAccount()), - regularVersionETLEnabled.orElse(originalStore.getEtlStoreConfig().isRegularVersionETLEnabled()), - futureVersionETLEnabled.orElse(originalStore.getEtlStoreConfig().isFutureVersionETLEnabled())); - storeMetadataUpdate(clusterName, storeName, store -> { - store.setEtlStoreConfig(etlStoreConfig); - return store; - }); - } - if (backupVersionRetentionMs.isPresent()) { - setBackupVersionRetentionMs(clusterName, storeName, backupVersionRetentionMs.get()); - } - - if (nativeReplicationSourceFabric.isPresent()) { - setNativeReplicationSourceFabric(clusterName, storeName, nativeReplicationSourceFabric.get()); - } - - if (params.disableMetaStore().isPresent() && params.disableMetaStore().get()) { - disableMetaSystemStore(clusterName, storeName); - } - - if (params.disableDavinciPushStatusStore().isPresent() && params.disableDavinciPushStatusStore().get()) { - disableDavinciPushStatusStore(clusterName, storeName); - } - - if (personaName.isPresent()) { - StoragePersonaRepository repository = getHelixVeniceClusterResources(clusterName).getStoragePersonaRepository(); - repository.addStoresToPersona(personaName.get(), Arrays.asList(storeName)); - } - - if (storeViews.isPresent()) { - addStoreViews(clusterName, storeName, storeViews.get()); - } - - if (latestSupersetSchemaId.isPresent()) { - setLatestSupersetSchemaId(clusterName, storeName, latestSupersetSchemaId.get()); - } - - if (minCompactionLagSeconds.isPresent()) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setMinCompactionLagSeconds(minCompactionLagSeconds.get()); - return store; - }); - } - if (maxCompactionLagSeconds.isPresent()) { - storeMetadataUpdate(clusterName, storeName, store -> { - store.setMaxCompactionLagSeconds(maxCompactionLagSeconds.get()); - return store; - }); - } - - maxRecordSizeBytes.ifPresent(aInt -> storeMetadataUpdate(clusterName, storeName, store -> { - store.setMaxRecordSizeBytes(aInt); - return store; - })); - - unusedSchemaDeletionEnabled.ifPresent(aBoolean -> storeMetadataUpdate(clusterName, storeName, store -> { - store.setUnusedSchemaDeletionEnabled(aBoolean); - return store; - })); - - storageNodeReadQuotaEnabled - .ifPresent(aBoolean -> setStorageNodeReadQuotaEnabled(clusterName, storeName, aBoolean)); - - blobTransferEnabled.ifPresent(aBoolean -> storeMetadataUpdate(clusterName, storeName, store -> { - store.setBlobTransferEnabled(aBoolean); - return store; - })); - LOGGER.info("Finished updating store: {} in cluster: {}", storeName, clusterName); } catch (VeniceException e) { LOGGER.error( @@ -4876,8 +4243,7 @@ private void internalUpdateStore(String clusterName, String storeName, UpdateSto // rollback to original store storeMetadataUpdate(clusterName, storeName, store -> originalStore); PubSubTopic rtTopic = pubSubTopicRepository.getTopic(Version.composeRealTimeTopic(storeName)); - if (originalStore.isHybrid() && newHybridStoreConfig.isPresent() - && getTopicManager().containsTopicAndAllPartitionsAreOnline(rtTopic)) { + if (originalStore.isHybrid() && getTopicManager().containsTopicAndAllPartitionsAreOnline(rtTopic)) { // Ensure the topic retention is rolled back too getTopicManager().updateTopicRetention( rtTopic, @@ -4892,35 +4258,6 @@ && getTopicManager().containsTopicAndAllPartitionsAreOnline(rtTopic)) { } } - /** - * Enabling hybrid mode for incremental push store is moved into - * {@link VeniceParentHelixAdmin#updateStore(String, String, UpdateStoreQueryParams)} - * TODO: Remove the method and its usage after the deployment of parent controller updateStore change. - */ - private void enableHybridModeOrUpdateSettings(String clusterName, String storeName) { - storeMetadataUpdate(clusterName, storeName, store -> { - HybridStoreConfig hybridStoreConfig = store.getHybridStoreConfig(); - if (hybridStoreConfig == null) { - store.setHybridStoreConfig( - new HybridStoreConfigImpl( - DEFAULT_REWIND_TIME_IN_SECONDS, - DEFAULT_HYBRID_OFFSET_LAG_THRESHOLD, - DEFAULT_HYBRID_TIME_LAG_THRESHOLD, - DataReplicationPolicy.NONE, - null)); - } else if (hybridStoreConfig.getDataReplicationPolicy() == null) { - store.setHybridStoreConfig( - new HybridStoreConfigImpl( - hybridStoreConfig.getRewindTimeInSeconds(), - hybridStoreConfig.getOffsetLagThresholdToGoOnline(), - hybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds(), - DataReplicationPolicy.NONE, - hybridStoreConfig.getBufferReplayPolicy())); - } - return store; - }); - } - /** * This method is invoked in parent controllers for store migration. */ @@ -4962,132 +4299,6 @@ public void replicateUpdateStore(String clusterName, String storeName, UpdateSto } } - /** - * Used by both the {@link VeniceHelixAdmin} and the {@link VeniceParentHelixAdmin} - * - * @param oldStore Existing Store that is the source for updates. This object will not be modified by this method. - * @param hybridRewindSeconds Optional is present if the returned object should include a new rewind time - * @param hybridOffsetLagThreshold Optional is present if the returned object should include a new offset lag threshold - * @return null if oldStore has no hybrid configs and optionals are not present, - * otherwise a fully specified {@link HybridStoreConfig} - */ - protected static HybridStoreConfig mergeNewSettingsIntoOldHybridStoreConfig( - Store oldStore, - Optional hybridRewindSeconds, - Optional hybridOffsetLagThreshold, - Optional hybridTimeLagThreshold, - Optional hybridDataReplicationPolicy, - Optional bufferReplayPolicy) { - if (!hybridRewindSeconds.isPresent() && !hybridOffsetLagThreshold.isPresent() && !oldStore.isHybrid()) { - return null; // For the nullable union in the avro record - } - HybridStoreConfig mergedHybridStoreConfig; - if (oldStore.isHybrid()) { // for an existing hybrid store, just replace any specified values - HybridStoreConfig oldHybridConfig = oldStore.getHybridStoreConfig().clone(); - mergedHybridStoreConfig = new HybridStoreConfigImpl( - hybridRewindSeconds.isPresent() ? hybridRewindSeconds.get() : oldHybridConfig.getRewindTimeInSeconds(), - hybridOffsetLagThreshold.isPresent() - ? hybridOffsetLagThreshold.get() - : oldHybridConfig.getOffsetLagThresholdToGoOnline(), - hybridTimeLagThreshold.isPresent() - ? hybridTimeLagThreshold.get() - : oldHybridConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds(), - hybridDataReplicationPolicy.isPresent() - ? hybridDataReplicationPolicy.get() - : oldHybridConfig.getDataReplicationPolicy(), - bufferReplayPolicy.isPresent() ? bufferReplayPolicy.get() : oldHybridConfig.getBufferReplayPolicy()); - } else { - // switching a non-hybrid store to hybrid; must specify: - // 1. rewind time - // 2. either offset lag threshold or time lag threshold, or both - if (!(hybridRewindSeconds.isPresent() - && (hybridOffsetLagThreshold.isPresent() || hybridTimeLagThreshold.isPresent()))) { - throw new VeniceException( - oldStore.getName() + " was not a hybrid store. In order to make it a hybrid store both " - + " rewind time in seconds and offset or time lag threshold must be specified"); - } - mergedHybridStoreConfig = new HybridStoreConfigImpl( - hybridRewindSeconds.get(), - // If not specified, offset/time lag threshold will be -1 and will not be used to determine whether - // a partition is ready to serve - hybridOffsetLagThreshold.orElse(DEFAULT_HYBRID_OFFSET_LAG_THRESHOLD), - hybridTimeLagThreshold.orElse(DEFAULT_HYBRID_TIME_LAG_THRESHOLD), - hybridDataReplicationPolicy.orElse(DataReplicationPolicy.NON_AGGREGATE), - bufferReplayPolicy.orElse(BufferReplayPolicy.REWIND_FROM_EOP)); - } - if (mergedHybridStoreConfig.getRewindTimeInSeconds() > 0 - && mergedHybridStoreConfig.getOffsetLagThresholdToGoOnline() < 0 - && mergedHybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds() < 0) { - throw new VeniceException( - "Both offset lag threshold and time lag threshold are negative when setting hybrid" + " configs for store " - + oldStore.getName()); - } - return mergedHybridStoreConfig; - } - - static PartitionerConfig mergeNewSettingsIntoOldPartitionerConfig( - Store oldStore, - Optional partitionerClass, - Optional> partitionerParams, - Optional amplificationFactor) { - PartitionerConfig originalPartitionerConfig; - if (oldStore.getPartitionerConfig() == null) { - originalPartitionerConfig = new PartitionerConfigImpl(); - } else { - originalPartitionerConfig = oldStore.getPartitionerConfig(); - } - return new PartitionerConfigImpl( - partitionerClass.orElse(originalPartitionerConfig.getPartitionerClass()), - partitionerParams.orElse(originalPartitionerConfig.getPartitionerParams()), - amplificationFactor.orElse(originalPartitionerConfig.getAmplificationFactor())); - } - - static Map mergeNewViewConfigsIntoOldConfigs( - Store oldStore, - Map viewParameters) throws VeniceException { - // Merge the existing configs with the incoming configs. The new configs will override existing ones which share the - // same key. - Map oldViewConfigMap = oldStore.getViewConfigs(); - if (oldViewConfigMap == null) { - oldViewConfigMap = new HashMap<>(); - } - Map mergedConfigs = - StoreViewUtils.convertViewConfigMapToStoreViewRecordMap(oldViewConfigMap); - mergedConfigs.putAll(StoreViewUtils.convertStringMapViewToStoreViewConfigRecordMap(viewParameters)); - return mergedConfigs; - } - - static Map addNewViewConfigsIntoOldConfigs( - Store oldStore, - String viewClass, - ViewConfig viewConfig) throws VeniceException { - // Add new view config into the existing config map. The new configs will override existing ones which share the - // same key. - Map oldViewConfigMap = oldStore.getViewConfigs(); - if (oldViewConfigMap == null) { - oldViewConfigMap = new HashMap<>(); - } - Map mergedConfigs = - StoreViewUtils.convertViewConfigMapToStoreViewRecordMap(oldViewConfigMap); - - StoreViewConfigRecord newStoreViewConfigRecord = - StoreViewUtils.convertViewConfigToStoreViewConfigRecord(viewConfig); - mergedConfigs.put(viewClass, newStoreViewConfigRecord); - return mergedConfigs; - } - - static Map removeViewConfigFromStoreViewConfigMap(Store oldStore, String viewClass) - throws VeniceException { - Map oldViewConfigMap = oldStore.getViewConfigs(); - if (oldViewConfigMap == null) { - oldViewConfigMap = new HashMap<>(); - } - Map mergedConfigs = - StoreViewUtils.convertViewConfigMapToStoreViewRecordMap(oldViewConfigMap); - mergedConfigs.remove(viewClass); - return mergedConfigs; - } - /** * Update the store metadata by applying provided operation. * @param clusterName name of the cluster. @@ -5381,8 +4592,9 @@ public SchemaEntry addValueSchema( DirectionalSchemaCompatibilityType expectedCompatibilityType) { checkControllerLeadershipFor(clusterName); ReadWriteSchemaRepository schemaRepository = getHelixVeniceClusterResources(clusterName).getSchemaRepository(); - schemaRepository.addValueSchema(storeName, valueSchemaStr, expectedCompatibilityType); - return new SchemaEntry(schemaRepository.getValueSchemaId(storeName, valueSchemaStr), valueSchemaStr); + int newValueSchemaId = + schemaRepository.preCheckValueSchemaAndGetNextAvailableId(storeName, valueSchemaStr, expectedCompatibilityType); + return addValueSchema(clusterName, storeName, valueSchemaStr, newValueSchemaId, expectedCompatibilityType); } /** @@ -5399,8 +4611,14 @@ public SchemaEntry addValueSchema( DirectionalSchemaCompatibilityType compatibilityType) { checkControllerLeadershipFor(clusterName); ReadWriteSchemaRepository schemaRepository = getHelixVeniceClusterResources(clusterName).getSchemaRepository(); + + if (schemaId == SchemaData.DUPLICATE_VALUE_SCHEMA_CODE) { + return new SchemaEntry(schemaRepository.getValueSchemaId(storeName, valueSchemaStr), valueSchemaStr); + } + int newValueSchemaId = schemaRepository.preCheckValueSchemaAndGetNextAvailableId(storeName, valueSchemaStr, compatibilityType); + if (newValueSchemaId != SchemaData.DUPLICATE_VALUE_SCHEMA_CODE && newValueSchemaId != schemaId) { throw new VeniceException( "Inconsistent value schema id between the caller and the local schema repository." @@ -5408,7 +4626,15 @@ public SchemaEntry addValueSchema( + newValueSchemaId + " for store " + storeName + " in cluster " + clusterName + " Schema: " + valueSchemaStr); } - return schemaRepository.addValueSchema(storeName, valueSchemaStr, newValueSchemaId); + SchemaEntry addedSchemaEntry = schemaRepository.addValueSchema(storeName, valueSchemaStr, schemaId); + + if (isPrimary() && !isParent() && newValueSchemaId != SchemaData.DUPLICATE_VALUE_SCHEMA_CODE) { + // Now register all inferred schemas for the store if this is a child controller in single-region mode. + // Parent in multi-region mode will register all inferred schemas via the admin channel. + PrimaryControllerConfigUpdateUtils.registerInferredSchemas(this, clusterName, storeName); + } + + return addedSchemaEntry; } /** @@ -5462,14 +4688,8 @@ public DerivedSchemaEntry removeDerivedSchema( .removeDerivedSchema(storeName, valueSchemaId, derivedSchemaId); } - /** - * Add a new superset schema for the given store with all specified properties. - *

- * Generate the superset schema off the current schema and latest superset schema (if any, if not pick the latest value schema) existing in the store. - * If the newly generated superset schema is unique add it to the store and update latestSuperSetValueSchemaId of the store. - */ @Override - public SchemaEntry addSupersetSchema( + public void addSupersetSchema( String clusterName, String storeName, String valueSchema, @@ -5479,12 +4699,16 @@ public SchemaEntry addSupersetSchema( checkControllerLeadershipFor(clusterName); ReadWriteSchemaRepository schemaRepository = getHelixVeniceClusterResources(clusterName).getSchemaRepository(); + if (valueSchemaId != SchemaData.INVALID_VALUE_SCHEMA_ID) { + // add the value schema + schemaRepository.addValueSchema(storeName, valueSchema, valueSchemaId); + } + final SchemaEntry existingSupersetSchemaEntry = schemaRepository.getValueSchema(storeName, supersetSchemaId); if (existingSupersetSchemaEntry == null) { // If the new superset schema does not exist in the schema repo, add it LOGGER.info("Adding superset schema: {} for store: {}", supersetSchemaStr, storeName); schemaRepository.addValueSchema(storeName, supersetSchemaStr, supersetSchemaId); - } else { final Schema newSupersetSchema = AvroSchemaParseUtils.parseSchemaFromJSONStrictValidation(supersetSchemaStr); if (!AvroSchemaUtils.compareSchemaIgnoreFieldOrder(existingSupersetSchemaEntry.getSchema(), newSupersetSchema)) { @@ -5494,25 +4718,10 @@ public SchemaEntry addSupersetSchema( } } - // add the value schema - return schemaRepository.addValueSchema(storeName, valueSchema, valueSchemaId); - } - - int getValueSchemaIdIgnoreFieldOrder( - String clusterName, - String storeName, - String valueSchemaStr, - Comparator schemaComparator) { - checkControllerLeadershipFor(clusterName); - SchemaEntry valueSchemaEntry = new SchemaEntry(SchemaData.UNKNOWN_SCHEMA_ID, valueSchemaStr); - - for (SchemaEntry schemaEntry: getValueSchemas(clusterName, storeName)) { - if (schemaComparator.compare(schemaEntry.getSchema(), valueSchemaEntry.getSchema()) == 0) { - return schemaEntry.getId(); - } - } - return SchemaData.INVALID_VALUE_SCHEMA_ID; - + storeMetadataUpdate(clusterName, storeName, store -> { + store.setLatestSuperSetValueSchemaId(supersetSchemaId); + return store; + }); } int checkPreConditionForAddValueSchemaAndGetNewSchemaId( @@ -5548,28 +4757,7 @@ public Collection getReplicationMetadataSchemas(String clusterNa return schemaRepo.getReplicationMetadataSchemas(storeName); } - boolean checkIfValueSchemaAlreadyHasRmdSchema( - String clusterName, - String storeName, - final int valueSchemaID, - final int replicationMetadataVersionId) { - checkControllerLeadershipFor(clusterName); - Collection schemaEntries = - getHelixVeniceClusterResources(clusterName).getSchemaRepository().getReplicationMetadataSchemas(storeName); - for (RmdSchemaEntry rmdSchemaEntry: schemaEntries) { - if (rmdSchemaEntry.getValueSchemaID() == valueSchemaID - && rmdSchemaEntry.getId() == replicationMetadataVersionId) { - return true; - } - } - return false; - } - - boolean checkIfMetadataSchemaAlreadyPresent( - String clusterName, - String storeName, - int valueSchemaId, - RmdSchemaEntry rmdSchemaEntry) { + boolean checkIfMetadataSchemaAlreadyPresent(String clusterName, String storeName, RmdSchemaEntry rmdSchemaEntry) { checkControllerLeadershipFor(clusterName); try { Collection schemaEntries = @@ -5601,7 +4789,7 @@ public RmdSchemaEntry addReplicationMetadataSchema( RmdSchemaEntry rmdSchemaEntry = new RmdSchemaEntry(valueSchemaId, replicationMetadataVersionId, replicationMetadataSchemaStr); - if (checkIfMetadataSchemaAlreadyPresent(clusterName, storeName, valueSchemaId, rmdSchemaEntry)) { + if (checkIfMetadataSchemaAlreadyPresent(clusterName, storeName, rmdSchemaEntry)) { LOGGER.info( "Timestamp metadata schema Already present: for store: {} in cluster: {} metadataSchema: {} " + "replicationMetadataVersionId: {} valueSchemaId: {}", @@ -5728,13 +4916,6 @@ public Map getStorageNodesStatus(String clusterName, boolean ena return instancesStatusesMap; } - Schema getSupersetOrLatestValueSchema(String clusterName, Store store) { - ReadWriteSchemaRepository schemaRepository = getHelixVeniceClusterResources(clusterName).getSchemaRepository(); - // If already a superset schema exists, try to generate the new superset from that and the input value schema - SchemaEntry existingSchema = schemaRepository.getSupersetOrLatestValueSchema(store.getName()); - return existingSchema == null ? null : existingSchema.getSchema(); - } - /** * Remove one storage node from the given cluster. *

@@ -7049,6 +6230,7 @@ void checkControllerLeadershipFor(String clusterName) { /** * @return the aggregate resources required by controller to manage a Venice cluster. */ + @Override public HelixVeniceClusterResources getHelixVeniceClusterResources(String cluster) { Optional resources = controllerStateModelFactory.getModel(cluster).getResources(); if (!resources.isPresent()) { @@ -7147,131 +6329,6 @@ public void deleteAclForStore(String clusterName, String storeName) { throw new VeniceUnsupportedOperationException("deleteAclForStore is not supported!"); } - /** - * @see Admin#configureNativeReplication(String, VeniceUserStoreType, Optional, boolean, Optional, Optional) - */ - @Override - public void configureNativeReplication( - String clusterName, - VeniceUserStoreType storeType, - Optional storeName, - boolean enableNativeReplicationForCluster, - Optional newSourceFabric, - Optional regionsFilter) { - /** - * Check whether the command affects this fabric. - */ - if (regionsFilter.isPresent()) { - Set fabrics = parseRegionsFilterList(regionsFilter.get()); - if (!fabrics.contains(multiClusterConfigs.getRegionName())) { - LOGGER.info( - "EnableNativeReplicationForCluster command will be skipped for cluster {}, because the fabrics filter " - + "is {} which doesn't include the current fabric: {}", - clusterName, - fabrics, - multiClusterConfigs.getRegionName()); - return; - } - } - - if (storeName.isPresent()) { - /** - * Legacy stores venice_system_store_davinci_push_status_store_ still exist. - * But {@link com.linkedin.venice.helix.HelixReadOnlyStoreRepositoryAdapter#getStore(String)} cannot find - * them by store names. Skip davinci push status stores until legacy znodes are cleaned up. - */ - VeniceSystemStoreType systemStoreType = VeniceSystemStoreType.getSystemStoreType(storeName.get()); - if (systemStoreType != null && systemStoreType.equals(VeniceSystemStoreType.DAVINCI_PUSH_STATUS_STORE)) { - LOGGER.info("Will not enable native replication for davinci push status store: {}", storeName.get()); - return; - } - - /** - * The function is invoked by {@link com.linkedin.venice.controller.kafka.consumer.AdminExecutionTask} if the - * storeName is present. - */ - Store originalStore = getStore(clusterName, storeName.get()); - if (originalStore == null) { - throw new VeniceNoStoreException(storeName.get(), clusterName); - } - boolean shouldUpdateNativeReplication = false; - switch (storeType) { - case BATCH_ONLY: - shouldUpdateNativeReplication = - !originalStore.isHybrid() && !originalStore.isIncrementalPushEnabled() && !originalStore.isSystemStore(); - break; - case HYBRID_ONLY: - shouldUpdateNativeReplication = - originalStore.isHybrid() && !originalStore.isIncrementalPushEnabled() && !originalStore.isSystemStore(); - break; - case INCREMENTAL_PUSH: - shouldUpdateNativeReplication = originalStore.isIncrementalPushEnabled() && !originalStore.isSystemStore(); - break; - case HYBRID_OR_INCREMENTAL: - shouldUpdateNativeReplication = - (originalStore.isHybrid() || originalStore.isIncrementalPushEnabled()) && !originalStore.isSystemStore(); - break; - case SYSTEM: - shouldUpdateNativeReplication = originalStore.isSystemStore(); - break; - case ALL: - shouldUpdateNativeReplication = true; - break; - default: - throw new VeniceException("Unsupported store type." + storeType); - } - if (shouldUpdateNativeReplication) { - LOGGER.info("Will enable native replication for store: {}", storeName.get()); - setNativeReplicationEnabled(clusterName, storeName.get(), enableNativeReplicationForCluster); - newSourceFabric.ifPresent(f -> setNativeReplicationSourceFabric(clusterName, storeName.get(), f)); - } else { - LOGGER.info("Will not enable native replication for store: {}", storeName.get()); - } - } else { - /** - * The batch update command hits child controller directly; all stores in the cluster will be updated - */ - List storesToBeConfigured; - switch (storeType) { - case BATCH_ONLY: - storesToBeConfigured = getAllStores(clusterName).stream() - .filter(s -> (!s.isHybrid() && !s.isIncrementalPushEnabled() && !s.isSystemStore())) - .collect(Collectors.toList()); - break; - case HYBRID_ONLY: - storesToBeConfigured = getAllStores(clusterName).stream() - .filter(s -> (s.isHybrid() && !s.isIncrementalPushEnabled() && !s.isSystemStore())) - .collect(Collectors.toList()); - break; - case INCREMENTAL_PUSH: - storesToBeConfigured = getAllStores(clusterName).stream() - .filter(s -> (s.isIncrementalPushEnabled() && !s.isSystemStore())) - .collect(Collectors.toList()); - break; - case HYBRID_OR_INCREMENTAL: - storesToBeConfigured = getAllStores(clusterName).stream() - .filter(s -> ((s.isHybrid() || s.isIncrementalPushEnabled()) && !s.isSystemStore())) - .collect(Collectors.toList()); - break; - case SYSTEM: - storesToBeConfigured = - getAllStores(clusterName).stream().filter(Store::isSystemStore).collect(Collectors.toList()); - break; - case ALL: - storesToBeConfigured = getAllStores(clusterName); - break; - default: - throw new VeniceException("Unsupported store type." + storeType); - } - - storesToBeConfigured.forEach(store -> { - LOGGER.info("Will enable native replication for store: {}", store.getName()); - setNativeReplicationEnabled(clusterName, store.getName(), enableNativeReplicationForCluster); - newSourceFabric.ifPresent(f -> setNativeReplicationSourceFabric(clusterName, storeName.get(), f)); - }); - } - } - /** * @see Admin#configureActiveActiveReplication(String, VeniceUserStoreType, Optional, boolean, Optional) */ @@ -7666,38 +6723,19 @@ Store checkPreConditionForAclOp(String clusterName, String storeName) { } /** - * A store is not hybrid in the following two scenarios: - * If hybridStoreConfig is null, it means store is not hybrid. - * If all the hybrid config values are negative, it indicates that the store is being set back to batch-only store. - */ - boolean isHybrid(HybridStoreConfig hybridStoreConfig) { - return hybridStoreConfig != null - && (hybridStoreConfig.getRewindTimeInSeconds() >= 0 || hybridStoreConfig.getOffsetLagThresholdToGoOnline() >= 0 - || hybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds() >= 0); - } - - /** - * @see VeniceHelixAdmin#isHybrid(HybridStoreConfig) + * @see Admin#isParent() */ - boolean isHybrid(HybridStoreConfigRecord hybridStoreConfigRecord) { - HybridStoreConfig hybridStoreConfig = null; - if (hybridStoreConfigRecord != null) { - hybridStoreConfig = new HybridStoreConfigImpl( - hybridStoreConfigRecord.rewindTimeInSeconds, - hybridStoreConfigRecord.offsetLagThresholdToGoOnline, - hybridStoreConfigRecord.producerTimestampLagThresholdToGoOnlineInSeconds, - DataReplicationPolicy.valueOf(hybridStoreConfigRecord.dataReplicationPolicy), - BufferReplayPolicy.valueOf(hybridStoreConfigRecord.bufferReplayPolicy)); - } - return isHybrid(hybridStoreConfig); + @Override + public boolean isParent() { + return multiClusterConfigs.isParent(); } /** - * @see Admin#isParent() + * @see Admin#isPrimary() */ @Override - public boolean isParent() { - return multiClusterConfigs.isParent(); + public boolean isPrimary() { + return !multiClusterConfigs.isMultiRegion() || isParent(); } /** @@ -7776,7 +6814,7 @@ public Optional getEmergencySourceRegion(@Nonnull String clusterName) { @Override public Optional getAggregateRealTimeTopicSource(String clusterName) { String sourceRegion = multiClusterConfigs.getControllerConfig(clusterName).getAggregateRealTimeSourceRegion(); - if (sourceRegion != null && sourceRegion.length() > 0) { + if (!StringUtils.isEmpty(sourceRegion)) { return Optional.of(getNativeReplicationKafkaBootstrapServerAddress(sourceRegion)); } else { return Optional.empty(); @@ -8087,12 +7125,12 @@ public boolean isAdminTopicConsumptionEnabled(String clusterName) { // HelixVeniceClusterResources should exist on leader controller HelixVeniceClusterResources resources = getHelixVeniceClusterResources(clusterName); try (AutoCloseableLock ignore = resources.getClusterLockManager().createClusterReadLock()) { - HelixReadWriteLiveClusterConfigRepository clusterConfigRepository = + HelixReadWriteLiveClusterConfigRepository liveConfigRepository = getReadWriteLiveClusterConfigRepository(clusterName); - // Enable child controller admin topic consumption when both cfg2 config and live config are true - adminTopicConsumptionEnabled = - clusterConfigRepository.getConfigs().isChildControllerAdminTopicConsumptionEnabled() - && multiClusterConfigs.getControllerConfig(clusterName).isChildControllerAdminTopicConsumptionEnabled(); + // Enable child controller admin topic consumption when configs applied at startup and live config are true. + // The live configs are used during store migration. + adminTopicConsumptionEnabled = liveConfigRepository.getConfigs().isChildControllerAdminTopicConsumptionEnabled() + && multiClusterConfigs.getControllerConfig(clusterName).isMultiRegion(); } return adminTopicConsumptionEnabled; } @@ -8116,9 +7154,13 @@ public void createStoragePersona( Set storesToEnforce, Set owners) { checkControllerLeadershipFor(clusterName); - HelixVeniceClusterResources resources = getHelixVeniceClusterResources(clusterName); + StoragePersonaRepository repository = getHelixVeniceClusterResources(clusterName).getStoragePersonaRepository(); + if (repository.hasPersona(name)) { + throw new VeniceException("Persona with name " + name + " already exists"); + } + repository.validatePersona(name, quotaNumber, storesToEnforce, owners); + try { - StoragePersonaRepository repository = resources.getStoragePersonaRepository(); repository.addPersona(name, quotaNumber, storesToEnforce, owners); } catch (Exception e) { LOGGER.error("Failed to execute CreateStoragePersonaOperation.", e); @@ -8158,9 +7200,10 @@ public void deleteStoragePersona(String clusterName, String name) { @Override public void updateStoragePersona(String clusterName, String name, UpdateStoragePersonaQueryParams queryParams) { checkControllerLeadershipFor(clusterName); - HelixVeniceClusterResources resources = getHelixVeniceClusterResources(clusterName); + StoragePersonaRepository repository = getHelixVeniceClusterResources(clusterName).getStoragePersonaRepository(); + repository.validatePersonaUpdate(name, queryParams); + try { - StoragePersonaRepository repository = resources.getStoragePersonaRepository(); repository.updatePersona(name, queryParams); } catch (Exception e) { LOGGER.error("Failed to execute UpdateStoragePersonaOperation.", e); @@ -8291,8 +7334,8 @@ public boolean isClusterWipeAllowed(String clusterName) { return multiClusterConfigs.getControllerConfig(clusterName).isClusterWipeAllowed(); } - // Visible for testing - VeniceControllerMultiClusterConfig getMultiClusterConfigs() { + @Override + public VeniceControllerMultiClusterConfig getMultiClusterConfigs() { return multiClusterConfigs; } @@ -8300,4 +7343,9 @@ VeniceControllerMultiClusterConfig getMultiClusterConfigs() { public void setPushJobDetailsStoreClient(AvroSpecificStoreClient client) { pushJobDetailsStoreClient = client; } + + @Override + public PubSubTopicRepository getPubSubTopicRepository() { + return pubSubTopicRepository; + } } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceParentHelixAdmin.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceParentHelixAdmin.java index d4d8e3dc99..312e8c9cf1 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceParentHelixAdmin.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceParentHelixAdmin.java @@ -2,7 +2,6 @@ import static com.linkedin.venice.controller.VeniceHelixAdmin.VERSION_ID_UNSET; import static com.linkedin.venice.controller.kafka.consumer.AdminConsumptionTask.IGNORED_CURRENT_VERSION; -import static com.linkedin.venice.controller.util.ParentControllerConfigUpdateUtils.addUpdateSchemaForStore; import static com.linkedin.venice.controllerapi.ControllerApiConstants.ACCESS_CONTROLLED; import static com.linkedin.venice.controllerapi.ControllerApiConstants.ACTIVE_ACTIVE_REPLICATION_ENABLED; import static com.linkedin.venice.controllerapi.ControllerApiConstants.AMPLIFICATION_FACTOR; @@ -56,10 +55,8 @@ import static com.linkedin.venice.controllerapi.ControllerApiConstants.UNUSED_SCHEMA_DELETION_ENABLED; import static com.linkedin.venice.controllerapi.ControllerApiConstants.VERSION; import static com.linkedin.venice.controllerapi.ControllerApiConstants.WRITE_COMPUTATION_ENABLED; -import static com.linkedin.venice.meta.HybridStoreConfigImpl.DEFAULT_HYBRID_OFFSET_LAG_THRESHOLD; -import static com.linkedin.venice.meta.HybridStoreConfigImpl.DEFAULT_HYBRID_TIME_LAG_THRESHOLD; -import static com.linkedin.venice.meta.HybridStoreConfigImpl.DEFAULT_REWIND_TIME_IN_SECONDS; -import static com.linkedin.venice.meta.VersionStatus.*; +import static com.linkedin.venice.meta.VersionStatus.ONLINE; +import static com.linkedin.venice.meta.VersionStatus.PUSHED; import static com.linkedin.venice.serialization.avro.AvroProtocolDefinition.BATCH_JOB_HEARTBEAT; import static com.linkedin.venice.serialization.avro.AvroProtocolDefinition.PUSH_JOB_DETAILS; @@ -68,7 +65,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.linkedin.venice.ConfigConstants; import com.linkedin.venice.SSLConfig; import com.linkedin.venice.acl.AclException; import com.linkedin.venice.acl.DynamicAccessController; @@ -82,7 +78,6 @@ import com.linkedin.venice.authorization.Resource; import com.linkedin.venice.common.VeniceSystemStoreType; import com.linkedin.venice.common.VeniceSystemStoreUtils; -import com.linkedin.venice.compression.CompressionStrategy; import com.linkedin.venice.controller.authorization.SystemStoreAclSynchronizationTask; import com.linkedin.venice.controller.init.DelegatingClusterLeaderInitializationRoutine; import com.linkedin.venice.controller.init.SharedInternalRTStoreInitializationRoutine; @@ -92,7 +87,6 @@ import com.linkedin.venice.controller.kafka.protocol.admin.AddVersion; import com.linkedin.venice.controller.kafka.protocol.admin.AdminOperation; import com.linkedin.venice.controller.kafka.protocol.admin.ConfigureActiveActiveReplicationForCluster; -import com.linkedin.venice.controller.kafka.protocol.admin.ConfigureNativeReplicationForCluster; import com.linkedin.venice.controller.kafka.protocol.admin.CreateStoragePersona; import com.linkedin.venice.controller.kafka.protocol.admin.DeleteAllVersions; import com.linkedin.venice.controller.kafka.protocol.admin.DeleteOldVersion; @@ -118,7 +112,6 @@ import com.linkedin.venice.controller.kafka.protocol.admin.SetStoreOwner; import com.linkedin.venice.controller.kafka.protocol.admin.SetStorePartitionCount; import com.linkedin.venice.controller.kafka.protocol.admin.StoreCreation; -import com.linkedin.venice.controller.kafka.protocol.admin.StoreViewConfigRecord; import com.linkedin.venice.controller.kafka.protocol.admin.SupersetSchemaCreation; import com.linkedin.venice.controller.kafka.protocol.admin.UpdateStoragePersona; import com.linkedin.venice.controller.kafka.protocol.admin.UpdateStore; @@ -131,7 +124,10 @@ import com.linkedin.venice.controller.migration.MigrationPushStrategyZKAccessor; import com.linkedin.venice.controller.supersetschema.DefaultSupersetSchemaGenerator; import com.linkedin.venice.controller.supersetschema.SupersetSchemaGenerator; -import com.linkedin.venice.controller.util.ParentControllerConfigUpdateUtils; +import com.linkedin.venice.controller.util.AdminUtils; +import com.linkedin.venice.controller.util.PrimaryControllerConfigUpdateUtils; +import com.linkedin.venice.controller.util.UpdateStoreUtils; +import com.linkedin.venice.controller.util.UpdateStoreWrapper; import com.linkedin.venice.controllerapi.AdminCommandExecution; import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.ControllerResponse; @@ -154,10 +150,8 @@ import com.linkedin.venice.exceptions.ConcurrentBatchPushException; import com.linkedin.venice.exceptions.ConfigurationException; import com.linkedin.venice.exceptions.ErrorType; -import com.linkedin.venice.exceptions.PartitionerSchemaMismatchException; import com.linkedin.venice.exceptions.ResourceStillExistsException; import com.linkedin.venice.exceptions.VeniceException; -import com.linkedin.venice.exceptions.VeniceHttpException; import com.linkedin.venice.exceptions.VeniceNoStoreException; import com.linkedin.venice.exceptions.VeniceUnsupportedOperationException; import com.linkedin.venice.helix.HelixReadOnlyStoreConfigRepository; @@ -167,13 +161,13 @@ import com.linkedin.venice.helix.Replica; import com.linkedin.venice.helix.StoragePersonaRepository; import com.linkedin.venice.helix.ZkStoreConfigAccessor; -import com.linkedin.venice.meta.BackupStrategy; import com.linkedin.venice.meta.BufferReplayPolicy; import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.ETLStoreConfig; import com.linkedin.venice.meta.HybridStoreConfig; import com.linkedin.venice.meta.Instance; import com.linkedin.venice.meta.PartitionerConfig; +import com.linkedin.venice.meta.ReadWriteSchemaRepository; import com.linkedin.venice.meta.ReadWriteStoreRepository; import com.linkedin.venice.meta.RegionPushDetails; import com.linkedin.venice.meta.RoutersClusterConfig; @@ -185,8 +179,6 @@ import com.linkedin.venice.meta.VeniceUserStoreType; import com.linkedin.venice.meta.Version; import com.linkedin.venice.meta.VersionStatus; -import com.linkedin.venice.meta.ViewConfig; -import com.linkedin.venice.meta.ViewConfigImpl; import com.linkedin.venice.persona.StoragePersona; import com.linkedin.venice.pubsub.PubSubConsumerAdapterFactory; import com.linkedin.venice.pubsub.PubSubTopicRepository; @@ -202,7 +194,6 @@ import com.linkedin.venice.schema.SchemaEntry; import com.linkedin.venice.schema.avro.DirectionalSchemaCompatibilityType; import com.linkedin.venice.schema.rmd.RmdSchemaEntry; -import com.linkedin.venice.schema.rmd.RmdSchemaGenerator; import com.linkedin.venice.schema.writecompute.DerivedSchemaEntry; import com.linkedin.venice.schema.writecompute.WriteComputeSchemaConverter; import com.linkedin.venice.security.SSLFactory; @@ -218,7 +209,6 @@ import com.linkedin.venice.utils.CollectionUtils; import com.linkedin.venice.utils.ObjectMapperFactory; import com.linkedin.venice.utils.Pair; -import com.linkedin.venice.utils.PartitionUtils; import com.linkedin.venice.utils.ReflectUtils; import com.linkedin.venice.utils.RegionUtils; import com.linkedin.venice.utils.SslUtils; @@ -229,7 +219,6 @@ import com.linkedin.venice.utils.concurrent.VeniceConcurrentHashMap; import com.linkedin.venice.utils.locks.AutoCloseableLock; import com.linkedin.venice.views.VeniceView; -import com.linkedin.venice.views.ViewUtils; import com.linkedin.venice.writer.VeniceWriter; import com.linkedin.venice.writer.VeniceWriterFactory; import com.linkedin.venice.writer.VeniceWriterOptions; @@ -258,13 +247,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import org.apache.avro.Schema; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.Validate; -import org.apache.http.HttpStatus; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -1001,7 +988,7 @@ public void addVersionAndStartIngestion( boolean versionSwapDeferred, int repushSourceVersion) { // Parent controller will always pick the replicationMetadataVersionId from configs. - final int replicationMetadataVersionId = getRmdVersionID(storeName, clusterName); + final int replicationMetadataVersionId = AdminUtils.getRmdVersionID(this, storeName, clusterName); Version version = getVeniceHelixAdmin().addVersionOnly( clusterName, storeName, @@ -1012,9 +999,6 @@ public void addVersionAndStartIngestion( remoteKafkaBootstrapServers, rewindTimeInSecondsOverride, replicationMetadataVersionId); - if (version.isActiveActiveReplicationEnabled()) { - updateReplicationMetadataSchemaForAllValueSchema(clusterName, storeName); - } acquireAdminMessageLock(clusterName, storeName); try { sendAddVersionAdminMessage(clusterName, storeName, pushJobId, version, numberOfPartitions, pushType, null, -1); @@ -1023,34 +1007,6 @@ public void addVersionAndStartIngestion( } } - private int getRmdVersionID(final String storeName, final String clusterName) { - final Store store = getVeniceHelixAdmin().getStore(clusterName, storeName); - if (store == null) { - LOGGER.warn( - "No store found in the store repository. Will get store-level RMD version ID from cluster config. " - + "Store name: {}, cluster: {}", - storeName, - clusterName); - } else if (store.getRmdVersion() == ConfigConstants.UNSPECIFIED_REPLICATION_METADATA_VERSION) { - LOGGER.info("No store-level RMD version ID found for store {} in cluster {}", storeName, clusterName); - } else { - LOGGER.info( - "Found store-level RMD version ID {} for store {} in cluster {}", - store.getRmdVersion(), - storeName, - clusterName); - return store.getRmdVersion(); - } - - final VeniceControllerConfig controllerClusterConfig = getMultiClusterConfigs().getControllerConfig(clusterName); - if (controllerClusterConfig == null) { - throw new VeniceException("No controller cluster config found for cluster " + clusterName); - } - final int rmdVersionID = controllerClusterConfig.getReplicationMetadataVersion(); - LOGGER.info("Use RMD version ID {} for cluster {}", rmdVersionID, clusterName); - return rmdVersionID; - } - /** * Since there is no offline push running in Parent Controller, * the old store versions won't be cleaned up by job completion action, so Parent Controller chooses @@ -1589,7 +1545,7 @@ Version addVersionAndTopicOnly( boolean versionSwapDeferred, String targetedRegions, int repushSourceVersion) { - final int replicationMetadataVersionId = getRmdVersionID(storeName, clusterName); + final int replicationMetadataVersionId = AdminUtils.getRmdVersionID(this, storeName, clusterName); Pair result = getVeniceHelixAdmin().addVersionAndTopicOnly( clusterName, storeName, @@ -1611,9 +1567,6 @@ Version addVersionAndTopicOnly( repushSourceVersion); Version newVersion = result.getSecond(); if (result.getFirst()) { - if (newVersion.isActiveActiveReplicationEnabled()) { - updateReplicationMetadataSchemaForAllValueSchema(clusterName, storeName); - } // Send admin message if the version is newly created. acquireAdminMessageLock(clusterName, storeName); try { @@ -2047,16 +2000,6 @@ public void rollbackToBackupVersion(String clusterName, String storeName, String } } - /** - * Unsupported operation in the parent controller. - */ - @Override - public void setStoreLargestUsedVersion(String clusterName, String storeName, int versionNumber) { - throw new VeniceUnsupportedOperationException( - "setStoreLargestUsedVersion", - "This is only supported in the Child Controller."); - } - /** * Update the owner of a specified store by sending {@link AdminMessageType#SET_STORE_OWNER SET_STORE_OWNER} admin message * to the admin topic. @@ -2200,297 +2143,175 @@ public void setStoreReadWriteability(String clusterName, String storeName, boole public void updateStore(String clusterName, String storeName, UpdateStoreQueryParams params) { acquireAdminMessageLock(clusterName, storeName); try { - Optional owner = params.getOwner(); - Optional readability = params.getEnableReads(); - Optional writeability = params.getEnableWrites(); - Optional partitionCount = params.getPartitionCount(); - Optional partitionerClass = params.getPartitionerClass(); - Optional> partitionerParams = params.getPartitionerParams(); - Optional amplificationFactor = params.getAmplificationFactor(); - Optional storageQuotaInByte = params.getStorageQuotaInByte(); - Optional readQuotaInCU = params.getReadQuotaInCU(); - Optional currentVersion = params.getCurrentVersion(); - Optional largestUsedVersionNumber = params.getLargestUsedVersionNumber(); - Optional hybridRewindSeconds = params.getHybridRewindSeconds(); - Optional hybridOffsetLagThreshold = params.getHybridOffsetLagThreshold(); - Optional hybridTimeLagThreshold = params.getHybridTimeLagThreshold(); - Optional hybridDataReplicationPolicy = params.getHybridDataReplicationPolicy(); - Optional hybridBufferReplayPolicy = params.getHybridBufferReplayPolicy(); - Optional accessControlled = params.getAccessControlled(); - Optional compressionStrategy = params.getCompressionStrategy(); - Optional clientDecompressionEnabled = params.getClientDecompressionEnabled(); - Optional chunkingEnabled = params.getChunkingEnabled(); - Optional rmdChunkingEnabled = params.getRmdChunkingEnabled(); - Optional batchGetLimit = params.getBatchGetLimit(); - Optional numVersionsToPreserve = params.getNumVersionsToPreserve(); - Optional incrementalPushEnabled = params.getIncrementalPushEnabled(); - Optional storeMigration = params.getStoreMigration(); - Optional writeComputationEnabled = params.getWriteComputationEnabled(); - Optional replicationMetadataVersionID = params.getReplicationMetadataVersionID(); - Optional readComputationEnabled = params.getReadComputationEnabled(); - Optional bootstrapToOnlineTimeoutInHours = params.getBootstrapToOnlineTimeoutInHours(); - Optional backupStrategy = params.getBackupStrategy(); - Optional autoSchemaRegisterPushJobEnabled = params.getAutoSchemaRegisterPushJobEnabled(); - Optional hybridStoreDiskQuotaEnabled = params.getHybridStoreDiskQuotaEnabled(); - Optional regularVersionETLEnabled = params.getRegularVersionETLEnabled(); - Optional futureVersionETLEnabled = params.getFutureVersionETLEnabled(); - Optional etledUserProxyAccount = params.getETLedProxyUserAccount(); - Optional nativeReplicationEnabled = params.getNativeReplicationEnabled(); - Optional pushStreamSourceAddress = params.getPushStreamSourceAddress(); - Optional backupVersionRetentionMs = params.getBackupVersionRetentionMs(); - Optional replicationFactor = params.getReplicationFactor(); - Optional migrationDuplicateStore = params.getMigrationDuplicateStore(); - Optional nativeReplicationSourceFabric = params.getNativeReplicationSourceFabric(); - Optional activeActiveReplicationEnabled = params.getActiveActiveReplicationEnabled(); - Optional regionsFilter = params.getRegionsFilter(); - Optional personaName = params.getStoragePersona(); - Optional> storeViewConfig = params.getStoreViews(); - Optional viewName = params.getViewName(); - Optional viewClassName = params.getViewClassName(); - Optional> viewParams = params.getViewClassParams(); - Optional removeView = params.getDisableStoreView(); - Optional latestSupersetSchemaId = params.getLatestSupersetSchemaId(); - Optional unusedSchemaDeletionEnabled = params.getUnusedSchemaDeletionEnabled(); - /** * Check whether parent controllers will only propagate the update configs to child controller, or all unchanged * configs should be replicated to children too. */ Optional replicateAll = params.getReplicateAllConfigs(); - Optional storageNodeReadQuotaEnabled = params.getStorageNodeReadQuotaEnabled(); - Optional minCompactionLagSeconds = params.getMinCompactionLagSeconds(); - Optional maxCompactionLagSeconds = params.getMaxCompactionLagSeconds(); - Optional maxRecordSizeBytes = params.getMaxRecordSizeBytes(); boolean replicateAllConfigs = replicateAll.isPresent() && replicateAll.get(); List updatedConfigsList = new LinkedList<>(); - String errorMessagePrefix = "Store update error for " + storeName + " in cluster: " + clusterName + ": "; - Store currStore = getVeniceHelixAdmin().getStore(clusterName, storeName); - if (currStore == null) { - LOGGER.error(errorMessagePrefix + "store does not exist, and thus cannot be updated."); - throw new VeniceNoStoreException(storeName, clusterName); + UpdateStoreWrapper updateStoreWrapper = UpdateStoreUtils.getStoreUpdate(this, clusterName, storeName, params); + if (updateStoreWrapper == null) { + return; } + + Store originalStore = updateStoreWrapper.originalStore; + Set updatedConfigs = updateStoreWrapper.updatedConfigs; + Store updatedStore = updateStoreWrapper.updatedStore; + + if (!replicateAllConfigs && updatedConfigs.isEmpty()) { + String errMsg = "UpdateStore command failed for store " + storeName + ". The command didn't change any specific" + + " store config and didn't specify \"--replicate-all-configs\" flag."; + LOGGER.error(errMsg); + throw new VeniceException(errMsg); + } + UpdateStore setStore = (UpdateStore) AdminMessageType.UPDATE_STORE.getNewInstance(); setStore.clusterName = clusterName; setStore.storeName = storeName; - setStore.owner = owner.map(addToUpdatedConfigList(updatedConfigsList, OWNER)).orElseGet(currStore::getOwner); - // Invalid config update on hybrid will not be populated to admin channel so subsequent updates on the store won't - // be blocked by retry mechanism. - if (currStore.isHybrid() && (partitionerClass.isPresent() || partitionerParams.isPresent())) { - String errorMessage = errorMessagePrefix + "Cannot change partitioner class and parameters for hybrid stores"; - LOGGER.error(errorMessage); - throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.BAD_REQUEST); + if (updatedConfigs.contains(OWNER)) { + setStore.owner = updatedStore.getOwner(); + updatedConfigsList.add(OWNER); + } else { + setStore.owner = originalStore.getOwner(); } - if (partitionCount.isPresent()) { - getVeniceHelixAdmin().preCheckStorePartitionCountUpdate(clusterName, currStore, partitionCount.get()); - setStore.partitionNum = partitionCount.get(); + if (updatedConfigs.contains(PARTITION_COUNT)) { + setStore.partitionNum = updatedStore.getPartitionCount(); updatedConfigsList.add(PARTITION_COUNT); } else { - setStore.partitionNum = currStore.getPartitionCount(); + setStore.partitionNum = originalStore.getPartitionCount(); } - /** - * TODO: We should build an UpdateStoreHelper that takes current store config and update command as input, and - * return whether the update command is valid. - */ - validateActiveActiveReplicationEnableConfigs(activeActiveReplicationEnabled, nativeReplicationEnabled, currStore); - - setStore.nativeReplicationEnabled = - nativeReplicationEnabled.map(addToUpdatedConfigList(updatedConfigsList, NATIVE_REPLICATION_ENABLED)) - .orElseGet(currStore::isNativeReplicationEnabled); - setStore.pushStreamSourceAddress = - pushStreamSourceAddress.map(addToUpdatedConfigList(updatedConfigsList, PUSH_STREAM_SOURCE_ADDRESS)) - .orElseGet(currStore::getPushStreamSourceAddress); - - if (storeViewConfig.isPresent() && viewName.isPresent()) { - throw new VeniceException("Cannot update a store view and overwrite store view setup together!"); - } - if (viewName.isPresent()) { - Map updatedViewSettings; - if (!removeView.isPresent()) { - if (!viewClassName.isPresent()) { - throw new VeniceException("View class name is required when configuring a view."); - } - // If View parameter is not provided, use emtpy map instead. It does not inherit from existing config. - ViewConfig viewConfig = new ViewConfigImpl(viewClassName.get(), viewParams.orElse(Collections.emptyMap())); - validateStoreViewConfig(currStore, viewConfig); - updatedViewSettings = VeniceHelixAdmin.addNewViewConfigsIntoOldConfigs(currStore, viewName.get(), viewConfig); - } else { - updatedViewSettings = VeniceHelixAdmin.removeViewConfigFromStoreViewConfigMap(currStore, viewName.get()); - } - setStore.views = updatedViewSettings; - updatedConfigsList.add(STORE_VIEW); + if (updatedConfigs.contains(NATIVE_REPLICATION_ENABLED)) { + setStore.nativeReplicationEnabled = updatedStore.isNativeReplicationEnabled(); + updatedConfigsList.add(NATIVE_REPLICATION_ENABLED); + } else { + setStore.nativeReplicationEnabled = originalStore.isNativeReplicationEnabled(); } - if (storeViewConfig.isPresent()) { - // Validate and overwrite store views if they're getting set - validateStoreViewConfigs(storeViewConfig.get(), currStore); - setStore.views = StoreViewUtils.convertStringMapViewToStoreViewConfigRecordMap(storeViewConfig.get()); + if (updatedConfigs.contains(PUSH_STREAM_SOURCE_ADDRESS)) { + setStore.pushStreamSourceAddress = updatedStore.getPushStreamSourceAddress(); + updatedConfigsList.add(PUSH_STREAM_SOURCE_ADDRESS); + } else { + setStore.pushStreamSourceAddress = originalStore.getPushStreamSourceAddress(); + } + + if (updatedConfigs.contains(STORE_VIEW)) { + setStore.views = StoreViewUtils.convertViewConfigMapToStoreViewRecordMap(updatedStore.getViewConfigs()); updatedConfigsList.add(STORE_VIEW); } - // Only update fields that are set, other fields will be read from the original store's partitioner config. - PartitionerConfig updatedPartitionerConfig = VeniceHelixAdmin.mergeNewSettingsIntoOldPartitionerConfig( - currStore, - partitionerClass, - partitionerParams, - amplificationFactor); - if (partitionerClass.isPresent() || partitionerParams.isPresent() || amplificationFactor.isPresent()) { - // Update updatedConfigsList. - partitionerClass.ifPresent(p -> updatedConfigsList.add(PARTITIONER_CLASS)); - partitionerParams.ifPresent(p -> updatedConfigsList.add(PARTITIONER_PARAMS)); - amplificationFactor.ifPresent(p -> updatedConfigsList.add(AMPLIFICATION_FACTOR)); + boolean partitionerChange = false; + + if (updatedConfigs.contains(PARTITIONER_CLASS)) { + updatedConfigsList.add(PARTITIONER_CLASS); + partitionerChange = true; + } + + if (updatedConfigs.contains(PARTITIONER_PARAMS)) { + updatedConfigsList.add(PARTITIONER_PARAMS); + partitionerChange = true; + } + + if (updatedConfigs.contains(AMPLIFICATION_FACTOR)) { + updatedConfigsList.add(AMPLIFICATION_FACTOR); + partitionerChange = true; + } + + if (partitionerChange) { // Create PartitionConfigRecord for admin channel transmission. + PartitionerConfig updatedPartitionerConfig = updatedStore.getPartitionerConfig(); PartitionerConfigRecord partitionerConfigRecord = new PartitionerConfigRecord(); partitionerConfigRecord.partitionerClass = updatedPartitionerConfig.getPartitionerClass(); partitionerConfigRecord.partitionerParams = CollectionUtils.getCharSequenceMapFromStringMap(updatedPartitionerConfig.getPartitionerParams()); partitionerConfigRecord.amplificationFactor = updatedPartitionerConfig.getAmplificationFactor(); - // Before setting partitioner config, verify the updated partitionerConfig can be built - try { - PartitionUtils.getVenicePartitioner( - partitionerConfigRecord.partitionerClass.toString(), - new VeniceProperties(partitionerConfigRecord.partitionerParams), - getKeySchema(clusterName, storeName).getSchema()); - } catch (PartitionerSchemaMismatchException e) { - String errorMessage = errorMessagePrefix + e.getMessage(); - LOGGER.error(errorMessage); - throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_SCHEMA); - } catch (Exception e) { - String errorMessage = errorMessagePrefix + "Partitioner Configs invalid, please verify that partitioner " - + "configs like classpath and parameters are correct!"; - LOGGER.error(errorMessage); - throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); - } setStore.partitionerConfig = partitionerConfigRecord; } - setStore.enableReads = - readability.map(addToUpdatedConfigList(updatedConfigsList, ENABLE_READS)).orElseGet(currStore::isEnableReads); - setStore.enableWrites = writeability.map(addToUpdatedConfigList(updatedConfigsList, ENABLE_WRITES)) - .orElseGet(currStore::isEnableWrites); - - setStore.readQuotaInCU = readQuotaInCU.map(addToUpdatedConfigList(updatedConfigsList, READ_QUOTA_IN_CU)) - .orElseGet(currStore::getReadQuotaInCU); - - // We need to be careful when handling currentVersion. - // Since it is not synced between parent and local controller, - // It is very likely to override local values unintentionally. - setStore.currentVersion = - currentVersion.map(addToUpdatedConfigList(updatedConfigsList, VERSION)).orElse(IGNORED_CURRENT_VERSION); - - hybridRewindSeconds.map(addToUpdatedConfigList(updatedConfigsList, REWIND_TIME_IN_SECONDS)); - hybridOffsetLagThreshold.map(addToUpdatedConfigList(updatedConfigsList, OFFSET_LAG_TO_GO_ONLINE)); - hybridTimeLagThreshold.map(addToUpdatedConfigList(updatedConfigsList, TIME_LAG_TO_GO_ONLINE)); - hybridDataReplicationPolicy.map(addToUpdatedConfigList(updatedConfigsList, DATA_REPLICATION_POLICY)); - hybridBufferReplayPolicy.map(addToUpdatedConfigList(updatedConfigsList, BUFFER_REPLAY_POLICY)); - HybridStoreConfig updatedHybridStoreConfig = VeniceHelixAdmin.mergeNewSettingsIntoOldHybridStoreConfig( - currStore, - hybridRewindSeconds, - hybridOffsetLagThreshold, - hybridTimeLagThreshold, - hybridDataReplicationPolicy, - hybridBufferReplayPolicy); - - // Get VeniceControllerClusterConfig for the cluster - VeniceControllerClusterConfig clusterConfig = - veniceHelixAdmin.getHelixVeniceClusterResources(clusterName).getConfig(); - // Check if the store is being converted to a hybrid store - boolean storeBeingConvertedToHybrid = !currStore.isHybrid() && updatedHybridStoreConfig != null - && veniceHelixAdmin.isHybrid(updatedHybridStoreConfig); - // Check if the store is being converted to a batch store - boolean storeBeingConvertedToBatch = currStore.isHybrid() && !veniceHelixAdmin.isHybrid(updatedHybridStoreConfig); - if (storeBeingConvertedToBatch && activeActiveReplicationEnabled.orElse(false)) { - throw new VeniceHttpException( - HttpStatus.SC_BAD_REQUEST, - "Cannot convert store to batch-only and enable Active/Active together.", - ErrorType.BAD_REQUEST); - } - if (storeBeingConvertedToBatch && incrementalPushEnabled.orElse(false)) { - throw new VeniceHttpException( - HttpStatus.SC_BAD_REQUEST, - "Cannot convert store to batch-only and enable incremental push together.", - ErrorType.BAD_REQUEST); - } - // Update active-active replication config. - setStore.activeActiveReplicationEnabled = activeActiveReplicationEnabled - .map(addToUpdatedConfigList(updatedConfigsList, ACTIVE_ACTIVE_REPLICATION_ENABLED)) - .orElseGet(currStore::isActiveActiveReplicationEnabled); - // Enable active-active replication automatically when batch user store being converted to hybrid store and - // active-active replication is enabled for all hybrid store via the cluster config - if (storeBeingConvertedToHybrid && !setStore.activeActiveReplicationEnabled && !currStore.isSystemStore() - && clusterConfig.isActiveActiveReplicationEnabledAsDefaultForHybrid()) { - setStore.activeActiveReplicationEnabled = true; - updatedConfigsList.add(ACTIVE_ACTIVE_REPLICATION_ENABLED); - if (!hybridDataReplicationPolicy.isPresent()) { - LOGGER.info( - "Data replication policy was not explicitly set when converting store to hybrid store: {}." - + " Setting it to active-active replication policy.", - storeName); - - updatedHybridStoreConfig.setDataReplicationPolicy(DataReplicationPolicy.ACTIVE_ACTIVE); - updatedConfigsList.add(DATA_REPLICATION_POLICY); - } - } - // When turning off hybrid store, we will also turn off A/A store config. - if (storeBeingConvertedToBatch && setStore.activeActiveReplicationEnabled) { - setStore.activeActiveReplicationEnabled = false; - updatedConfigsList.add(ACTIVE_ACTIVE_REPLICATION_ENABLED); + if (updatedConfigs.contains(ENABLE_READS)) { + setStore.enableReads = updatedStore.isEnableReads(); + updatedConfigsList.add(ENABLE_READS); + } else { + setStore.enableReads = originalStore.isEnableReads(); } - // Update incremental push config. - setStore.incrementalPushEnabled = - incrementalPushEnabled.map(addToUpdatedConfigList(updatedConfigsList, INCREMENTAL_PUSH_ENABLED)) - .orElseGet(currStore::isIncrementalPushEnabled); - // Enable incremental push automatically when batch user store being converted to hybrid store and active-active - // replication is enabled or being and the cluster config allows it. - if (!setStore.incrementalPushEnabled && !currStore.isSystemStore() && storeBeingConvertedToHybrid - && setStore.activeActiveReplicationEnabled - && clusterConfig.enabledIncrementalPushForHybridActiveActiveUserStores()) { - setStore.incrementalPushEnabled = true; - updatedConfigsList.add(INCREMENTAL_PUSH_ENABLED); + if (updatedConfigs.contains(ENABLE_WRITES)) { + setStore.enableWrites = updatedStore.isEnableWrites(); + updatedConfigsList.add(ENABLE_WRITES); + } else { + setStore.enableWrites = originalStore.isEnableWrites(); } - // When turning off hybrid store, we will also turn off incremental store config. - if (storeBeingConvertedToBatch && setStore.incrementalPushEnabled) { - setStore.incrementalPushEnabled = false; - updatedConfigsList.add(INCREMENTAL_PUSH_ENABLED); + + if (updatedConfigs.contains(READ_QUOTA_IN_CU)) { + setStore.readQuotaInCU = updatedStore.getReadQuotaInCU(); + updatedConfigsList.add(READ_QUOTA_IN_CU); + } else { + setStore.readQuotaInCU = originalStore.getReadQuotaInCU(); } - if (updatedHybridStoreConfig == null) { - setStore.hybridStoreConfig = null; + if (updatedConfigsList.contains(VERSION)) { + setStore.currentVersion = updatedStore.getCurrentVersion(); + updatedConfigsList.add(VERSION); } else { - HybridStoreConfigRecord hybridStoreConfigRecord = new HybridStoreConfigRecord(); - hybridStoreConfigRecord.offsetLagThresholdToGoOnline = - updatedHybridStoreConfig.getOffsetLagThresholdToGoOnline(); - hybridStoreConfigRecord.rewindTimeInSeconds = updatedHybridStoreConfig.getRewindTimeInSeconds(); - hybridStoreConfigRecord.producerTimestampLagThresholdToGoOnlineInSeconds = - updatedHybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds(); - hybridStoreConfigRecord.dataReplicationPolicy = updatedHybridStoreConfig.getDataReplicationPolicy().getValue(); - hybridStoreConfigRecord.bufferReplayPolicy = updatedHybridStoreConfig.getBufferReplayPolicy().getValue(); - setStore.hybridStoreConfig = hybridStoreConfigRecord; + setStore.currentVersion = IGNORED_CURRENT_VERSION; } - if (incrementalPushEnabled.orElse(currStore.isIncrementalPushEnabled()) - && !veniceHelixAdmin.isHybrid(currStore.getHybridStoreConfig()) - && !veniceHelixAdmin.isHybrid(updatedHybridStoreConfig)) { - LOGGER.info( - "Enabling incremental push for a batch store:{}. Converting it to a hybrid store with default configs.", - storeName); - HybridStoreConfigRecord hybridStoreConfigRecord = new HybridStoreConfigRecord(); - hybridStoreConfigRecord.rewindTimeInSeconds = DEFAULT_REWIND_TIME_IN_SECONDS; + if (updatedConfigs.contains(REWIND_TIME_IN_SECONDS)) { updatedConfigsList.add(REWIND_TIME_IN_SECONDS); - hybridStoreConfigRecord.offsetLagThresholdToGoOnline = DEFAULT_HYBRID_OFFSET_LAG_THRESHOLD; + } + + if (updatedConfigs.contains(OFFSET_LAG_TO_GO_ONLINE)) { updatedConfigsList.add(OFFSET_LAG_TO_GO_ONLINE); - hybridStoreConfigRecord.producerTimestampLagThresholdToGoOnlineInSeconds = DEFAULT_HYBRID_TIME_LAG_THRESHOLD; + } + + if (updatedConfigs.contains(TIME_LAG_TO_GO_ONLINE)) { updatedConfigsList.add(TIME_LAG_TO_GO_ONLINE); - hybridStoreConfigRecord.dataReplicationPolicy = DataReplicationPolicy.NONE.getValue(); + } + + if (updatedConfigs.contains(DATA_REPLICATION_POLICY)) { updatedConfigsList.add(DATA_REPLICATION_POLICY); - hybridStoreConfigRecord.bufferReplayPolicy = BufferReplayPolicy.REWIND_FROM_EOP.getValue(); + } + + if (updatedConfigs.contains(BUFFER_REPLAY_POLICY)) { updatedConfigsList.add(BUFFER_REPLAY_POLICY); - setStore.hybridStoreConfig = hybridStoreConfigRecord; + } + + HybridStoreConfig updatedHybridStoreConfig = updatedStore.getHybridStoreConfig(); + setStore.hybridStoreConfig = new HybridStoreConfigRecord(); + if (updatedHybridStoreConfig == null) { + setStore.hybridStoreConfig.offsetLagThresholdToGoOnline = -1; + setStore.hybridStoreConfig.rewindTimeInSeconds = -1; + setStore.hybridStoreConfig.producerTimestampLagThresholdToGoOnlineInSeconds = -1; + setStore.hybridStoreConfig.dataReplicationPolicy = DataReplicationPolicy.NONE.getValue(); + setStore.hybridStoreConfig.bufferReplayPolicy = BufferReplayPolicy.REWIND_FROM_EOP.getValue(); + } else { + setStore.hybridStoreConfig.offsetLagThresholdToGoOnline = + updatedHybridStoreConfig.getOffsetLagThresholdToGoOnline(); + setStore.hybridStoreConfig.rewindTimeInSeconds = updatedHybridStoreConfig.getRewindTimeInSeconds(); + setStore.hybridStoreConfig.producerTimestampLagThresholdToGoOnlineInSeconds = + updatedHybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds(); + setStore.hybridStoreConfig.dataReplicationPolicy = + updatedHybridStoreConfig.getDataReplicationPolicy().getValue(); + setStore.hybridStoreConfig.bufferReplayPolicy = updatedHybridStoreConfig.getBufferReplayPolicy().getValue(); + } + + if (updatedConfigs.contains(ACTIVE_ACTIVE_REPLICATION_ENABLED)) { + setStore.activeActiveReplicationEnabled = updatedStore.isActiveActiveReplicationEnabled(); + updatedConfigsList.add(ACTIVE_ACTIVE_REPLICATION_ENABLED); + } else { + setStore.activeActiveReplicationEnabled = originalStore.isActiveActiveReplicationEnabled(); + } + + if (updatedConfigs.contains(INCREMENTAL_PUSH_ENABLED)) { + setStore.incrementalPushEnabled = updatedStore.isIncrementalPushEnabled(); + updatedConfigsList.add(INCREMENTAL_PUSH_ENABLED); + } else { + setStore.incrementalPushEnabled = originalStore.isIncrementalPushEnabled(); } /** @@ -2498,204 +2319,248 @@ public void updateStore(String clusterName, String storeName, UpdateStoreQueryPa * do append-only and compaction will happen later. * We expose actual disk usage to users, instead of multiplying/dividing the overhead ratio by situations. */ - setStore.storageQuotaInByte = - storageQuotaInByte.map(addToUpdatedConfigList(updatedConfigsList, STORAGE_QUOTA_IN_BYTE)) - .orElseGet(currStore::getStorageQuotaInByte); - - setStore.accessControlled = accessControlled.map(addToUpdatedConfigList(updatedConfigsList, ACCESS_CONTROLLED)) - .orElseGet(currStore::isAccessControlled); - setStore.compressionStrategy = - compressionStrategy.map(addToUpdatedConfigList(updatedConfigsList, COMPRESSION_STRATEGY)) - .map(CompressionStrategy::getValue) - .orElse(currStore.getCompressionStrategy().getValue()); - setStore.clientDecompressionEnabled = - clientDecompressionEnabled.map(addToUpdatedConfigList(updatedConfigsList, CLIENT_DECOMPRESSION_ENABLED)) - .orElseGet(currStore::getClientDecompressionEnabled); - setStore.batchGetLimit = batchGetLimit.map(addToUpdatedConfigList(updatedConfigsList, BATCH_GET_LIMIT)) - .orElseGet(currStore::getBatchGetLimit); - setStore.numVersionsToPreserve = - numVersionsToPreserve.map(addToUpdatedConfigList(updatedConfigsList, NUM_VERSIONS_TO_PRESERVE)) - .orElseGet(currStore::getNumVersionsToPreserve); - setStore.isMigrating = storeMigration.map(addToUpdatedConfigList(updatedConfigsList, STORE_MIGRATION)) - .orElseGet(currStore::isMigrating); - setStore.replicationMetadataVersionID = replicationMetadataVersionID - .map(addToUpdatedConfigList(updatedConfigsList, REPLICATION_METADATA_PROTOCOL_VERSION_ID)) - .orElse(currStore.getRmdVersion()); - setStore.readComputationEnabled = - readComputationEnabled.map(addToUpdatedConfigList(updatedConfigsList, READ_COMPUTATION_ENABLED)) - .orElseGet(currStore::isReadComputationEnabled); - setStore.bootstrapToOnlineTimeoutInHours = bootstrapToOnlineTimeoutInHours - .map(addToUpdatedConfigList(updatedConfigsList, BOOTSTRAP_TO_ONLINE_TIMEOUT_IN_HOURS)) - .orElseGet(currStore::getBootstrapToOnlineTimeoutInHours); + if (updatedConfigs.contains(STORAGE_QUOTA_IN_BYTE)) { + setStore.storageQuotaInByte = updatedStore.getStorageQuotaInByte(); + updatedConfigsList.add(STORAGE_QUOTA_IN_BYTE); + } else { + setStore.storageQuotaInByte = originalStore.getStorageQuotaInByte(); + } + + if (updatedConfigs.contains(ACCESS_CONTROLLED)) { + setStore.accessControlled = updatedStore.isAccessControlled(); + updatedConfigsList.add(ACCESS_CONTROLLED); + } else { + setStore.accessControlled = originalStore.isAccessControlled(); + } + + if (updatedConfigs.contains(COMPRESSION_STRATEGY)) { + setStore.compressionStrategy = updatedStore.getCompressionStrategy().getValue(); + updatedConfigsList.add(COMPRESSION_STRATEGY); + } else { + setStore.compressionStrategy = originalStore.getCompressionStrategy().getValue(); + } + + if (updatedConfigs.contains(CLIENT_DECOMPRESSION_ENABLED)) { + setStore.clientDecompressionEnabled = updatedStore.getClientDecompressionEnabled(); + updatedConfigsList.add(CLIENT_DECOMPRESSION_ENABLED); + } else { + setStore.clientDecompressionEnabled = originalStore.getClientDecompressionEnabled(); + } + + if (updatedConfigs.contains(BATCH_GET_LIMIT)) { + setStore.batchGetLimit = updatedStore.getBatchGetLimit(); + updatedConfigsList.add(BATCH_GET_LIMIT); + } else { + setStore.batchGetLimit = originalStore.getBatchGetLimit(); + } + + if (updatedConfigs.contains(NUM_VERSIONS_TO_PRESERVE)) { + setStore.numVersionsToPreserve = updatedStore.getNumVersionsToPreserve(); + updatedConfigsList.add(NUM_VERSIONS_TO_PRESERVE); + } else { + setStore.numVersionsToPreserve = originalStore.getNumVersionsToPreserve(); + } + + if (updatedConfigs.contains(STORE_MIGRATION)) { + setStore.isMigrating = updatedStore.isMigrating(); + updatedConfigsList.add(STORE_MIGRATION); + } else { + setStore.isMigrating = originalStore.isMigrating(); + } + + if (updatedConfigs.contains(REPLICATION_METADATA_PROTOCOL_VERSION_ID)) { + setStore.replicationMetadataVersionID = updatedStore.getRmdVersion(); + updatedConfigsList.add(REPLICATION_METADATA_PROTOCOL_VERSION_ID); + } else { + setStore.replicationMetadataVersionID = originalStore.getRmdVersion(); + } + + if (updatedConfigs.contains(READ_COMPUTATION_ENABLED)) { + setStore.readComputationEnabled = updatedStore.isReadComputationEnabled(); + updatedConfigsList.add(READ_COMPUTATION_ENABLED); + } else { + setStore.readComputationEnabled = originalStore.isReadComputationEnabled(); + } + + if (updatedConfigs.contains(BOOTSTRAP_TO_ONLINE_TIMEOUT_IN_HOURS)) { + setStore.bootstrapToOnlineTimeoutInHours = updatedStore.getBootstrapToOnlineTimeoutInHours(); + updatedConfigsList.add(BOOTSTRAP_TO_ONLINE_TIMEOUT_IN_HOURS); + } else { + setStore.bootstrapToOnlineTimeoutInHours = originalStore.getBootstrapToOnlineTimeoutInHours(); + } + setStore.leaderFollowerModelEnabled = true; // do not mess up during upgrades - setStore.backupStrategy = (backupStrategy.map(addToUpdatedConfigList(updatedConfigsList, BACKUP_STRATEGY)) - .orElse(currStore.getBackupStrategy())).ordinal(); - - setStore.schemaAutoRegisterFromPushJobEnabled = autoSchemaRegisterPushJobEnabled - .map(addToUpdatedConfigList(updatedConfigsList, AUTO_SCHEMA_REGISTER_FOR_PUSHJOB_ENABLED)) - .orElse(currStore.isSchemaAutoRegisterFromPushJobEnabled()); - - setStore.hybridStoreDiskQuotaEnabled = - hybridStoreDiskQuotaEnabled.map(addToUpdatedConfigList(updatedConfigsList, HYBRID_STORE_DISK_QUOTA_ENABLED)) - .orElse(currStore.isHybridStoreDiskQuotaEnabled()); - - regularVersionETLEnabled.map(addToUpdatedConfigList(updatedConfigsList, REGULAR_VERSION_ETL_ENABLED)); - futureVersionETLEnabled.map(addToUpdatedConfigList(updatedConfigsList, FUTURE_VERSION_ETL_ENABLED)); - etledUserProxyAccount.map(addToUpdatedConfigList(updatedConfigsList, ETLED_PROXY_USER_ACCOUNT)); - setStore.ETLStoreConfig = mergeNewSettingIntoOldETLStoreConfig( - currStore, - regularVersionETLEnabled, - futureVersionETLEnabled, - etledUserProxyAccount); - - setStore.largestUsedVersionNumber = - largestUsedVersionNumber.map(addToUpdatedConfigList(updatedConfigsList, LARGEST_USED_VERSION_NUMBER)) - .orElseGet(currStore::getLargestUsedVersionNumber); - - setStore.backupVersionRetentionMs = - backupVersionRetentionMs.map(addToUpdatedConfigList(updatedConfigsList, BACKUP_VERSION_RETENTION_MS)) - .orElseGet(currStore::getBackupVersionRetentionMs); - setStore.replicationFactor = replicationFactor.map(addToUpdatedConfigList(updatedConfigsList, REPLICATION_FACTOR)) - .orElseGet(currStore::getReplicationFactor); - setStore.migrationDuplicateStore = - migrationDuplicateStore.map(addToUpdatedConfigList(updatedConfigsList, MIGRATION_DUPLICATE_STORE)) - .orElseGet(currStore::isMigrationDuplicateStore); - setStore.nativeReplicationSourceFabric = nativeReplicationSourceFabric - .map(addToUpdatedConfigList(updatedConfigsList, NATIVE_REPLICATION_SOURCE_FABRIC)) - .orElseGet((currStore::getNativeReplicationSourceFabric)); - - setStore.disableMetaStore = - params.disableMetaStore().map(addToUpdatedConfigList(updatedConfigsList, DISABLE_META_STORE)).orElse(false); - - setStore.disableDavinciPushStatusStore = params.disableDavinciPushStatusStore() - .map(addToUpdatedConfigList(updatedConfigsList, DISABLE_DAVINCI_PUSH_STATUS_STORE)) - .orElse(false); - - setStore.storagePersona = personaName.map(addToUpdatedConfigList(updatedConfigsList, PERSONA_NAME)).orElse(null); - - setStore.blobTransferEnabled = params.getBlobTransferEnabled() - .map(addToUpdatedConfigList(updatedConfigsList, BLOB_TRANSFER_ENABLED)) - .orElseGet(currStore::isBlobTransferEnabled); - - // Check whether the passed param is valid or not - if (latestSupersetSchemaId.isPresent()) { - if (latestSupersetSchemaId.get() != SchemaData.INVALID_VALUE_SCHEMA_ID) { - if (veniceHelixAdmin.getValueSchema(clusterName, storeName, latestSupersetSchemaId.get()) == null) { - throw new VeniceException( - "Unknown value schema id: " + latestSupersetSchemaId.get() + " in store: " + storeName); - } - } + + if (updatedConfigs.contains(BACKUP_STRATEGY)) { + setStore.backupStrategy = updatedStore.getBackupStrategy().getValue(); + updatedConfigsList.add(BACKUP_STRATEGY); + } else { + setStore.backupStrategy = originalStore.getBackupStrategy().getValue(); } - setStore.latestSuperSetValueSchemaId = - latestSupersetSchemaId.map(addToUpdatedConfigList(updatedConfigsList, LATEST_SUPERSET_SCHEMA_ID)) - .orElseGet(currStore::getLatestSuperSetValueSchemaId); - setStore.storageNodeReadQuotaEnabled = - storageNodeReadQuotaEnabled.map(addToUpdatedConfigList(updatedConfigsList, STORAGE_NODE_READ_QUOTA_ENABLED)) - .orElseGet(currStore::isStorageNodeReadQuotaEnabled); - setStore.unusedSchemaDeletionEnabled = - unusedSchemaDeletionEnabled.map(addToUpdatedConfigList(updatedConfigsList, UNUSED_SCHEMA_DELETION_ENABLED)) - .orElseGet(currStore::isUnusedSchemaDeletionEnabled); - setStore.minCompactionLagSeconds = - minCompactionLagSeconds.map(addToUpdatedConfigList(updatedConfigsList, MIN_COMPACTION_LAG_SECONDS)) - .orElseGet(currStore::getMinCompactionLagSeconds); - setStore.maxCompactionLagSeconds = - maxCompactionLagSeconds.map(addToUpdatedConfigList(updatedConfigsList, MAX_COMPACTION_LAG_SECONDS)) - .orElseGet(currStore::getMaxCompactionLagSeconds); - if (setStore.maxCompactionLagSeconds < setStore.minCompactionLagSeconds) { - throw new VeniceException( - "Store's max compaction lag seconds: " + setStore.maxCompactionLagSeconds + " shouldn't be smaller than " - + "store's min compaction lag seconds: " + setStore.minCompactionLagSeconds); - } - setStore.maxRecordSizeBytes = - maxRecordSizeBytes.map(addToUpdatedConfigList(updatedConfigsList, MAX_RECORD_SIZE_BYTES)) - .orElseGet(currStore::getMaxRecordSizeBytes); - - StoragePersonaRepository repository = - getVeniceHelixAdmin().getHelixVeniceClusterResources(clusterName).getStoragePersonaRepository(); - StoragePersona personaToValidate = null; - StoragePersona existingPersona = repository.getPersonaContainingStore(currStore.getName()); - - if (params.getStoragePersona().isPresent()) { - personaToValidate = getVeniceHelixAdmin().getStoragePersona(clusterName, params.getStoragePersona().get()); - if (personaToValidate == null) { - String errMsg = "UpdateStore command failed for store " + storeName + ". The provided StoragePersona " - + params.getStoragePersona().get() + " does not exist."; - throw new VeniceException(errMsg); - } - } else if (existingPersona != null) { - personaToValidate = existingPersona; + + if (updatedConfigs.contains(AUTO_SCHEMA_REGISTER_FOR_PUSHJOB_ENABLED)) { + setStore.schemaAutoRegisterFromPushJobEnabled = updatedStore.isSchemaAutoRegisterFromPushJobEnabled(); + updatedConfigsList.add(AUTO_SCHEMA_REGISTER_FOR_PUSHJOB_ENABLED); + } else { + setStore.schemaAutoRegisterFromPushJobEnabled = originalStore.isSchemaAutoRegisterFromPushJobEnabled(); } - if (personaToValidate != null) { - /** - * Create a new copy of the store with an updated quota, and validate this. - */ - Store updatedQuotaStore = getVeniceHelixAdmin().getStore(clusterName, storeName); - updatedQuotaStore.setStorageQuotaInByte(setStore.getStorageQuotaInByte()); - repository.validateAddUpdatedStore(personaToValidate, Optional.of(updatedQuotaStore)); + if (updatedConfigs.contains(HYBRID_STORE_DISK_QUOTA_ENABLED)) { + setStore.hybridStoreDiskQuotaEnabled = updatedStore.isHybridStoreDiskQuotaEnabled(); + updatedConfigsList.add(HYBRID_STORE_DISK_QUOTA_ENABLED); + } else { + setStore.hybridStoreDiskQuotaEnabled = originalStore.isHybridStoreDiskQuotaEnabled(); } - /** - * Fabrics filter is not a store config, so we don't need to add it into {@link UpdateStore#updatedConfigsList} - */ - setStore.regionsFilter = regionsFilter.orElse(null); + if (updatedConfigs.contains(REGULAR_VERSION_ETL_ENABLED)) { + updatedConfigsList.add(REGULAR_VERSION_ETL_ENABLED); + } - // Update Partial Update config. - boolean partialUpdateConfigUpdated = ParentControllerConfigUpdateUtils.checkAndMaybeApplyPartialUpdateConfig( - this, - clusterName, - storeName, - writeComputationEnabled, - setStore, - storeBeingConvertedToHybrid); - if (partialUpdateConfigUpdated) { + if (updatedConfigs.contains(FUTURE_VERSION_ETL_ENABLED)) { + updatedConfigsList.add(FUTURE_VERSION_ETL_ENABLED); + } + + if (updatedConfigs.contains(ETLED_PROXY_USER_ACCOUNT)) { + updatedConfigsList.add(ETLED_PROXY_USER_ACCOUNT); + } + + ETLStoreConfig etlStoreConfig = updatedStore.getEtlStoreConfig(); + ETLStoreConfigRecord etlStoreConfigRecord = new ETLStoreConfigRecord(); + etlStoreConfigRecord.regularVersionETLEnabled = etlStoreConfig.isRegularVersionETLEnabled(); + etlStoreConfigRecord.futureVersionETLEnabled = etlStoreConfig.isFutureVersionETLEnabled(); + etlStoreConfigRecord.etledUserProxyAccount = etlStoreConfig.getEtledUserProxyAccount(); + setStore.ETLStoreConfig = etlStoreConfigRecord; + + if (updatedConfigs.contains(LARGEST_USED_VERSION_NUMBER)) { + setStore.largestUsedVersionNumber = updatedStore.getLargestUsedVersionNumber(); + updatedConfigsList.add(LARGEST_USED_VERSION_NUMBER); + } else { + setStore.largestUsedVersionNumber = originalStore.getLargestUsedVersionNumber(); + } + + if (updatedConfigs.contains(BACKUP_VERSION_RETENTION_MS)) { + setStore.backupVersionRetentionMs = updatedStore.getBackupVersionRetentionMs(); + updatedConfigsList.add(BACKUP_VERSION_RETENTION_MS); + } else { + setStore.backupVersionRetentionMs = originalStore.getBackupVersionRetentionMs(); + } + + if (updatedConfigs.contains(REPLICATION_FACTOR)) { + setStore.replicationFactor = updatedStore.getReplicationFactor(); + updatedConfigsList.add(REPLICATION_FACTOR); + } else { + setStore.replicationFactor = originalStore.getReplicationFactor(); + } + + if (updatedConfigs.contains(MIGRATION_DUPLICATE_STORE)) { + setStore.migrationDuplicateStore = updatedStore.isMigrationDuplicateStore(); + updatedConfigsList.add(MIGRATION_DUPLICATE_STORE); + } else { + setStore.migrationDuplicateStore = originalStore.isMigrationDuplicateStore(); + } + + if (updatedConfigs.contains(NATIVE_REPLICATION_SOURCE_FABRIC)) { + setStore.nativeReplicationSourceFabric = updatedStore.getNativeReplicationSourceFabric(); + updatedConfigsList.add(NATIVE_REPLICATION_SOURCE_FABRIC); + } else { + setStore.nativeReplicationSourceFabric = originalStore.getNativeReplicationSourceFabric(); + } + + if (updatedConfigs.contains(DISABLE_META_STORE)) { + setStore.disableMetaStore = !updatedStore.isStoreMetaSystemStoreEnabled(); + updatedConfigsList.add(DISABLE_META_STORE); + } else { + setStore.disableMetaStore = !originalStore.isStoreMetaSystemStoreEnabled(); + } + + if (updatedConfigs.contains(DISABLE_DAVINCI_PUSH_STATUS_STORE)) { + setStore.disableDavinciPushStatusStore = !updatedStore.isDaVinciPushStatusStoreEnabled(); + updatedConfigsList.add(DISABLE_DAVINCI_PUSH_STATUS_STORE); + } else { + setStore.disableDavinciPushStatusStore = !originalStore.isDaVinciPushStatusStoreEnabled(); + } + + if (updatedConfigs.contains(PERSONA_NAME)) { + setStore.storagePersona = params.getStoragePersona().get(); + updatedConfigsList.add(PERSONA_NAME); + } else { + setStore.storagePersona = null; + } + + if (updatedConfigs.contains(BLOB_TRANSFER_ENABLED)) { + setStore.blobTransferEnabled = updatedStore.isBlobTransferEnabled(); + updatedConfigsList.add(BLOB_TRANSFER_ENABLED); + } else { + setStore.blobTransferEnabled = originalStore.isBlobTransferEnabled(); + } + + if (updatedConfigs.contains(MAX_RECORD_SIZE_BYTES)) { + setStore.maxRecordSizeBytes = updatedStore.getMaxRecordSizeBytes(); + updatedConfigsList.add(MAX_RECORD_SIZE_BYTES); + } else { + setStore.maxRecordSizeBytes = originalStore.getMaxRecordSizeBytes(); + } + + if (updatedConfigs.contains(LATEST_SUPERSET_SCHEMA_ID)) { + setStore.latestSuperSetValueSchemaId = updatedStore.getLatestSuperSetValueSchemaId(); + updatedConfigsList.add(LATEST_SUPERSET_SCHEMA_ID); + } else { + setStore.latestSuperSetValueSchemaId = originalStore.getLatestSuperSetValueSchemaId(); + } + + if (updatedConfigs.contains(STORAGE_NODE_READ_QUOTA_ENABLED)) { + setStore.storageNodeReadQuotaEnabled = updatedStore.isStorageNodeReadQuotaEnabled(); + updatedConfigsList.add(STORAGE_NODE_READ_QUOTA_ENABLED); + } else { + setStore.storageNodeReadQuotaEnabled = originalStore.isStorageNodeReadQuotaEnabled(); + } + + if (updatedConfigs.contains(UNUSED_SCHEMA_DELETION_ENABLED)) { + setStore.unusedSchemaDeletionEnabled = updatedStore.isUnusedSchemaDeletionEnabled(); + updatedConfigsList.add(UNUSED_SCHEMA_DELETION_ENABLED); + } else { + setStore.unusedSchemaDeletionEnabled = originalStore.isUnusedSchemaDeletionEnabled(); + } + + if (updatedConfigs.contains(MIN_COMPACTION_LAG_SECONDS)) { + setStore.minCompactionLagSeconds = updatedStore.getMinCompactionLagSeconds(); + updatedConfigsList.add(MIN_COMPACTION_LAG_SECONDS); + } else { + setStore.minCompactionLagSeconds = originalStore.getMinCompactionLagSeconds(); + } + + if (updatedConfigs.contains(MAX_COMPACTION_LAG_SECONDS)) { + setStore.maxCompactionLagSeconds = updatedStore.getMaxCompactionLagSeconds(); + updatedConfigsList.add(MAX_COMPACTION_LAG_SECONDS); + } else { + setStore.maxCompactionLagSeconds = originalStore.getMaxCompactionLagSeconds(); + } + + if (updatedConfigs.contains(WRITE_COMPUTATION_ENABLED)) { + setStore.writeComputationEnabled = updatedStore.isWriteComputationEnabled(); updatedConfigsList.add(WRITE_COMPUTATION_ENABLED); + } else { + setStore.writeComputationEnabled = originalStore.isWriteComputationEnabled(); } - boolean partialUpdateJustEnabled = setStore.writeComputationEnabled && !currStore.isWriteComputationEnabled(); - // Update Chunking config. - boolean chunkingConfigUpdated = ParentControllerConfigUpdateUtils - .checkAndMaybeApplyChunkingConfigChange(this, clusterName, storeName, chunkingEnabled, setStore); - if (chunkingConfigUpdated) { + + if (updatedConfigs.contains(CHUNKING_ENABLED)) { + setStore.chunkingEnabled = updatedStore.isChunkingEnabled(); updatedConfigsList.add(CHUNKING_ENABLED); + } else { + setStore.chunkingEnabled = originalStore.isChunkingEnabled(); } - // Update RMD Chunking config. - boolean rmdChunkingConfigUpdated = ParentControllerConfigUpdateUtils - .checkAndMaybeApplyRmdChunkingConfigChange(this, clusterName, storeName, rmdChunkingEnabled, setStore); - if (rmdChunkingConfigUpdated) { + if (updatedConfigs.contains(RMD_CHUNKING_ENABLED)) { + setStore.rmdChunkingEnabled = updatedStore.isRmdChunkingEnabled(); updatedConfigsList.add(RMD_CHUNKING_ENABLED); + } else { + setStore.rmdChunkingEnabled = originalStore.isRmdChunkingEnabled(); } - // Validate Amplification Factor config based on latest A/A and partial update status. - if ((setStore.getActiveActiveReplicationEnabled() || setStore.getWriteComputationEnabled()) - && updatedPartitionerConfig.getAmplificationFactor() > 1) { - throw new VeniceHttpException( - HttpStatus.SC_BAD_REQUEST, - "Non-default amplification factor is not compatible with active-active replication and/or partial update.", - ErrorType.BAD_REQUEST); - } - - if (!getVeniceHelixAdmin().isHybrid(currStore.getHybridStoreConfig()) - && getVeniceHelixAdmin().isHybrid(setStore.getHybridStoreConfig()) && setStore.getPartitionNum() == 0) { - // This is a new hybrid store and partition count is not specified. - VeniceControllerClusterConfig config = - getVeniceHelixAdmin().getHelixVeniceClusterResources(clusterName).getConfig(); - setStore.setPartitionNum( - PartitionUtils.calculatePartitionCount( - storeName, - setStore.getStorageQuotaInByte(), - 0, - config.getPartitionSize(), - config.getMinNumberOfPartitionsForHybrid(), - config.getMaxNumberOfPartitions(), - config.isPartitionCountRoundUpEnabled(), - config.getPartitionCountRoundUpSize())); - LOGGER.info( - "Enforcing default hybrid partition count:{} for a new hybrid store:{}.", - setStore.getPartitionNum(), - storeName); - updatedConfigsList.add(PARTITION_COUNT); - } + /** + * Fabrics filter is not a store config, so we don't need to add it into {@link UpdateStore#updatedConfigsList} + */ + setStore.regionsFilter = params.getRegionsFilter().orElse(null); /** * By default, parent controllers will not try to replicate the unchanged store configs to child controllers; @@ -2703,90 +2568,30 @@ && getVeniceHelixAdmin().isHybrid(setStore.getHybridStoreConfig()) && setStore.g */ setStore.replicateAllConfigs = replicateAllConfigs; if (!replicateAllConfigs) { - if (updatedConfigsList.isEmpty()) { - String errMsg = - "UpdateStore command failed for store " + storeName + ". The command didn't change any specific" - + " store config and didn't specify \"--replicate-all-configs\" flag."; - LOGGER.error(errMsg); - throw new VeniceException(errMsg); - } setStore.updatedConfigsList = new ArrayList<>(updatedConfigsList); } else { setStore.updatedConfigsList = Collections.emptyList(); } - final boolean readComputeJustEnabled = - readComputationEnabled.orElse(false) && !currStore.isReadComputationEnabled(); - boolean needToGenerateSupersetSchema = - !currStore.isSystemStore() && (readComputeJustEnabled || partialUpdateJustEnabled); - if (needToGenerateSupersetSchema) { - // dry run to make sure superset schema generation can work - getSupersetSchemaGenerator(clusterName) - .generateSupersetSchemaFromSchemas(getValueSchemas(clusterName, storeName)); - } - AdminOperation message = new AdminOperation(); message.operationType = AdminMessageType.UPDATE_STORE.getValue(); message.payloadUnion = setStore; sendAdminMessageAndWaitForConsumed(clusterName, storeName, message); - - if (needToGenerateSupersetSchema) { - addSupersetSchemaForStore(clusterName, storeName, currStore.isActiveActiveReplicationEnabled()); - } - if (partialUpdateJustEnabled) { - LOGGER.info("Enabling partial update for the first time on store: {} in cluster: {}", storeName, clusterName); - addUpdateSchemaForStore(this, clusterName, storeName, false); - } - - /** - * If active-active replication is getting enabled for the store, generate and register the Replication metadata schema - * for all existing value schemas. - */ - final boolean activeActiveReplicationJustEnabled = - activeActiveReplicationEnabled.orElse(false) && !currStore.isActiveActiveReplicationEnabled(); - if (activeActiveReplicationJustEnabled) { - updateReplicationMetadataSchemaForAllValueSchema(clusterName, storeName); - } + UpdateStoreUtils.handlePostUpdateActions(this, clusterName, storeName); } finally { releaseAdminMessageLock(clusterName, storeName); } } - private void validateStoreViewConfigs(Map stringMap, Store store) { - Map configs = StoreViewUtils.convertStringMapViewToViewConfigMap(stringMap); - for (Map.Entry viewConfigEntry: configs.entrySet()) { - validateStoreViewConfig(store, viewConfigEntry.getValue()); - } - } - - private void validateStoreViewConfig(Store store, ViewConfig viewConfig) { - // TODO: Pass a proper properties object here. Today this isn't used in this context - VeniceView view = - ViewUtils.getVeniceView(viewConfig.getViewClassName(), new Properties(), store, viewConfig.getViewParameters()); - view.validateConfigs(); - } - - private SupersetSchemaGenerator getSupersetSchemaGenerator(String clusterName) { - if (externalSupersetSchemaGenerator.isPresent() && getMultiClusterConfigs().getControllerConfig(clusterName) - .isParentExternalSupersetSchemaGenerationEnabled()) { + @Override + public SupersetSchemaGenerator getSupersetSchemaGenerator(String clusterName) { + if (externalSupersetSchemaGenerator.isPresent() + && getMultiClusterConfigs().getControllerConfig(clusterName).isExternalSupersetSchemaGenerationEnabled()) { return externalSupersetSchemaGenerator.get(); } return defaultSupersetSchemaGenerator; } - private void addSupersetSchemaForStore(String clusterName, String storeName, boolean activeActiveReplicationEnabled) { - // Generate a superset schema and add it. - SchemaEntry supersetSchemaEntry = getSupersetSchemaGenerator(clusterName) - .generateSupersetSchemaFromSchemas(getValueSchemas(clusterName, storeName)); - final Schema supersetSchema = supersetSchemaEntry.getSchema(); - final int supersetSchemaID = supersetSchemaEntry.getId(); - addValueSchemaEntry(clusterName, storeName, supersetSchema.toString(), supersetSchemaID, true); - - if (activeActiveReplicationEnabled) { - updateReplicationMetadataSchema(clusterName, storeName, supersetSchema, supersetSchemaID); - } - } - /** * @see VeniceHelixAdmin#updateClusterConfig(String, UpdateClusterConfigQueryParams) */ @@ -2795,28 +2600,6 @@ public void updateClusterConfig(String clusterName, UpdateClusterConfigQueryPara getVeniceHelixAdmin().updateClusterConfig(clusterName, params); } - private void validateActiveActiveReplicationEnableConfigs( - Optional activeActiveReplicationEnabledOptional, - Optional nativeReplicationEnabledOptional, - Store store) { - final boolean activeActiveReplicationEnabled = activeActiveReplicationEnabledOptional.orElse(false); - if (!activeActiveReplicationEnabled) { - return; - } - - final boolean nativeReplicationEnabled = nativeReplicationEnabledOptional.isPresent() - ? nativeReplicationEnabledOptional.get() - : store.isNativeReplicationEnabled(); - - if (!nativeReplicationEnabled) { - throw new VeniceHttpException( - HttpStatus.SC_BAD_REQUEST, - "Active/Active Replication cannot be enabled for store " + store.getName() - + " since Native Replication is not enabled on it.", - ErrorType.INVALID_CONFIG); - } - } - /** * @see VeniceHelixAdmin#getStorageEngineOverheadRatio(String) */ @@ -2908,102 +2691,11 @@ public SchemaEntry addValueSchema( } } - private SchemaEntry addValueAndSupersetSchemaEntries( - String clusterName, - String storeName, - SchemaEntry newValueSchemaEntry, - SchemaEntry newSupersetSchemaEntry, - final boolean isWriteComputationEnabled) { - validateNewSupersetAndValueSchemaEntries(storeName, clusterName, newValueSchemaEntry, newSupersetSchemaEntry); - LOGGER.info( - "Adding value schema {} and superset schema {} to store: {} in cluster: {}", - newValueSchemaEntry, - newSupersetSchemaEntry, - storeName, - clusterName); - - SupersetSchemaCreation supersetSchemaCreation = - (SupersetSchemaCreation) AdminMessageType.SUPERSET_SCHEMA_CREATION.getNewInstance(); - supersetSchemaCreation.clusterName = clusterName; - supersetSchemaCreation.storeName = storeName; - SchemaMeta valueSchemaMeta = new SchemaMeta(); - valueSchemaMeta.definition = newValueSchemaEntry.getSchemaStr(); - valueSchemaMeta.schemaType = SchemaType.AVRO_1_4.getValue(); - supersetSchemaCreation.valueSchema = valueSchemaMeta; - supersetSchemaCreation.valueSchemaId = newValueSchemaEntry.getId(); - - SchemaMeta supersetSchemaMeta = new SchemaMeta(); - supersetSchemaMeta.definition = newSupersetSchemaEntry.getSchemaStr(); - supersetSchemaMeta.schemaType = SchemaType.AVRO_1_4.getValue(); - supersetSchemaCreation.supersetSchema = supersetSchemaMeta; - supersetSchemaCreation.supersetSchemaId = newSupersetSchemaEntry.getId(); - - AdminOperation message = new AdminOperation(); - message.operationType = AdminMessageType.SUPERSET_SCHEMA_CREATION.getValue(); - message.payloadUnion = supersetSchemaCreation; - - sendAdminMessageAndWaitForConsumed(clusterName, storeName, message); - // Need to add RMD schemas for both new value schema and new superset schema. - updateReplicationMetadataSchema( - clusterName, - storeName, - newValueSchemaEntry.getSchema(), - newValueSchemaEntry.getId()); - updateReplicationMetadataSchema( - clusterName, - storeName, - newSupersetSchemaEntry.getSchema(), - newSupersetSchemaEntry.getId()); - if (isWriteComputationEnabled) { - Schema newValueWriteComputeSchema = - writeComputeSchemaConverter.convertFromValueRecordSchema(newValueSchemaEntry.getSchema()); - Schema newSuperSetWriteComputeSchema = - writeComputeSchemaConverter.convertFromValueRecordSchema(newSupersetSchemaEntry.getSchema()); - addDerivedSchema(clusterName, storeName, newValueSchemaEntry.getId(), newValueWriteComputeSchema.toString()); - addDerivedSchema( - clusterName, - storeName, - newSupersetSchemaEntry.getId(), - newSuperSetWriteComputeSchema.toString()); - } - updateStore( - clusterName, - storeName, - new UpdateStoreQueryParams().setLatestSupersetSchemaId(newSupersetSchemaEntry.getId())); - return newValueSchemaEntry; - } - - private void validateNewSupersetAndValueSchemaEntries( - String storeName, - String clusterName, - SchemaEntry newValueSchemaEntry, - SchemaEntry newSupersetSchemaEntry) { - if (newValueSchemaEntry.getId() == newSupersetSchemaEntry.getId()) { - throw new IllegalArgumentException( - String.format( - "Superset schema ID and value schema ID are expected to be different for store %s in cluster %s. " - + "Got ID: %d", - storeName, - clusterName, - newValueSchemaEntry.getId())); - } - if (AvroSchemaUtils - .compareSchemaIgnoreFieldOrder(newValueSchemaEntry.getSchema(), newSupersetSchemaEntry.getSchema())) { - throw new IllegalArgumentException( - String.format( - "Superset and value schemas are expected to be different for store %s in cluster %s. Got schema: %s", - storeName, - clusterName, - newValueSchemaEntry.getSchema())); - } - } - private SchemaEntry addValueSchemaEntry( String clusterName, String storeName, String valueSchemaStr, - final int newValueSchemaId, - final boolean doUpdateSupersetSchemaID) { + final int newValueSchemaId) { LOGGER.info("Adding value schema: {} to store: {} in cluster: {}", valueSchemaStr, storeName, clusterName); ValueSchemaCreation valueSchemaCreation = @@ -3029,25 +2721,71 @@ private SchemaEntry addValueSchemaEntry( + actualValueSchemaId); } - if (doUpdateSupersetSchemaID) { - updateStore(clusterName, storeName, new UpdateStoreQueryParams().setLatestSupersetSchemaId(newValueSchemaId)); - } - return new SchemaEntry(actualValueSchemaId, valueSchemaStr); } - /** - * Unsupported operation in the parent controller. - */ @Override - public SchemaEntry addSupersetSchema( + public void addSupersetSchema( String clusterName, String storeName, String valueSchemaStr, int valueSchemaId, String supersetSchemaStr, int supersetSchemaId) { - throw new VeniceUnsupportedOperationException("addSupersetSchema"); + acquireAdminMessageLock(clusterName, storeName); + try { + if (supersetSchemaId == SchemaData.INVALID_VALUE_SCHEMA_ID) { + throw new VeniceException("Invalid superset schema id: " + supersetSchemaId); + } + + ReadWriteSchemaRepository schemaRepository = getHelixVeniceClusterResources(clusterName).getSchemaRepository(); + final SchemaEntry existingSupersetSchemaEntry = schemaRepository.getValueSchema(storeName, supersetSchemaId); + if (existingSupersetSchemaEntry != null) { + final Schema newSupersetSchema = AvroSchemaParseUtils.parseSchemaFromJSONStrictValidation(supersetSchemaStr); + if (!AvroSchemaUtils + .compareSchemaIgnoreFieldOrder(existingSupersetSchemaEntry.getSchema(), newSupersetSchema)) { + throw new VeniceException( + "Existing schema with id " + existingSupersetSchemaEntry.getId() + " does not match with new schema " + + supersetSchemaStr); + } + } + + if (valueSchemaId == SchemaData.INVALID_VALUE_SCHEMA_ID) { + LOGGER.info( + "Adding superset schema {} with id {} to store: {} in cluster: {}", + supersetSchemaStr, + supersetSchemaId, + storeName, + clusterName); + valueSchemaStr = ""; + } else if (StringUtils.isEmpty(valueSchemaStr)) { + throw new VeniceException("Invalid value schema string: " + valueSchemaStr); + } + + SupersetSchemaCreation supersetSchemaCreation = + (SupersetSchemaCreation) AdminMessageType.SUPERSET_SCHEMA_CREATION.getNewInstance(); + supersetSchemaCreation.clusterName = clusterName; + supersetSchemaCreation.storeName = storeName; + SchemaMeta valueSchemaMeta = new SchemaMeta(); + valueSchemaMeta.definition = valueSchemaStr; + valueSchemaMeta.schemaType = SchemaType.AVRO_1_4.getValue(); + supersetSchemaCreation.valueSchema = valueSchemaMeta; + supersetSchemaCreation.valueSchemaId = valueSchemaId; + + SchemaMeta supersetSchemaMeta = new SchemaMeta(); + supersetSchemaMeta.definition = supersetSchemaStr; + supersetSchemaMeta.schemaType = SchemaType.AVRO_1_4.getValue(); + supersetSchemaCreation.supersetSchema = supersetSchemaMeta; + supersetSchemaCreation.supersetSchemaId = supersetSchemaId; + + AdminOperation message = new AdminOperation(); + message.operationType = AdminMessageType.SUPERSET_SCHEMA_CREATION.getValue(); + message.payloadUnion = supersetSchemaCreation; + + sendAdminMessageAndWaitForConsumed(clusterName, storeName, message); + } finally { + releaseAdminMessageLock(clusterName, storeName); + } } @Override @@ -3059,80 +2797,14 @@ public SchemaEntry addValueSchema( DirectionalSchemaCompatibilityType expectedCompatibilityType) { acquireAdminMessageLock(clusterName, storeName); try { - Schema newValueSchema = AvroSchemaParseUtils.parseSchemaFromJSONStrictValidation(newValueSchemaStr); - - final Store store = getVeniceHelixAdmin().getStore(clusterName, storeName); - Schema existingValueSchema = getVeniceHelixAdmin().getSupersetOrLatestValueSchema(clusterName, store); - - final boolean doUpdateSupersetSchemaID; - if (existingValueSchema != null && (store.isReadComputationEnabled() || store.isWriteComputationEnabled())) { - SupersetSchemaGenerator supersetSchemaGenerator = getSupersetSchemaGenerator(clusterName); - Schema newSuperSetSchema = supersetSchemaGenerator.generateSupersetSchema(existingValueSchema, newValueSchema); - String newSuperSetSchemaStr = newSuperSetSchema.toString(); - - if (supersetSchemaGenerator.compareSchema(newSuperSetSchema, newValueSchema)) { - doUpdateSupersetSchemaID = true; - - } else if (supersetSchemaGenerator.compareSchema(newSuperSetSchema, existingValueSchema)) { - doUpdateSupersetSchemaID = false; - - } else if (store.isSystemStore()) { - /** - * Do not register superset schema for system store for now. Because some system stores specify the schema ID - * explicitly, which may conflict with the superset schema generated internally, the new value schema registration - * could fail. - * - * TODO: Design a long-term plan. - */ - doUpdateSupersetSchemaID = false; - - } else { - // Register superset schema only if it does not match with existing or new schema. - - // validate compatibility of the new superset schema - getVeniceHelixAdmin().checkPreConditionForAddValueSchemaAndGetNewSchemaId( - clusterName, - storeName, - newSuperSetSchemaStr, - expectedCompatibilityType); - // Check if the superset schema already exists or not. If exists use the same ID, else bump the value ID by - // one. - int supersetSchemaId = getVeniceHelixAdmin().getValueSchemaIdIgnoreFieldOrder( - clusterName, - storeName, - newSuperSetSchemaStr, - (s1, s2) -> supersetSchemaGenerator.compareSchema(s1, s2) ? 0 : 1); - if (supersetSchemaId == SchemaData.INVALID_VALUE_SCHEMA_ID) { - supersetSchemaId = schemaId + 1; - } - return addValueAndSupersetSchemaEntries( - clusterName, - storeName, - new SchemaEntry(schemaId, newValueSchema), - new SchemaEntry(supersetSchemaId, newSuperSetSchema), - store.isWriteComputationEnabled()); - } - } else { - doUpdateSupersetSchemaID = false; + if (schemaId == SchemaData.DUPLICATE_VALUE_SCHEMA_CODE) { + return new SchemaEntry(getValueSchemaId(clusterName, storeName, newValueSchemaStr), newValueSchemaStr); } - SchemaEntry addedSchemaEntry = - addValueSchemaEntry(clusterName, storeName, newValueSchemaStr, schemaId, doUpdateSupersetSchemaID); + SchemaEntry addedSchemaEntry = addValueSchemaEntry(clusterName, storeName, newValueSchemaStr, schemaId); - /** - * if active-active replication is enabled for the store then generate and register the new Replication metadata schema - * for this newly added value schema. - */ - if (store.isActiveActiveReplicationEnabled()) { - Schema latestValueSchema = getVeniceHelixAdmin().getSupersetOrLatestValueSchema(clusterName, store); - final int valueSchemaId = getValueSchemaId(clusterName, storeName, latestValueSchema.toString()); - updateReplicationMetadataSchema(clusterName, storeName, latestValueSchema, valueSchemaId); - } - if (store.isWriteComputationEnabled()) { - Schema newWriteComputeSchema = - writeComputeSchemaConverter.convertFromValueRecordSchema(addedSchemaEntry.getSchema()); - addDerivedSchema(clusterName, storeName, addedSchemaEntry.getId(), newWriteComputeSchema.toString()); - } + // Now register all inferred schemas for the store. + PrimaryControllerConfigUpdateUtils.registerInferredSchemas(this, clusterName, storeName); return addedSchemaEntry; } finally { @@ -3258,8 +2930,8 @@ public RmdSchemaEntry addReplicationMetadataSchema( try { RmdSchemaEntry rmdSchemaEntry = new RmdSchemaEntry(valueSchemaId, replicationMetadataVersionId, replicationMetadataSchemaStr); - final boolean replicationMetadataSchemaAlreadyPresent = getVeniceHelixAdmin() - .checkIfMetadataSchemaAlreadyPresent(clusterName, storeName, valueSchemaId, rmdSchemaEntry); + final boolean replicationMetadataSchemaAlreadyPresent = + getVeniceHelixAdmin().checkIfMetadataSchemaAlreadyPresent(clusterName, storeName, rmdSchemaEntry); if (replicationMetadataSchemaAlreadyPresent) { LOGGER.info( "Replication metadata schema already exists for store: {} in cluster: {} metadataSchema: {} " @@ -3364,36 +3036,6 @@ public void validateAndMaybeRetrySystemStoreAutoCreation( throw new VeniceUnsupportedOperationException("validateAndMaybeRetrySystemStoreAutoCreation"); } - private void updateReplicationMetadataSchemaForAllValueSchema(String clusterName, String storeName) { - final Collection valueSchemas = getValueSchemas(clusterName, storeName); - for (SchemaEntry valueSchemaEntry: valueSchemas) { - updateReplicationMetadataSchema(clusterName, storeName, valueSchemaEntry.getSchema(), valueSchemaEntry.getId()); - } - } - - private void updateReplicationMetadataSchema( - String clusterName, - String storeName, - Schema valueSchema, - int valueSchemaId) { - final int rmdVersionId = getRmdVersionID(storeName, clusterName); - final boolean valueSchemaAlreadyHasRmdSchema = getVeniceHelixAdmin() - .checkIfValueSchemaAlreadyHasRmdSchema(clusterName, storeName, valueSchemaId, rmdVersionId); - if (valueSchemaAlreadyHasRmdSchema) { - LOGGER.info( - "Store {} in cluster {} already has a replication metadata schema for its value schema with ID {} and " - + "replication metadata version ID {}. So skip updating this value schema's RMD schema.", - storeName, - clusterName, - valueSchemaId, - rmdVersionId); - return; - } - String replicationMetadataSchemaStr = - RmdSchemaGenerator.generateMetadataSchema(valueSchema, rmdVersionId).toString(); - addReplicationMetadataSchema(clusterName, storeName, valueSchemaId, rmdVersionId, replicationMetadataSchemaStr); - } - /** * Unsupported operation in the parent controller. */ @@ -3826,7 +3468,7 @@ public NodeRemovableResult isInstanceRemovable( */ @Override public Pair> nodeReplicaReadiness(String cluster, String helixNodeId) { - throw new VeniceUnsupportedOperationException("nodeReplicaReadiness is not supported"); + throw new VeniceUnsupportedOperationException("nodeReplicaReadiness"); } private StoreInfo getStoreInChildRegion(String regionName, String clusterName, String storeName) { @@ -4469,36 +4111,6 @@ public StoreMetaValue getMetaStoreValue(StoreMetaKey metaKey, String storeName) throw new VeniceException("Not implemented in parent"); } - /** - * Check if etled proxy account is set before enabling any ETL and return a {@link ETLStoreConfigRecord} - */ - private ETLStoreConfigRecord mergeNewSettingIntoOldETLStoreConfig( - Store store, - Optional regularVersionETLEnabled, - Optional futureVersionETLEnabled, - Optional etledUserProxyAccount) { - ETLStoreConfig etlStoreConfig = store.getEtlStoreConfig(); - /** - * If etl enabled is true (either current version or future version), then account name must be specified in the command - * and it's not empty, or the store metadata already contains a non-empty account name. - */ - if (regularVersionETLEnabled.orElse(false) || futureVersionETLEnabled.orElse(false)) { - if ((!etledUserProxyAccount.isPresent() || etledUserProxyAccount.get().isEmpty()) - && (etlStoreConfig.getEtledUserProxyAccount() == null - || etlStoreConfig.getEtledUserProxyAccount().isEmpty())) { - throw new VeniceException("Cannot enable ETL for this store because etled user proxy account is not set"); - } - } - ETLStoreConfigRecord etlStoreConfigRecord = new ETLStoreConfigRecord(); - etlStoreConfigRecord.etledUserProxyAccount = - etledUserProxyAccount.orElse(etlStoreConfig.getEtledUserProxyAccount()); - etlStoreConfigRecord.regularVersionETLEnabled = - regularVersionETLEnabled.orElse(etlStoreConfig.isRegularVersionETLEnabled()); - etlStoreConfigRecord.futureVersionETLEnabled = - futureVersionETLEnabled.orElse(etlStoreConfig.isFutureVersionETLEnabled()); - return etlStoreConfigRecord; - } - /** * This parses the input accessPermission string to create ACL's and provision them using the authorizerService interface. * @@ -4649,7 +4261,7 @@ public void updateAclForStore(String clusterName, String storeName, String acces try (AutoCloseableLock ignore = resources.getClusterLockManager().createStoreWriteLock(storeName)) { LOGGER.info("ACLProvisioning: UpdateAcl for store: {} in cluster: {}", storeName, clusterName); if (!authorizerService.isPresent()) { - throw new VeniceUnsupportedOperationException("updateAclForStore is not supported yet!"); + throw new VeniceUnsupportedOperationException("updateAclForStore"); } Store store = getVeniceHelixAdmin().checkPreConditionForAclOp(clusterName, storeName); provisionAclsForStore( @@ -4669,7 +4281,7 @@ public void updateSystemStoreAclForStore( HelixVeniceClusterResources resources = getVeniceHelixAdmin().getHelixVeniceClusterResources(clusterName); try (AutoCloseableLock ignore = resources.getClusterLockManager().createStoreWriteLock(regularStoreName)) { if (!authorizerService.isPresent()) { - throw new VeniceUnsupportedOperationException("updateAclForStore is not supported yet!"); + throw new VeniceUnsupportedOperationException("updateAclForStore"); } getVeniceHelixAdmin().checkPreConditionForAclOp(clusterName, regularStoreName); authorizerService.get().setAcls(systemStoreAclBinding); @@ -4685,7 +4297,7 @@ public String getAclForStore(String clusterName, String storeName) { try (AutoCloseableLock ignore = resources.getClusterLockManager().createStoreReadLock(storeName)) { LOGGER.info("ACLProvisioning: GetAcl for store: {} in cluster: {}", storeName, clusterName); if (!authorizerService.isPresent()) { - throw new VeniceUnsupportedOperationException("getAclForStore is not supported yet!"); + throw new VeniceUnsupportedOperationException("getAclForStore"); } getVeniceHelixAdmin().checkPreConditionForAclOp(clusterName, storeName); String accessPerms = fetchAclsForStore(storeName); @@ -4702,7 +4314,7 @@ public void deleteAclForStore(String clusterName, String storeName) { try (AutoCloseableLock ignore = resources.getClusterLockManager().createStoreWriteLock(storeName)) { LOGGER.info("ACLProvisioning: DeleteAcl for store: {} in cluster: {}", storeName, clusterName); if (!authorizerService.isPresent()) { - throw new VeniceUnsupportedOperationException("deleteAclForStore is not supported yet!"); + throw new VeniceUnsupportedOperationException("deleteAclForStore"); } Store store = getVeniceHelixAdmin().checkPreConditionForAclOp(clusterName, storeName); if (!store.isMigrating()) { @@ -4713,32 +4325,6 @@ public void deleteAclForStore(String clusterName, String storeName) { } } - /** - * @see Admin#configureNativeReplication(String, VeniceUserStoreType, Optional, boolean, Optional, Optional) - */ - @Override - public void configureNativeReplication( - String clusterName, - VeniceUserStoreType storeType, - Optional storeName, - boolean enableNativeReplicationForCluster, - Optional newSourceRegion, - Optional regionsFilter) { - ConfigureNativeReplicationForCluster migrateClusterToNativeReplication = - (ConfigureNativeReplicationForCluster) AdminMessageType.CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER - .getNewInstance(); - migrateClusterToNativeReplication.clusterName = clusterName; - migrateClusterToNativeReplication.storeType = storeType.toString(); - migrateClusterToNativeReplication.enabled = enableNativeReplicationForCluster; - migrateClusterToNativeReplication.nativeReplicationSourceRegion = newSourceRegion.orElse(null); - migrateClusterToNativeReplication.regionsFilter = regionsFilter.orElse(null); - - AdminOperation message = new AdminOperation(); - message.operationType = AdminMessageType.CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER.getValue(); - message.payloadUnion = migrateClusterToNativeReplication; - sendAdminMessageAndWaitForConsumed(clusterName, null, message); - } - /** * @see Admin#configureActiveActiveReplication(String, VeniceUserStoreType, Optional, boolean, Optional) */ @@ -4791,7 +4377,7 @@ public Map getClusterStaleStores(String clusterName) { StoreDataAudit audit = store.getValue(); Optional currentPushJobTopic = getTopicForCurrentPushJob(clusterName, store.getValue().getStoreName(), false, false); - if (audit.getStaleRegions().size() > 0 && !currentPushJobTopic.isPresent()) { + if (!audit.getStaleRegions().isEmpty() && !currentPushJobTopic.isPresent()) { retMap.put(store.getKey(), audit); } } @@ -4893,6 +4479,14 @@ public boolean isParent() { return getVeniceHelixAdmin().isParent(); } + /** + * @see Admin#isPrimary() + */ + @Override + public boolean isPrimary() { + return getVeniceHelixAdmin().isPrimary(); + } + /** * @see Admin#getChildDataCenterControllerUrlMap(String) */ @@ -4987,13 +4581,6 @@ public VeniceHelixAdmin getVeniceHelixAdmin() { return veniceHelixAdmin; } - private Function addToUpdatedConfigList(List updatedConfigList, String config) { - return (configValue) -> { - updatedConfigList.add(config); - return configValue; - }; - } - /** * @see Admin#getBackupVersionDefaultRetentionMs() */ @@ -5254,7 +4841,8 @@ LingeringStoreVersionChecker getLingeringStoreVersionChecker() { return lingeringStoreVersionChecker; } - VeniceControllerMultiClusterConfig getMultiClusterConfigs() { + @Override + public VeniceControllerMultiClusterConfig getMultiClusterConfigs() { return multiClusterConfigs; } @@ -5306,6 +4894,13 @@ public void createStoragePersona( Set owners) { getVeniceHelixAdmin().checkControllerLeadershipFor(clusterName); + StoragePersonaRepository repository = + getVeniceHelixAdmin().getHelixVeniceClusterResources(clusterName).getStoragePersonaRepository(); + if (repository.hasPersona(name)) { + throw new VeniceException("Persona with name " + name + " already exists"); + } + repository.validatePersona(name, quotaNumber, storesToEnforce, owners); + CreateStoragePersona createStoragePersona = (CreateStoragePersona) AdminMessageType.CREATE_STORAGE_PERSONA.getNewInstance(); createStoragePersona.setClusterName(clusterName); @@ -5318,12 +4913,6 @@ public void createStoragePersona( message.operationType = AdminMessageType.CREATE_STORAGE_PERSONA.getValue(); message.payloadUnion = createStoragePersona; - StoragePersonaRepository repository = - getVeniceHelixAdmin().getHelixVeniceClusterResources(clusterName).getStoragePersonaRepository(); - if (repository.hasPersona(name)) { - throw new VeniceException("Persona with name " + name + " already exists"); - } - repository.validatePersona(name, quotaNumber, storesToEnforce, owners); sendAdminMessageAndWaitForConsumed(clusterName, null, message); } @@ -5360,6 +4949,11 @@ public void deleteStoragePersona(String clusterName, String name) { @Override public void updateStoragePersona(String clusterName, String name, UpdateStoragePersonaQueryParams queryParams) { getVeniceHelixAdmin().checkControllerLeadershipFor(clusterName); + + StoragePersonaRepository repository = + getVeniceHelixAdmin().getHelixVeniceClusterResources(clusterName).getStoragePersonaRepository(); + repository.validatePersonaUpdate(name, queryParams); + UpdateStoragePersona updateStoragePersona = (UpdateStoragePersona) AdminMessageType.UPDATE_STORAGE_PERSONA.getNewInstance(); updateStoragePersona.setClusterName(clusterName); @@ -5371,9 +4965,6 @@ public void updateStoragePersona(String clusterName, String name, UpdateStorageP message.operationType = AdminMessageType.UPDATE_STORAGE_PERSONA.getValue(); message.payloadUnion = updateStoragePersona; - StoragePersonaRepository repository = - getVeniceHelixAdmin().getHelixVeniceClusterResources(clusterName).getStoragePersonaRepository(); - repository.validatePersonaUpdate(name, queryParams); sendAdminMessageAndWaitForConsumed(clusterName, null, message); } @@ -5442,4 +5033,14 @@ public void sendHeartbeatToSystemStore(String clusterName, String systemStoreNam public long getHeartbeatFromSystemStore(String clusterName, String storeName) { throw new VeniceUnsupportedOperationException("getHeartbeatFromSystemStore"); } + + @Override + public HelixVeniceClusterResources getHelixVeniceClusterResources(String cluster) { + return getVeniceHelixAdmin().getHelixVeniceClusterResources(cluster); + } + + @Override + public PubSubTopicRepository getPubSubTopicRepository() { + return pubSubTopicRepository; + } } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/init/SystemSchemaInitializationRoutine.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/init/SystemSchemaInitializationRoutine.java index f3a179c730..de50543b28 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/init/SystemSchemaInitializationRoutine.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/init/SystemSchemaInitializationRoutine.java @@ -5,14 +5,13 @@ import com.linkedin.venice.VeniceConstants; import com.linkedin.venice.controller.VeniceControllerMultiClusterConfig; import com.linkedin.venice.controller.VeniceHelixAdmin; +import com.linkedin.venice.controller.util.PrimaryControllerConfigUpdateUtils; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.exceptions.VeniceNoStoreException; import com.linkedin.venice.meta.Store; -import com.linkedin.venice.schema.GeneratedSchemaID; import com.linkedin.venice.schema.SchemaEntry; import com.linkedin.venice.schema.avro.DirectionalSchemaCompatibilityType; -import com.linkedin.venice.schema.writecompute.WriteComputeSchemaConverter; import com.linkedin.venice.serialization.avro.AvroProtocolDefinition; import com.linkedin.venice.utils.Pair; import com.linkedin.venice.utils.Utils; @@ -34,13 +33,12 @@ public class SystemSchemaInitializationRoutine implements ClusterLeaderInitializ private final VeniceHelixAdmin admin; private final Optional keySchema; private final Optional storeMetadataUpdate; - private final boolean autoRegisterDerivedComputeSchema; public SystemSchemaInitializationRoutine( AvroProtocolDefinition protocolDefinition, VeniceControllerMultiClusterConfig multiClusterConfigs, VeniceHelixAdmin admin) { - this(protocolDefinition, multiClusterConfigs, admin, Optional.empty(), Optional.empty(), false); + this(protocolDefinition, multiClusterConfigs, admin, Optional.empty(), Optional.empty()); } public SystemSchemaInitializationRoutine( @@ -48,14 +46,12 @@ public SystemSchemaInitializationRoutine( VeniceControllerMultiClusterConfig multiClusterConfigs, VeniceHelixAdmin admin, Optional keySchema, - Optional storeMetadataUpdate, - boolean autoRegisterDerivedComputeSchema) { + Optional storeMetadataUpdate) { this.protocolDefinition = protocolDefinition; this.multiClusterConfigs = multiClusterConfigs; this.admin = admin; this.keySchema = keySchema; this.storeMetadataUpdate = storeMetadataUpdate; - this.autoRegisterDerivedComputeSchema = autoRegisterDerivedComputeSchema; } /** @@ -194,31 +190,13 @@ public void execute(String clusterToInit) { schemaInLocalResources.toString(true)); } } - if (autoRegisterDerivedComputeSchema) { - // Check and register Write Compute schema - String writeComputeSchema = - WriteComputeSchemaConverter.getInstance().convertFromValueRecordSchema(schemaInLocalResources).toString(); - GeneratedSchemaID derivedSchemaInfo = - admin.getDerivedSchemaId(clusterToInit, systemStoreName, writeComputeSchema); - if (!derivedSchemaInfo.isValid()) { - /** - * The derived schema doesn't exist right now, try to register it. - */ - try { - admin.addDerivedSchema(clusterToInit, systemStoreName, valueSchemaVersion, writeComputeSchema); - } catch (Exception e) { - LOGGER.error( - "Caught Exception when attempting to register the derived compute schema for '{}' schema version '{}'. Will bubble up.", - protocolDefinition.name(), - valueSchemaVersion, - e); - throw e; - } - LOGGER.info( - "Added the derived compute schema for the new schema v{} to system store '{}'.", - valueSchemaVersion, - systemStoreName); - } + + boolean writeComputationEnabled = + storeMetadataUpdate.map(params -> params.getWriteComputationEnabled().orElse(false)).orElse(false); + + if (writeComputationEnabled) { + // Register partial update schemas (aka derived schemas) + PrimaryControllerConfigUpdateUtils.addUpdateSchemaForStore(admin, clusterToInit, systemStoreName, false); } } } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/init/SystemStoreInitializationHelper.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/init/SystemStoreInitializationHelper.java index 49af4bc9fe..fc4270c6bc 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/init/SystemStoreInitializationHelper.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/init/SystemStoreInitializationHelper.java @@ -3,6 +3,7 @@ import com.linkedin.venice.VeniceConstants; import com.linkedin.venice.controller.Admin; import com.linkedin.venice.controller.VeniceControllerMultiClusterConfig; +import com.linkedin.venice.controller.util.PrimaryControllerConfigUpdateUtils; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.meta.Store; @@ -161,6 +162,11 @@ public static void setupSystemStore( } if (updateStoreQueryParams != null && updateStoreCheckSupplier.apply(store)) { + if (store.getPartitionCount() == 0 && !updateStoreQueryParams.getPartitionCount().isPresent()) { + updateStoreQueryParams + .setPartitionCount(multiClusterConfigs.getControllerConfig(clusterName).getMinNumberOfPartitions()); + } + admin.updateStore(clusterName, systemStoreName, updateStoreQueryParams); store = RetryUtils.executeWithMaxAttempt(() -> { @@ -172,14 +178,29 @@ public static void setupSystemStore( throw new VeniceException("Unable to update store " + systemStoreName); } + if (internalStore.getPartitionCount() == 0) { + throw new VeniceException("Partition count is still 0 after updating store " + systemStoreName); + } + return internalStore; }, 5, delayBetweenStoreUpdateRetries, Collections.singletonList(VeniceException.class)); LOGGER.info("Updated internal store " + systemStoreName + " in cluster " + clusterName); } + boolean activeActiveReplicationEnabled = false; + if (updateStoreQueryParams != null) { + activeActiveReplicationEnabled = updateStoreQueryParams.getActiveActiveReplicationEnabled().orElse(false); + } + + if (activeActiveReplicationEnabled) { + // Now that store has enabled A/A and all value schemas are registered, register RMD schemas + PrimaryControllerConfigUpdateUtils + .updateReplicationMetadataSchemaForAllValueSchema(admin, clusterName, systemStoreName); + } + if (store.getCurrentVersion() <= 0) { - int partitionCount = multiClusterConfigs.getControllerConfig(clusterName).getMinNumberOfPartitions(); + int partitionCount = store.getPartitionCount(); int replicationFactor = admin.getReplicationFactor(clusterName, systemStoreName); Version version = admin.incrementVersionIdempotent( clusterName, diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminConsumptionTask.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminConsumptionTask.java index 4d4bd4a5b3..aaf74ab305 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminConsumptionTask.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminConsumptionTask.java @@ -323,6 +323,8 @@ public void run() { while (isRunning.get()) { try { Utils.sleep(READ_CYCLE_DELAY_MS); + // We don't really need to check if the controller is the leader for the cluster here, because it is checked in + // isAdminTopicConsumptionEnabled. However, we still check it here because it helps in testing. if (!admin.isLeaderControllerFor(clusterName) || !admin.isAdminTopicConsumptionEnabled(clusterName)) { unSubscribe(); continue; diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminExecutionTask.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminExecutionTask.java index d919b9319c..d405e5c0f5 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminExecutionTask.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/kafka/consumer/AdminExecutionTask.java @@ -426,8 +426,6 @@ private void handleSetStoreCurrentVersion(SetStoreCurrentVersion message) { String storeName = message.storeName.toString(); int version = message.currentVersion; admin.setStoreCurrentVersion(clusterName, storeName, version); - - LOGGER.info("Set store: {} version to {} in cluster: {}", storeName, version, clusterName); } private void handleSetStoreOwner(SetStoreOwner message) { @@ -546,7 +544,7 @@ private void handleSetStore(UpdateStore message) { if (message.replicateAllConfigs) { finalParams = params; } else { - if (message.updatedConfigsList == null || message.updatedConfigsList.size() == 0) { + if (message.updatedConfigsList == null || message.updatedConfigsList.isEmpty()) { throw new VeniceException( "UpdateStore failed for store " + storeName + ". replicateAllConfigs flag was off " + "but there was no config updates."); @@ -676,21 +674,9 @@ private void handleAddVersion(AddVersion message) { } private void handleEnableNativeReplicationForCluster(ConfigureNativeReplicationForCluster message) { - String clusterName = message.clusterName.toString(); - VeniceUserStoreType storeType = VeniceUserStoreType.valueOf(message.storeType.toString().toUpperCase()); - boolean enableNativeReplication = message.enabled; - Optional nativeReplicationSourceFabric = (message.nativeReplicationSourceRegion == null) - ? Optional.empty() - : Optional.of(message.nativeReplicationSourceRegion.toString()); - Optional regionsFilter = - (message.regionsFilter == null) ? Optional.empty() : Optional.of(message.regionsFilter.toString()); - admin.configureNativeReplication( - clusterName, - storeType, - Optional.of(storeName), - enableNativeReplication, - nativeReplicationSourceFabric, - regionsFilter); + LOGGER.info( + "Received message to configure native replication for cluster: {} but ignoring it as native replication is the only mode", + message.clusterName); } private void handleEnableActiveActiveReplicationForCluster(ConfigureActiveActiveReplicationForCluster message) { diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/AdminSparkServer.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/AdminSparkServer.java index 779fb4ad4e..3765eb4557 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/AdminSparkServer.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/AdminSparkServer.java @@ -15,7 +15,6 @@ import static com.linkedin.venice.controllerapi.ControllerRoute.COMPARE_STORE; import static com.linkedin.venice.controllerapi.ControllerRoute.COMPLETE_MIGRATION; import static com.linkedin.venice.controllerapi.ControllerRoute.CONFIGURE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER; -import static com.linkedin.venice.controllerapi.ControllerRoute.CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER; import static com.linkedin.venice.controllerapi.ControllerRoute.CREATE_STORAGE_PERSONA; import static com.linkedin.venice.controllerapi.ControllerRoute.ClUSTER_HEALTH_INSTANCES; import static com.linkedin.venice.controllerapi.ControllerRoute.DATA_RECOVERY; @@ -404,9 +403,6 @@ public boolean startInner() throws Exception { httpService.post(UPLOAD_PUSH_JOB_STATUS.getPath(), jobRoutes.uploadPushJobStatus(admin)); httpService.post(SEND_PUSH_JOB_DETAILS.getPath(), jobRoutes.sendPushJobDetails(admin)); - httpService.post( - CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER.getPath(), - storesRoutes.enableNativeReplicationForCluster(admin)); httpService.post( CONFIGURE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER.getPath(), storesRoutes.enableActiveActiveReplicationForCluster(admin)); diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/CreateVersion.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/CreateVersion.java index 94489f5005..651dc45932 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/CreateVersion.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/CreateVersion.java @@ -86,7 +86,6 @@ public Route requestTopicForPushing(Admin admin) { && (!hasWriteAccessToTopic(request) || (this.checkReadMethodForKafka && !hasReadAccessToTopic(request)))) { response.status(HttpStatus.SC_FORBIDDEN); String userId = getPrincipalId(request); - String storeName = request.queryParams(NAME); /** * When partners have ACL issues for their push, we should provide an accurate and informative messages that diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java index e39d4852be..9deaed5d86 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java @@ -9,7 +9,6 @@ import static com.linkedin.venice.controllerapi.ControllerApiConstants.HEARTBEAT_TIMESTAMP; import static com.linkedin.venice.controllerapi.ControllerApiConstants.INCLUDE_SYSTEM_STORES; import static com.linkedin.venice.controllerapi.ControllerApiConstants.NAME; -import static com.linkedin.venice.controllerapi.ControllerApiConstants.NATIVE_REPLICATION_SOURCE_FABRIC; import static com.linkedin.venice.controllerapi.ControllerApiConstants.OPERATION; import static com.linkedin.venice.controllerapi.ControllerApiConstants.OWNER; import static com.linkedin.venice.controllerapi.ControllerApiConstants.PARTITION_DETAIL_ENABLED; @@ -30,7 +29,6 @@ import static com.linkedin.venice.controllerapi.ControllerRoute.COMPARE_STORE; import static com.linkedin.venice.controllerapi.ControllerRoute.COMPLETE_MIGRATION; import static com.linkedin.venice.controllerapi.ControllerRoute.CONFIGURE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER; -import static com.linkedin.venice.controllerapi.ControllerRoute.CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER; import static com.linkedin.venice.controllerapi.ControllerRoute.DELETE_ALL_VERSIONS; import static com.linkedin.venice.controllerapi.ControllerRoute.DELETE_KAFKA_TOPIC; import static com.linkedin.venice.controllerapi.ControllerRoute.DELETE_STORE; @@ -782,9 +780,9 @@ public void internalHandle(Request request, StorageEngineOverheadRatioResponse v } /** - * @see Admin#configureNativeReplication(String, VeniceUserStoreType, Optional, boolean, Optional, Optional) + * @see Admin#configureActiveActiveReplication(String, VeniceUserStoreType, Optional, boolean, Optional) */ - public Route enableNativeReplicationForCluster(Admin admin) { + public Route enableActiveActiveReplicationForCluster(Admin admin) { return new VeniceRouteHandler(ControllerResponse.class) { @Override public void internalHandle(Request request, ControllerResponse veniceResponse) { @@ -793,43 +791,20 @@ public void internalHandle(Request request, ControllerResponse veniceResponse) { return; } - AdminSparkServer.validateParams(request, CONFIGURE_NATIVE_REPLICATION_FOR_CLUSTER.getParams(), admin); + AdminSparkServer.validateParams(request, CONFIGURE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER.getParams(), admin); VeniceUserStoreType storeType = VeniceUserStoreType.valueOf(request.queryParams(STORE_TYPE).toUpperCase()); - String cluster = request.queryParams(CLUSTER); - boolean enableNativeReplicationForCluster = Utils.parseBooleanFromString(request.queryParams(STATUS), STATUS); - String sourceRegionParams = request.queryParamOrDefault(NATIVE_REPLICATION_SOURCE_FABRIC, null); - String regionsFilterParams = request.queryParamOrDefault(REGIONS_FILTER, null); - - admin.configureNativeReplication( - cluster, - storeType, - Optional.empty(), - enableNativeReplicationForCluster, - Optional.ofNullable(sourceRegionParams), - Optional.ofNullable(regionsFilterParams)); - - veniceResponse.setCluster(cluster); - } - }; - } - - /** - * @see Admin#configureActiveActiveReplication(String, VeniceUserStoreType, Optional, boolean, Optional) - */ - public Route enableActiveActiveReplicationForCluster(Admin admin) { - return new VeniceRouteHandler(ControllerResponse.class) { - @Override - public void internalHandle(Request request, ControllerResponse veniceResponse) { - // Only allow allowlist users to run this command - if (!checkIsAllowListUser(request, veniceResponse, () -> isAllowListUser(request))) { + if (storeType == VeniceUserStoreType.INCREMENTAL_PUSH) { + veniceResponse.setError( + "Cannot set cluster-level active-active replication for incremental push stores. Please set for hybrid store instead"); + veniceResponse.setErrorType(ErrorType.BAD_REQUEST); return; } - AdminSparkServer.validateParams(request, CONFIGURE_ACTIVE_ACTIVE_REPLICATION_FOR_CLUSTER.getParams(), admin); - - VeniceUserStoreType storeType = VeniceUserStoreType.valueOf(request.queryParams(STORE_TYPE).toUpperCase()); + if (storeType == VeniceUserStoreType.HYBRID_ONLY) { + storeType = VeniceUserStoreType.HYBRID_OR_INCREMENTAL; + } String cluster = request.queryParams(CLUSTER); boolean enableActiveActiveReplicationForCluster = diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/AdminUtils.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/AdminUtils.java new file mode 100644 index 0000000000..c5c27f903d --- /dev/null +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/AdminUtils.java @@ -0,0 +1,73 @@ +package com.linkedin.venice.controller.util; + +import com.linkedin.venice.ConfigConstants; +import com.linkedin.venice.controller.Admin; +import com.linkedin.venice.controller.VeniceControllerConfig; +import com.linkedin.venice.controller.kafka.protocol.admin.HybridStoreConfigRecord; +import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.meta.BufferReplayPolicy; +import com.linkedin.venice.meta.DataReplicationPolicy; +import com.linkedin.venice.meta.HybridStoreConfig; +import com.linkedin.venice.meta.HybridStoreConfigImpl; +import com.linkedin.venice.meta.Store; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +public class AdminUtils { + private static final Logger LOGGER = LogManager.getLogger(AdminUtils.class); + + private AdminUtils() { + } + + public static boolean isHybrid(HybridStoreConfigRecord hybridStoreConfigRecord) { + HybridStoreConfig hybridStoreConfig = null; + if (hybridStoreConfigRecord != null) { + hybridStoreConfig = new HybridStoreConfigImpl( + hybridStoreConfigRecord.rewindTimeInSeconds, + hybridStoreConfigRecord.offsetLagThresholdToGoOnline, + hybridStoreConfigRecord.producerTimestampLagThresholdToGoOnlineInSeconds, + DataReplicationPolicy.valueOf(hybridStoreConfigRecord.dataReplicationPolicy), + BufferReplayPolicy.valueOf(hybridStoreConfigRecord.bufferReplayPolicy)); + } + return isHybrid(hybridStoreConfig); + } + + /** + * A store is not hybrid in the following two scenarios: + * If hybridStoreConfig is null, it means store is not hybrid. + * If all the hybrid config values are negative, it indicates that the store is being set back to batch-only store. + */ + public static boolean isHybrid(HybridStoreConfig hybridStoreConfig) { + return hybridStoreConfig != null && hybridStoreConfig.isHybrid(); + } + + public static int getRmdVersionID(Admin admin, String storeName, String clusterName) { + final Store store = admin.getStore(clusterName, storeName); + if (store == null) { + LOGGER.warn( + "No store found in the store repository. Will get store-level RMD version ID from cluster config. " + + "Store name: {}, cluster: {}", + storeName, + clusterName); + } else if (store.getRmdVersion() == ConfigConstants.UNSPECIFIED_REPLICATION_METADATA_VERSION) { + LOGGER.info("No store-level RMD version ID found for store {} in cluster {}", storeName, clusterName); + } else { + LOGGER.info( + "Found store-level RMD version ID {} for store {} in cluster {}", + store.getRmdVersion(), + storeName, + clusterName); + return store.getRmdVersion(); + } + + final VeniceControllerConfig controllerClusterConfig = + admin.getMultiClusterConfigs().getControllerConfig(clusterName); + if (controllerClusterConfig == null) { + throw new VeniceException("No controller cluster config found for cluster " + clusterName); + } + final int rmdVersionID = controllerClusterConfig.getReplicationMetadataVersion(); + LOGGER.info("Use RMD version ID {} for cluster {}", rmdVersionID, clusterName); + return rmdVersionID; + } +} diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/ParentControllerConfigUpdateUtils.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/ParentControllerConfigUpdateUtils.java deleted file mode 100644 index e8ee519dd7..0000000000 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/ParentControllerConfigUpdateUtils.java +++ /dev/null @@ -1,171 +0,0 @@ -package com.linkedin.venice.controller.util; - -import com.linkedin.venice.controller.VeniceControllerClusterConfig; -import com.linkedin.venice.controller.VeniceParentHelixAdmin; -import com.linkedin.venice.controller.kafka.protocol.admin.UpdateStore; -import com.linkedin.venice.exceptions.VeniceException; -import com.linkedin.venice.meta.Store; -import com.linkedin.venice.schema.SchemaEntry; -import com.linkedin.venice.schema.writecompute.WriteComputeSchemaConverter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import org.apache.avro.Schema; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - - -/** - * This class is a utility class for Parent Controller store update logics. - * The method here aims to take in current status and request params to determine if certain feature is updated / should - * be updated based on some customized logics. - */ -public class ParentControllerConfigUpdateUtils { - public static final Logger LOGGER = LogManager.getLogger(ParentControllerConfigUpdateUtils.class); - public static final WriteComputeSchemaConverter updateSchemaConverter = WriteComputeSchemaConverter.getInstance(); - - /** - * This method takes in current status and request and try to determine whether to change partial update config. - * The check logic is: - * Step (1): If there is explict request, we will respect the request and maybe update config if new request value is - * different from existing config value. In this step, if we are enabling partial update, we will also perform a dry - * run to validate schema. If validation fails, it will throw exception and fail the whole request. - * Step (2): If there is NO explict request and store is being converted into hybrid store, we will check the cluster - * config and store's latest A/A config to see whether we should by default enable partial update. If so, we will also - * perform a dry on to validate schema. If validation fails, it will swallow the exception and log warning message. It - * will not turn on partial update and will not fail the whole request. - */ - public static boolean checkAndMaybeApplyPartialUpdateConfig( - VeniceParentHelixAdmin parentHelixAdmin, - String clusterName, - String storeName, - Optional partialUpdateRequest, - UpdateStore setStore, - boolean storeBeingConvertedToHybrid) { - Store currentStore = parentHelixAdmin.getVeniceHelixAdmin().getStore(clusterName, storeName); - VeniceControllerClusterConfig clusterConfig = - parentHelixAdmin.getVeniceHelixAdmin().getHelixVeniceClusterResources(clusterName).getConfig(); - boolean partialUpdateConfigChanged = false; - setStore.writeComputationEnabled = currentStore.isWriteComputationEnabled(); - if (partialUpdateRequest.isPresent()) { - setStore.writeComputationEnabled = partialUpdateRequest.get(); - if (partialUpdateRequest.get() && !currentStore.isWriteComputationEnabled()) { - // Dry-run generating update schemas before sending admin messages to enable partial update because - // update schema generation may fail due to some reasons. If that happens, abort the store update process. - addUpdateSchemaForStore(parentHelixAdmin, clusterName, storeName, true); - } - // Explicit request to change partial update config has the highest priority. - return true; - } - /** - * If a store: - * (1) Is being converted to hybrid; - * (2) Is not partial update enabled for now; - * (3) Does not request to change partial update config; - * It means partial update is not enabled, and there is no explict intention to change it. In this case, we will - * check cluster default config based on the replication policy to determine whether to try to enable partial update. - */ - final boolean shouldEnablePartialUpdateBasedOnClusterConfig = - storeBeingConvertedToHybrid && (setStore.activeActiveReplicationEnabled - ? clusterConfig.isEnablePartialUpdateForHybridActiveActiveUserStores() - : clusterConfig.isEnablePartialUpdateForHybridNonActiveActiveUserStores()); - if (!currentStore.isWriteComputationEnabled() && shouldEnablePartialUpdateBasedOnClusterConfig) { - LOGGER.info("Controller will try to enable partial update based on cluster config for store: " + storeName); - /** - * When trying to turn on partial update based on cluster config, if schema generation failed, we will not fail the - * whole request, but just do NOT turn on partial update, as other config update should still be respected. - */ - try { - addUpdateSchemaForStore(parentHelixAdmin, clusterName, storeName, true); - setStore.writeComputationEnabled = true; - partialUpdateConfigChanged = true; - } catch (Exception e) { - LOGGER.warn( - "Caught exception when trying to enable partial update base on cluster config, will not enable partial update for store: " - + storeName, - e); - } - } - return partialUpdateConfigChanged; - } - - public static boolean checkAndMaybeApplyChunkingConfigChange( - VeniceParentHelixAdmin parentHelixAdmin, - String clusterName, - String storeName, - Optional chunkingRequest, - UpdateStore setStore) { - Store currentStore = parentHelixAdmin.getVeniceHelixAdmin().getStore(clusterName, storeName); - setStore.chunkingEnabled = currentStore.isChunkingEnabled(); - if (chunkingRequest.isPresent()) { - setStore.chunkingEnabled = chunkingRequest.get(); - // Explicit request to change chunking config has the highest priority. - return true; - } - // If partial update is just enabled, we will by default enable chunking, if no explict request to update chunking - // config. - if (!currentStore.isWriteComputationEnabled() && setStore.writeComputationEnabled - && !currentStore.isChunkingEnabled()) { - setStore.chunkingEnabled = true; - return true; - } - return false; - } - - public static boolean checkAndMaybeApplyRmdChunkingConfigChange( - VeniceParentHelixAdmin parentHelixAdmin, - String clusterName, - String storeName, - Optional rmdChunkingRequest, - UpdateStore setStore) { - Store currentStore = parentHelixAdmin.getVeniceHelixAdmin().getStore(clusterName, storeName); - setStore.rmdChunkingEnabled = currentStore.isRmdChunkingEnabled(); - if (rmdChunkingRequest.isPresent()) { - setStore.rmdChunkingEnabled = rmdChunkingRequest.get(); - // Explicit request to change RMD chunking config has the highest priority. - return true; - } - // If partial update is just enabled and A/A is enabled, we will by default enable RMD chunking, if no explict - // request to update RMD chunking config. - if (!currentStore.isWriteComputationEnabled() && setStore.writeComputationEnabled - && setStore.activeActiveReplicationEnabled && !currentStore.isRmdChunkingEnabled()) { - setStore.rmdChunkingEnabled = true; - return true; - } - return false; - } - - public static void addUpdateSchemaForStore( - VeniceParentHelixAdmin parentHelixAdmin, - String clusterName, - String storeName, - boolean dryRun) { - Collection valueSchemaEntries = parentHelixAdmin.getValueSchemas(clusterName, storeName); - List updateSchemaEntries = new ArrayList<>(valueSchemaEntries.size()); - int maxId = valueSchemaEntries.stream().map(SchemaEntry::getId).max(Comparator.naturalOrder()).get(); - for (SchemaEntry valueSchemaEntry: valueSchemaEntries) { - try { - Schema updateSchema = updateSchemaConverter.convertFromValueRecordSchema(valueSchemaEntry.getSchema()); - updateSchemaEntries.add(new SchemaEntry(valueSchemaEntry.getId(), updateSchema)); - } catch (Exception e) { - // Allow failure in update schema generation in all schema except the latest value schema - if (valueSchemaEntry.getId() == maxId) { - throw new VeniceException( - "For store " + storeName + " cannot generate update schema for value schema ID :" - + valueSchemaEntry.getId() + ", top level field probably missing defaults.", - e); - } - } - } - // Add update schemas only after all update schema generation succeeded. - if (dryRun) { - return; - } - for (SchemaEntry updateSchemaEntry: updateSchemaEntries) { - parentHelixAdmin - .addDerivedSchema(clusterName, storeName, updateSchemaEntry.getId(), updateSchemaEntry.getSchemaStr()); - } - } -} diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/PrimaryControllerConfigUpdateUtils.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/PrimaryControllerConfigUpdateUtils.java new file mode 100644 index 0000000000..cecc7a955e --- /dev/null +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/PrimaryControllerConfigUpdateUtils.java @@ -0,0 +1,170 @@ +package com.linkedin.venice.controller.util; + +import com.linkedin.venice.controller.Admin; +import com.linkedin.venice.controller.supersetschema.SupersetSchemaGenerator; +import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.meta.Store; +import com.linkedin.venice.schema.SchemaData; +import com.linkedin.venice.schema.SchemaEntry; +import com.linkedin.venice.schema.rmd.RmdSchemaEntry; +import com.linkedin.venice.schema.rmd.RmdSchemaGenerator; +import com.linkedin.venice.schema.writecompute.WriteComputeSchemaConverter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import org.apache.avro.Schema; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/** + * This class is a utility class for Primary Controller store update logics. + * Primary controller is the Parent controller in a multi-region deployment, and it is the Child Controller in a single-region deployment. + * The method here aims to take in current status and request params to determine if certain feature is updated / should + * be updated based on some customized logics. + */ +public class PrimaryControllerConfigUpdateUtils { + public static final Logger LOGGER = LogManager.getLogger(PrimaryControllerConfigUpdateUtils.class); + public static final WriteComputeSchemaConverter UPDATE_SCHEMA_CONVERTER = WriteComputeSchemaConverter.getInstance(); + + /** + * A store can have various schemas that are inferred based on the store's other properties (store configs, existing schemas, etc) + * This function is expected to register all such inferred schemas and it should be invoked on updates to the store's + * configs or schemas. + * + * This should only be executed in the primary controller. In a multi-region mode, the child controller is expected to + * get these updates via the admin channel. + */ + public static void registerInferredSchemas(Admin admin, String clusterName, String storeName) { + if (!UpdateStoreUtils.isInferredStoreUpdateAllowed(admin, storeName)) { + return; + } + + Store store = admin.getStore(clusterName, storeName); + + /** + * Register new superset schemas if either of the following conditions are met: + * 1. There is an existing superset schema + * 2. Read computation is enabled + * 3. Write computation is enabled + */ + if (store.isReadComputationEnabled() || store.isWriteComputationEnabled() + || store.getLatestSuperSetValueSchemaId() != SchemaData.INVALID_VALUE_SCHEMA_ID) { + addSupersetSchemaForStore(admin, clusterName, store); + } + + if (store.isWriteComputationEnabled()) { + // Register partial update schemas (aka derived schemas) + addUpdateSchemaForStore(admin, clusterName, storeName, false); + } + + if (store.isActiveActiveReplicationEnabled()) { + // Register RMD schemas + updateReplicationMetadataSchemaForAllValueSchema(admin, clusterName, storeName); + } + } + + private static void addSupersetSchemaForStore(Admin admin, String clusterName, Store store) { + String storeName = store.getName(); + SupersetSchemaGenerator supersetSchemaGenerator = admin.getSupersetSchemaGenerator(clusterName); + SchemaEntry supersetSchemaEntry = + supersetSchemaGenerator.generateSupersetSchemaFromSchemas(admin.getValueSchemas(clusterName, storeName)); + admin.addSupersetSchema( + clusterName, + storeName, + null, + SchemaData.INVALID_VALUE_SCHEMA_ID, + supersetSchemaEntry.getSchemaStr(), + supersetSchemaEntry.getId()); + } + + public static void addUpdateSchemaForStore(Admin admin, String clusterName, String storeName, boolean dryRun) { + Collection valueSchemaEntries = admin.getValueSchemas(clusterName, storeName); + List updateSchemaEntries = new ArrayList<>(valueSchemaEntries.size()); + int maxId = valueSchemaEntries.stream().map(SchemaEntry::getId).max(Comparator.naturalOrder()).get(); + for (SchemaEntry valueSchemaEntry: valueSchemaEntries) { + try { + Schema updateSchema = UPDATE_SCHEMA_CONVERTER.convertFromValueRecordSchema(valueSchemaEntry.getSchema()); + updateSchemaEntries.add(new SchemaEntry(valueSchemaEntry.getId(), updateSchema)); + } catch (Exception e) { + // Allow failure in update schema generation in all schema except the latest value schema + if (valueSchemaEntry.getId() == maxId) { + throw new VeniceException( + "For store " + storeName + " cannot generate update schema for value schema ID :" + + valueSchemaEntry.getId() + ", top level field probably missing defaults.", + e); + } + } + } + // Add update schemas only after all update schema generation succeeded. + if (dryRun) { + return; + } + for (SchemaEntry updateSchemaEntry: updateSchemaEntries) { + admin.addDerivedSchema(clusterName, storeName, updateSchemaEntry.getId(), updateSchemaEntry.getSchemaStr()); + } + } + + public static void updateReplicationMetadataSchemaForAllValueSchema( + Admin admin, + String clusterName, + String storeName) { + final Collection valueSchemas = admin.getValueSchemas(clusterName, storeName); + for (SchemaEntry valueSchemaEntry: valueSchemas) { + updateReplicationMetadataSchema( + admin, + clusterName, + storeName, + valueSchemaEntry.getSchema(), + valueSchemaEntry.getId()); + } + } + + private static void updateReplicationMetadataSchema( + Admin admin, + String clusterName, + String storeName, + Schema valueSchema, + int valueSchemaId) { + final int rmdVersionId = AdminUtils.getRmdVersionID(admin, storeName, clusterName); + final boolean valueSchemaAlreadyHasRmdSchema = + checkIfValueSchemaAlreadyHasRmdSchema(admin, clusterName, storeName, valueSchemaId, rmdVersionId); + if (valueSchemaAlreadyHasRmdSchema) { + LOGGER.info( + "Store {} in cluster {} already has a replication metadata schema for its value schema with ID {} and " + + "replication metadata version ID {}. So skip updating this value schema's RMD schema.", + storeName, + clusterName, + valueSchemaId, + rmdVersionId); + return; + } + String replicationMetadataSchemaStr = + RmdSchemaGenerator.generateMetadataSchema(valueSchema, rmdVersionId).toString(); + admin.addReplicationMetadataSchema( + clusterName, + storeName, + valueSchemaId, + rmdVersionId, + replicationMetadataSchemaStr); + } + + private static boolean checkIfValueSchemaAlreadyHasRmdSchema( + Admin admin, + String clusterName, + String storeName, + final int valueSchemaID, + final int replicationMetadataVersionId) { + Collection schemaEntries = admin.getHelixVeniceClusterResources(clusterName) + .getSchemaRepository() + .getReplicationMetadataSchemas(storeName); + for (RmdSchemaEntry rmdSchemaEntry: schemaEntries) { + if (rmdSchemaEntry.getValueSchemaID() == valueSchemaID + && rmdSchemaEntry.getId() == replicationMetadataVersionId) { + return true; + } + } + return false; + } +} diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/UpdateStoreUtils.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/UpdateStoreUtils.java new file mode 100644 index 0000000000..da43176ab0 --- /dev/null +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/UpdateStoreUtils.java @@ -0,0 +1,1134 @@ +package com.linkedin.venice.controller.util; + +import static com.linkedin.venice.controllerapi.ControllerApiConstants.ACCESS_CONTROLLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.ACTIVE_ACTIVE_REPLICATION_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.AMPLIFICATION_FACTOR; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.AUTO_SCHEMA_REGISTER_FOR_PUSHJOB_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.BACKUP_STRATEGY; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.BACKUP_VERSION_RETENTION_MS; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.BATCH_GET_LIMIT; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.BLOB_TRANSFER_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.BOOTSTRAP_TO_ONLINE_TIMEOUT_IN_HOURS; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.BUFFER_REPLAY_POLICY; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.CHUNKING_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.CLIENT_DECOMPRESSION_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.COMPRESSION_STRATEGY; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.DATA_REPLICATION_POLICY; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.DISABLE_DAVINCI_PUSH_STATUS_STORE; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.DISABLE_META_STORE; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.ENABLE_READS; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.ENABLE_WRITES; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.ETLED_PROXY_USER_ACCOUNT; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.FUTURE_VERSION_ETL_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.HYBRID_STORE_DISK_QUOTA_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.INCREMENTAL_PUSH_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.LARGEST_USED_VERSION_NUMBER; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.LATEST_SUPERSET_SCHEMA_ID; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.MAX_COMPACTION_LAG_SECONDS; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.MAX_RECORD_SIZE_BYTES; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.MIGRATION_DUPLICATE_STORE; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.MIN_COMPACTION_LAG_SECONDS; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.NATIVE_REPLICATION_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.NATIVE_REPLICATION_SOURCE_FABRIC; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.NUM_VERSIONS_TO_PRESERVE; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.OFFSET_LAG_TO_GO_ONLINE; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.OWNER; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.PARTITIONER_CLASS; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.PARTITIONER_PARAMS; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.PARTITION_COUNT; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.PERSONA_NAME; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.PUSH_STREAM_SOURCE_ADDRESS; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.READ_COMPUTATION_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.READ_QUOTA_IN_CU; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.REGULAR_VERSION_ETL_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.REPLICATION_FACTOR; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.REPLICATION_METADATA_PROTOCOL_VERSION_ID; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.REWIND_TIME_IN_SECONDS; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.RMD_CHUNKING_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.STORAGE_NODE_READ_QUOTA_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.STORAGE_QUOTA_IN_BYTE; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.STORE_MIGRATION; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.STORE_VIEW; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.TIME_LAG_TO_GO_ONLINE; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.UNUSED_SCHEMA_DELETION_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.VERSION; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.WRITE_COMPUTATION_ENABLED; +import static com.linkedin.venice.utils.RegionUtils.parseRegionsFilterList; + +import com.linkedin.venice.common.VeniceSystemStoreUtils; +import com.linkedin.venice.compression.CompressionStrategy; +import com.linkedin.venice.controller.Admin; +import com.linkedin.venice.controller.HelixVeniceClusterResources; +import com.linkedin.venice.controller.StoreViewUtils; +import com.linkedin.venice.controller.VeniceControllerClusterConfig; +import com.linkedin.venice.controller.VeniceControllerConfig; +import com.linkedin.venice.controller.VeniceControllerMultiClusterConfig; +import com.linkedin.venice.controller.VeniceHelixAdmin; +import com.linkedin.venice.controller.VeniceParentHelixAdmin; +import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; +import com.linkedin.venice.exceptions.ErrorType; +import com.linkedin.venice.exceptions.PartitionerSchemaMismatchException; +import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.exceptions.VeniceHttpException; +import com.linkedin.venice.exceptions.VeniceNoStoreException; +import com.linkedin.venice.helix.StoragePersonaRepository; +import com.linkedin.venice.helix.ZkRoutersClusterManager; +import com.linkedin.venice.meta.BackupStrategy; +import com.linkedin.venice.meta.BufferReplayPolicy; +import com.linkedin.venice.meta.DataReplicationPolicy; +import com.linkedin.venice.meta.ETLStoreConfig; +import com.linkedin.venice.meta.ETLStoreConfigImpl; +import com.linkedin.venice.meta.HybridStoreConfig; +import com.linkedin.venice.meta.HybridStoreConfigImpl; +import com.linkedin.venice.meta.PartitionerConfig; +import com.linkedin.venice.meta.PartitionerConfigImpl; +import com.linkedin.venice.meta.Store; +import com.linkedin.venice.meta.Version; +import com.linkedin.venice.meta.ViewConfig; +import com.linkedin.venice.meta.ViewConfigImpl; +import com.linkedin.venice.persona.StoragePersona; +import com.linkedin.venice.pubsub.api.PubSubTopic; +import com.linkedin.venice.pubsub.manager.TopicManager; +import com.linkedin.venice.schema.SchemaData; +import com.linkedin.venice.utils.PartitionUtils; +import com.linkedin.venice.utils.Utils; +import com.linkedin.venice.utils.VeniceProperties; +import com.linkedin.venice.views.VeniceView; +import com.linkedin.venice.views.ViewUtils; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.function.Function; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +public class UpdateStoreUtils { + private static final Logger LOGGER = LogManager.getLogger(UpdateStoreUtils.class); + + private UpdateStoreUtils() { + } + + public static UpdateStoreWrapper getStoreUpdate( + Admin admin, + String clusterName, + String storeName, + UpdateStoreQueryParams params) { + VeniceControllerMultiClusterConfig multiClusterConfigs = admin.getMultiClusterConfigs(); + + // There are certain configs that are only allowed to be updated in child regions. We might still want the ability + // to update such configs in the parent region via the Admin tool for operational reasons. So, we allow such updates + // if the regions filter only specifies one region, which is the parent region. + boolean onlyParentRegionFilter = false; + + // Check whether the command affects this region. + if (params.getRegionsFilter().isPresent()) { + Set regionsFilter = parseRegionsFilterList(params.getRegionsFilter().get()); + if (!regionsFilter.contains(multiClusterConfigs.getRegionName())) { + LOGGER.info( + "UpdateStore command will be skipped for store: {} in cluster: {}, because the region filter is {}" + + " which doesn't include the current region: {}", + storeName, + clusterName, + regionsFilter, + multiClusterConfigs.getRegionName()); + return null; + } + + if (admin.isParent() && regionsFilter.size() == 1) { + onlyParentRegionFilter = true; + } + } + + Store originalStore = admin.getStore(clusterName, storeName); + if (originalStore == null) { + throw new VeniceNoStoreException(storeName, clusterName); + } + + UpdateStoreWrapper updateStoreWrapper = new UpdateStoreWrapper(originalStore); + Set updatedConfigs = updateStoreWrapper.updatedConfigs; + Store updatedStore = updateStoreWrapper.updatedStore; + + Optional owner = params.getOwner(); + Optional readability = params.getEnableReads(); + Optional writeability = params.getEnableWrites(); + Optional partitionCount = params.getPartitionCount(); + Optional partitionerClass = params.getPartitionerClass(); + Optional> partitionerParams = params.getPartitionerParams(); + Optional amplificationFactor = params.getAmplificationFactor(); + Optional storageQuotaInByte = params.getStorageQuotaInByte(); + Optional readQuotaInCU = params.getReadQuotaInCU(); + Optional currentVersion = params.getCurrentVersion(); + Optional largestUsedVersionNumber = params.getLargestUsedVersionNumber(); + Optional hybridRewindSeconds = params.getHybridRewindSeconds(); + Optional hybridOffsetLagThreshold = params.getHybridOffsetLagThreshold(); + Optional hybridTimeLagThreshold = params.getHybridTimeLagThreshold(); + Optional hybridDataReplicationPolicy = params.getHybridDataReplicationPolicy(); + Optional hybridBufferReplayPolicy = params.getHybridBufferReplayPolicy(); + Optional accessControlled = params.getAccessControlled(); + Optional compressionStrategy = params.getCompressionStrategy(); + Optional clientDecompressionEnabled = params.getClientDecompressionEnabled(); + Optional chunkingEnabled = params.getChunkingEnabled(); + Optional rmdChunkingEnabled = params.getRmdChunkingEnabled(); + Optional batchGetLimit = params.getBatchGetLimit(); + Optional numVersionsToPreserve = params.getNumVersionsToPreserve(); + Optional incrementalPushEnabled = params.getIncrementalPushEnabled(); + Optional storeMigration = params.getStoreMigration(); + Optional writeComputationEnabled = params.getWriteComputationEnabled(); + Optional replicationMetadataVersionID = params.getReplicationMetadataVersionID(); + Optional readComputationEnabled = params.getReadComputationEnabled(); + Optional bootstrapToOnlineTimeoutInHours = params.getBootstrapToOnlineTimeoutInHours(); + Optional backupStrategy = params.getBackupStrategy(); + Optional autoSchemaRegisterPushJobEnabled = params.getAutoSchemaRegisterPushJobEnabled(); + Optional hybridStoreDiskQuotaEnabled = params.getHybridStoreDiskQuotaEnabled(); + Optional regularVersionETLEnabled = params.getRegularVersionETLEnabled(); + Optional futureVersionETLEnabled = params.getFutureVersionETLEnabled(); + Optional etledUserProxyAccount = params.getETLedProxyUserAccount(); + Optional nativeReplicationEnabled = params.getNativeReplicationEnabled(); + Optional pushStreamSourceAddress = params.getPushStreamSourceAddress(); + Optional backupVersionRetentionMs = params.getBackupVersionRetentionMs(); + Optional replicationFactor = params.getReplicationFactor(); + Optional migrationDuplicateStore = params.getMigrationDuplicateStore(); + Optional nativeReplicationSourceFabric = params.getNativeReplicationSourceFabric(); + Optional activeActiveReplicationEnabled = params.getActiveActiveReplicationEnabled(); + Optional personaName = params.getStoragePersona(); + Optional> storeViewConfig = params.getStoreViews(); + Optional viewName = params.getViewName(); + Optional viewClassName = params.getViewClassName(); + Optional> viewParams = params.getViewClassParams(); + Optional removeView = params.getDisableStoreView(); + Optional latestSupersetSchemaId = params.getLatestSupersetSchemaId(); + Optional storageNodeReadQuotaEnabled = params.getStorageNodeReadQuotaEnabled(); + Optional minCompactionLagSeconds = params.getMinCompactionLagSeconds(); + Optional maxCompactionLagSeconds = params.getMaxCompactionLagSeconds(); + Optional maxRecordSizeBytes = params.getMaxRecordSizeBytes(); + Optional unusedSchemaDeletionEnabled = params.getUnusedSchemaDeletionEnabled(); + Optional blobTransferEnabled = params.getBlobTransferEnabled(); + + owner.map(addToUpdatedConfigs(updatedConfigs, OWNER)).ifPresent(updatedStore::setOwner); + readability.map(addToUpdatedConfigs(updatedConfigs, ENABLE_READS)).ifPresent(updatedStore::setEnableReads); + writeability.map(addToUpdatedConfigs(updatedConfigs, ENABLE_WRITES)).ifPresent(updatedStore::setEnableWrites); + partitionCount.map(addToUpdatedConfigs(updatedConfigs, PARTITION_COUNT)).ifPresent(updatedStore::setPartitionCount); + largestUsedVersionNumber.map(addToUpdatedConfigs(updatedConfigs, LARGEST_USED_VERSION_NUMBER)) + .ifPresent(updatedStore::setLargestUsedVersionNumber); + bootstrapToOnlineTimeoutInHours.map(addToUpdatedConfigs(updatedConfigs, BOOTSTRAP_TO_ONLINE_TIMEOUT_IN_HOURS)) + .ifPresent(updatedStore::setBootstrapToOnlineTimeoutInHours); + storageQuotaInByte.map(addToUpdatedConfigs(updatedConfigs, STORAGE_QUOTA_IN_BYTE)) + .ifPresent(updatedStore::setStorageQuotaInByte); + accessControlled.map(addToUpdatedConfigs(updatedConfigs, ACCESS_CONTROLLED)) + .ifPresent(updatedStore::setAccessControlled); + compressionStrategy.map(addToUpdatedConfigs(updatedConfigs, COMPRESSION_STRATEGY)) + .ifPresent(updatedStore::setCompressionStrategy); + clientDecompressionEnabled.map(addToUpdatedConfigs(updatedConfigs, CLIENT_DECOMPRESSION_ENABLED)) + .ifPresent(updatedStore::setClientDecompressionEnabled); + chunkingEnabled.map(addToUpdatedConfigs(updatedConfigs, CHUNKING_ENABLED)) + .ifPresent(updatedStore::setChunkingEnabled); + rmdChunkingEnabled.map(addToUpdatedConfigs(updatedConfigs, RMD_CHUNKING_ENABLED)) + .ifPresent(updatedStore::setRmdChunkingEnabled); + batchGetLimit.map(addToUpdatedConfigs(updatedConfigs, BATCH_GET_LIMIT)).ifPresent(updatedStore::setBatchGetLimit); + numVersionsToPreserve.map(addToUpdatedConfigs(updatedConfigs, NUM_VERSIONS_TO_PRESERVE)) + .ifPresent(updatedStore::setNumVersionsToPreserve); + replicationFactor.map(addToUpdatedConfigs(updatedConfigs, REPLICATION_FACTOR)) + .ifPresent(updatedStore::setReplicationFactor); + storeMigration.map(addToUpdatedConfigs(updatedConfigs, STORE_MIGRATION)).ifPresent(updatedStore::setMigrating); + migrationDuplicateStore.map(addToUpdatedConfigs(updatedConfigs, MIGRATION_DUPLICATE_STORE)) + .ifPresent(updatedStore::setMigrationDuplicateStore); + writeComputationEnabled.map(addToUpdatedConfigs(updatedConfigs, WRITE_COMPUTATION_ENABLED)) + .ifPresent(updatedStore::setWriteComputationEnabled); + replicationMetadataVersionID.map(addToUpdatedConfigs(updatedConfigs, REPLICATION_METADATA_PROTOCOL_VERSION_ID)) + .ifPresent(updatedStore::setRmdVersion); + readComputationEnabled.map(addToUpdatedConfigs(updatedConfigs, READ_COMPUTATION_ENABLED)) + .ifPresent(updatedStore::setReadComputationEnabled); + nativeReplicationEnabled.map(addToUpdatedConfigs(updatedConfigs, NATIVE_REPLICATION_ENABLED)) + .ifPresent(updatedStore::setNativeReplicationEnabled); + activeActiveReplicationEnabled.map(addToUpdatedConfigs(updatedConfigs, ACTIVE_ACTIVE_REPLICATION_ENABLED)) + .ifPresent(updatedStore::setActiveActiveReplicationEnabled); + pushStreamSourceAddress.map(addToUpdatedConfigs(updatedConfigs, PUSH_STREAM_SOURCE_ADDRESS)) + .ifPresent(updatedStore::setPushStreamSourceAddress); + backupStrategy.map(addToUpdatedConfigs(updatedConfigs, BACKUP_STRATEGY)).ifPresent(updatedStore::setBackupStrategy); + autoSchemaRegisterPushJobEnabled.map(addToUpdatedConfigs(updatedConfigs, AUTO_SCHEMA_REGISTER_FOR_PUSHJOB_ENABLED)) + .ifPresent(updatedStore::setSchemaAutoRegisterFromPushJobEnabled); + hybridStoreDiskQuotaEnabled.map(addToUpdatedConfigs(updatedConfigs, HYBRID_STORE_DISK_QUOTA_ENABLED)) + .ifPresent(updatedStore::setHybridStoreDiskQuotaEnabled); + backupVersionRetentionMs.map(addToUpdatedConfigs(updatedConfigs, BACKUP_VERSION_RETENTION_MS)) + .ifPresent(updatedStore::setBackupVersionRetentionMs); + nativeReplicationSourceFabric.map(addToUpdatedConfigs(updatedConfigs, NATIVE_REPLICATION_SOURCE_FABRIC)) + .ifPresent(updatedStore::setNativeReplicationSourceFabric); + latestSupersetSchemaId.map(addToUpdatedConfigs(updatedConfigs, LATEST_SUPERSET_SCHEMA_ID)) + .ifPresent(updatedStore::setLatestSuperSetValueSchemaId); + minCompactionLagSeconds.map(addToUpdatedConfigs(updatedConfigs, MIN_COMPACTION_LAG_SECONDS)) + .ifPresent(updatedStore::setMinCompactionLagSeconds); + maxCompactionLagSeconds.map(addToUpdatedConfigs(updatedConfigs, MAX_COMPACTION_LAG_SECONDS)) + .ifPresent(updatedStore::setMaxCompactionLagSeconds); + maxRecordSizeBytes.map(addToUpdatedConfigs(updatedConfigs, MAX_RECORD_SIZE_BYTES)) + .ifPresent(updatedStore::setMaxRecordSizeBytes); + unusedSchemaDeletionEnabled.map(addToUpdatedConfigs(updatedConfigs, UNUSED_SCHEMA_DELETION_ENABLED)) + .ifPresent(updatedStore::setUnusedSchemaDeletionEnabled); + blobTransferEnabled.map(addToUpdatedConfigs(updatedConfigs, BLOB_TRANSFER_ENABLED)) + .ifPresent(updatedStore::setBlobTransferEnabled); + storageNodeReadQuotaEnabled.map(addToUpdatedConfigs(updatedConfigs, STORAGE_NODE_READ_QUOTA_ENABLED)) + .ifPresent(updatedStore::setStorageNodeReadQuotaEnabled); + regularVersionETLEnabled.map(addToUpdatedConfigs(updatedConfigs, REGULAR_VERSION_ETL_ENABLED)) + .ifPresent(regularVersionETL -> { + ETLStoreConfig etlStoreConfig = updatedStore.getEtlStoreConfig(); + if (etlStoreConfig == null) { + etlStoreConfig = new ETLStoreConfigImpl(); + } + etlStoreConfig.setRegularVersionETLEnabled(regularVersionETL); + updatedStore.setEtlStoreConfig(etlStoreConfig); + }); + futureVersionETLEnabled.map(addToUpdatedConfigs(updatedConfigs, FUTURE_VERSION_ETL_ENABLED)) + .ifPresent(futureVersionETL -> { + ETLStoreConfig etlStoreConfig = updatedStore.getEtlStoreConfig(); + if (etlStoreConfig == null) { + etlStoreConfig = new ETLStoreConfigImpl(); + } + etlStoreConfig.setFutureVersionETLEnabled(futureVersionETL); + updatedStore.setEtlStoreConfig(etlStoreConfig); + }); + etledUserProxyAccount.map(addToUpdatedConfigs(updatedConfigs, ETLED_PROXY_USER_ACCOUNT)) + .ifPresent(etlProxyAccount -> { + ETLStoreConfig etlStoreConfig = updatedStore.getEtlStoreConfig(); + if (etlStoreConfig == null) { + etlStoreConfig = new ETLStoreConfigImpl(); + } + etlStoreConfig.setEtledUserProxyAccount(etlProxyAccount); + updatedStore.setEtlStoreConfig(etlStoreConfig); + }); + + if (incrementalPushEnabled.isPresent()) { + if (isInferredStoreUpdateAllowed(admin, storeName)) { + LOGGER.info( + "Incremental push cannot be configured for store {} in cluster {}. It is inferred through other configs", + storeName, + clusterName); + } else { + updatedConfigs.add(INCREMENTAL_PUSH_ENABLED); + updatedStore.setIncrementalPushEnabled(incrementalPushEnabled.get()); + } + } + + // No matter what, set native replication to enabled in multi-region mode if the store currently doesn't enable it, + // and it is not explicitly asked to be updated + if (multiClusterConfigs.isMultiRegion() && !originalStore.isNativeReplicationEnabled()) { + updateInferredConfig( + admin, + updatedStore, + NATIVE_REPLICATION_ENABLED, + updatedConfigs, + () -> updatedStore.setNativeReplicationEnabled(true)); + } + + PartitionerConfig newPartitionerConfig = mergeNewSettingsIntoOldPartitionerConfig( + originalStore, + partitionerClass, + partitionerParams, + amplificationFactor); + + if (newPartitionerConfig != originalStore.getPartitionerConfig()) { + partitionerClass.ifPresent(p -> updatedConfigs.add(PARTITIONER_CLASS)); + partitionerParams.ifPresent(p -> updatedConfigs.add(PARTITIONER_PARAMS)); + amplificationFactor.ifPresent(p -> updatedConfigs.add(AMPLIFICATION_FACTOR)); + updatedStore.setPartitionerConfig(newPartitionerConfig); + } + + HelixVeniceClusterResources resources = admin.getHelixVeniceClusterResources(clusterName); + VeniceControllerClusterConfig clusterConfig = resources.getConfig(); + + readQuotaInCU.map(addToUpdatedConfigs(updatedConfigs, READ_QUOTA_IN_CU)).ifPresent(readQuota -> { + ZkRoutersClusterManager routersClusterManager = resources.getRoutersClusterManager(); + int routerCount = routersClusterManager.getLiveRoutersCount(); + int defaultReadQuotaPerRouter = clusterConfig.getDefaultReadQuotaPerRouter(); + + if (Math.max(defaultReadQuotaPerRouter, routerCount * defaultReadQuotaPerRouter) < readQuota) { + throw new VeniceException( + "Cannot update read quota for store " + storeName + " in cluster " + clusterName + ". Read quota " + + readQuota + " requested is more than the cluster quota."); + } + + updatedStore.setReadQuotaInCU(readQuota); + }); + + if (currentVersion.isPresent()) { + if (admin.isParent() && !onlyParentRegionFilter) { + LOGGER.warn( + "Skipping current version update in parent region for store: {} in cluster: {}", + storeName, + clusterName); + } else { + updatedConfigs.add(VERSION); + updatedStore.setCurrentVersion(currentVersion.get()); + } + } + + HybridStoreConfig originalHybridStoreConfig = originalStore.getHybridStoreConfig(); + HybridStoreConfig newHybridStoreConfig = mergeNewSettingsIntoOldHybridStoreConfig( + originalStore, + hybridRewindSeconds, + hybridOffsetLagThreshold, + hybridTimeLagThreshold, + hybridDataReplicationPolicy, + hybridBufferReplayPolicy); + if (!AdminUtils.isHybrid(newHybridStoreConfig) && AdminUtils.isHybrid(originalHybridStoreConfig)) { + /** + * If all the hybrid config values are negative, it indicates that the store is being set back to batch-only store. + * We cannot remove the RT topic immediately because with NR and AA, existing current version is + * still consuming the RT topic. + */ + updatedStore.setHybridStoreConfig(null); + + updatedConfigs.add(REWIND_TIME_IN_SECONDS); + updatedConfigs.add(OFFSET_LAG_TO_GO_ONLINE); + updatedConfigs.add(TIME_LAG_TO_GO_ONLINE); + updatedConfigs.add(DATA_REPLICATION_POLICY); + updatedConfigs.add(BUFFER_REPLAY_POLICY); + + updateInferredConfigsForHybridToBatch(admin, clusterConfig, updatedStore, updatedConfigs); + } else if (AdminUtils.isHybrid(newHybridStoreConfig)) { + if (!originalStore.isHybrid()) { + updateInferredConfigsForBatchToHybrid(admin, clusterConfig, updatedStore, updatedConfigs); + } + + // If a store is being made Active-Active and a data-replication policy has not been defined, set a default one. + if (updatedStore.isActiveActiveReplicationEnabled() && !originalStore.isActiveActiveReplicationEnabled() + && !hybridDataReplicationPolicy.isPresent()) { + updateInferredConfig(admin, updatedStore, DATA_REPLICATION_POLICY, updatedConfigs, () -> { + LOGGER.info( + "Data replication policy was not explicitly set when converting store to Active-Active store: {}." + + " Setting it to active-active replication policy.", + storeName); + newHybridStoreConfig.setDataReplicationPolicy(DataReplicationPolicy.ACTIVE_ACTIVE); + }); + } + + if (AdminUtils.isHybrid(originalHybridStoreConfig)) { + if (originalHybridStoreConfig.getRewindTimeInSeconds() != newHybridStoreConfig.getRewindTimeInSeconds()) { + updatedConfigs.add(REWIND_TIME_IN_SECONDS); + } + + if (originalHybridStoreConfig.getOffsetLagThresholdToGoOnline() != newHybridStoreConfig + .getOffsetLagThresholdToGoOnline()) { + updatedConfigs.add(OFFSET_LAG_TO_GO_ONLINE); + } + + if (originalHybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds() != newHybridStoreConfig + .getProducerTimestampLagThresholdToGoOnlineInSeconds()) { + updatedConfigs.add(TIME_LAG_TO_GO_ONLINE); + } + + if (originalHybridStoreConfig.getDataReplicationPolicy() != newHybridStoreConfig.getDataReplicationPolicy()) { + updatedConfigs.add(DATA_REPLICATION_POLICY); + } + + if (originalHybridStoreConfig.getBufferReplayPolicy() != newHybridStoreConfig.getBufferReplayPolicy()) { + updatedConfigs.add(BUFFER_REPLAY_POLICY); + } + } else { + updatedConfigs.add(REWIND_TIME_IN_SECONDS); + updatedConfigs.add(OFFSET_LAG_TO_GO_ONLINE); + updatedConfigs.add(TIME_LAG_TO_GO_ONLINE); + updatedConfigs.add(DATA_REPLICATION_POLICY); + updatedConfigs.add(BUFFER_REPLAY_POLICY); + } + + updateInferredConfig( + admin, + updatedStore, + INCREMENTAL_PUSH_ENABLED, + updatedConfigs, + () -> updatedStore.setIncrementalPushEnabled( + isIncrementalPushEnabled(clusterConfig.isMultiRegion(), newHybridStoreConfig))); + + updatedStore.setHybridStoreConfig(newHybridStoreConfig); + } + + if (!updatedStore.isChunkingEnabled() && updatedStore.isWriteComputationEnabled()) { + updateInferredConfig(admin, updatedStore, CHUNKING_ENABLED, updatedConfigs, () -> { + LOGGER.info("Enabling chunking because write compute is enabled for store: " + storeName); + updatedStore.setChunkingEnabled(true); + }); + } + + if (!updatedStore.isRmdChunkingEnabled() && updatedStore.isWriteComputationEnabled()) { + updateInferredConfig(admin, updatedStore, RMD_CHUNKING_ENABLED, updatedConfigs, () -> { + LOGGER.info("Enabling RMD chunking because write compute is enabled for Active/Active store: " + storeName); + updatedStore.setRmdChunkingEnabled(true); + }); + } + + if (!updatedStore.isRmdChunkingEnabled() && updatedStore.isActiveActiveReplicationEnabled()) { + updateInferredConfig(admin, updatedStore, RMD_CHUNKING_ENABLED, updatedConfigs, () -> { + LOGGER.info("Enabling RMD chunking because Active/Active is enabled for Active/Active store: " + storeName); + updatedStore.setRmdChunkingEnabled(true); + }); + } + + if (params.disableMetaStore().isPresent() && params.disableMetaStore().get()) { + LOGGER.info("Disabling meta system store for store: {} of cluster: {}", storeName, clusterName); + updatedConfigs.add(DISABLE_META_STORE); + updatedStore.setStoreMetaSystemStoreEnabled(false); + updatedStore.setStoreMetadataSystemStoreEnabled(false); + } + + if (params.disableDavinciPushStatusStore().isPresent() && params.disableDavinciPushStatusStore().get()) { + updatedConfigs.add(DISABLE_DAVINCI_PUSH_STATUS_STORE); + LOGGER.info("Disabling davinci push status store for store: {} of cluster: {}", storeName, clusterName); + updatedStore.setDaVinciPushStatusStoreEnabled(false); + } + + if (storeViewConfig.isPresent() && viewName.isPresent()) { + throw new VeniceException("Cannot update a store view and overwrite store view setup together!"); + } + + if (viewName.isPresent()) { + Map updatedViewSettings; + if (!removeView.isPresent()) { + if (!viewClassName.isPresent()) { + throw new VeniceException("View class name is required when configuring a view."); + } + // If View parameter is not provided, use emtpy map instead. It does not inherit from existing config. + ViewConfig viewConfig = new ViewConfigImpl(viewClassName.get(), viewParams.orElse(Collections.emptyMap())); + validateStoreViewConfig(originalStore, viewConfig); + updatedViewSettings = addNewViewConfigsIntoOldConfigs(originalStore, viewName.get(), viewConfig); + } else { + updatedViewSettings = removeViewConfigFromStoreViewConfigMap(originalStore, viewName.get()); + } + updatedStore.setViewConfigs(updatedViewSettings); + updatedConfigs.add(STORE_VIEW); + } + + if (storeViewConfig.isPresent()) { + // Validate and overwrite store views if they're getting set + validateStoreViewConfigs(storeViewConfig.get(), updatedStore); + updatedStore.setViewConfigs(StoreViewUtils.convertStringMapViewToViewConfigMap(storeViewConfig.get())); + updatedConfigs.add(STORE_VIEW); + } + + if (personaName.isPresent()) { + updatedConfigs.add(PERSONA_NAME); + } + + validateStoreConfigs(admin, clusterName, updatedStore); + validateStoreUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore); + validatePersona(admin, clusterName, updatedStore, personaName); + + return updateStoreWrapper; + } + + private static Function addToUpdatedConfigs(Set updatedConfigs, String config) { + return (configValue) -> { + updatedConfigs.add(config); + return configValue; + }; + } + + static void updateInferredConfig( + Admin admin, + Store store, + String configName, + Set updatedConfigs, + Runnable updater) { + if (!isInferredStoreUpdateAllowed(admin, store.getName())) { + return; + } + + if (!updatedConfigs.contains(configName)) { + updater.run(); + updatedConfigs.add(configName); + } + } + + static void updateInferredConfigsForHybridToBatch( + Admin admin, + VeniceControllerClusterConfig clusterConfig, + Store updatedStore, + Set updatedConfigs) { + updateInferredConfig( + admin, + updatedStore, + INCREMENTAL_PUSH_ENABLED, + updatedConfigs, + () -> updatedStore.setIncrementalPushEnabled(false)); + updateInferredConfig( + admin, + updatedStore, + NATIVE_REPLICATION_SOURCE_FABRIC, + updatedConfigs, + () -> updatedStore + .setNativeReplicationSourceFabric(clusterConfig.getNativeReplicationSourceFabricAsDefaultForBatchOnly())); + updateInferredConfig( + admin, + updatedStore, + ACTIVE_ACTIVE_REPLICATION_ENABLED, + updatedConfigs, + () -> updatedStore.setActiveActiveReplicationEnabled(false)); + } + + static void updateInferredConfigsForBatchToHybrid( + Admin admin, + VeniceControllerClusterConfig clusterConfig, + Store updatedStore, + Set updatedConfigs) { + String clusterName = clusterConfig.getClusterName(); + String storeName = updatedStore.getName(); + + updateInferredConfig( + admin, + updatedStore, + NATIVE_REPLICATION_SOURCE_FABRIC, + updatedConfigs, + () -> updatedStore + .setNativeReplicationSourceFabric(clusterConfig.getNativeReplicationSourceFabricAsDefaultForHybrid())); + + /* + * Enable/disable active-active replication for user hybrid stores if the cluster level config + * for new hybrid stores is on. + */ + updateInferredConfig( + admin, + updatedStore, + ACTIVE_ACTIVE_REPLICATION_ENABLED, + updatedConfigs, + () -> updatedStore.setActiveActiveReplicationEnabled( + updatedStore.isActiveActiveReplicationEnabled() + || (clusterConfig.isActiveActiveReplicationEnabledAsDefaultForHybrid() + && !updatedStore.isSystemStore()))); + + if (updatedStore.getPartitionCount() == 0) { + updateInferredConfig(admin, updatedStore, PARTITION_COUNT, updatedConfigs, () -> { + int updatedPartitionCount = PartitionUtils.calculatePartitionCount( + storeName, + updatedStore.getStorageQuotaInByte(), + 0, + clusterConfig.getPartitionSize(), + clusterConfig.getMinNumberOfPartitionsForHybrid(), + clusterConfig.getMaxNumberOfPartitions(), + clusterConfig.isPartitionCountRoundUpEnabled(), + clusterConfig.getPartitionCountRoundUpSize()); + updatedStore.setPartitionCount(updatedPartitionCount); + LOGGER.info( + "Enforcing default hybrid partition count: {} for a new hybrid store: {}", + updatedPartitionCount, + storeName); + }); + } + + /** + * If a store: + * (1) Is being converted to hybrid; + * (2) Is not partial update enabled for now; + * (3) Does not request to change partial update config; + * It means partial update is not enabled, and there is no explict intention to change it. In this case, we will + * check cluster default config based on the replication policy to determine whether to try to enable partial update. + */ + final boolean shouldEnablePartialUpdateBasedOnClusterConfig = (updatedStore.isActiveActiveReplicationEnabled() + ? clusterConfig.isEnablePartialUpdateForHybridActiveActiveUserStores() + : clusterConfig.isEnablePartialUpdateForHybridNonActiveActiveUserStores()); + if (shouldEnablePartialUpdateBasedOnClusterConfig) { + LOGGER.info("Controller will enable partial update based on cluster config for store: " + storeName); + /** + * When trying to turn on partial update based on cluster config, if schema generation failed, we will not fail the + * whole request, but just do NOT turn on partial update, as other config update should still be respected. + */ + try { + PrimaryControllerConfigUpdateUtils.addUpdateSchemaForStore(admin, clusterName, updatedStore.getName(), true); + updateInferredConfig(admin, updatedStore, WRITE_COMPUTATION_ENABLED, updatedConfigs, () -> { + updatedStore.setWriteComputationEnabled(true); + }); + } catch (Exception e) { + LOGGER.warn( + "Caught exception when trying to enable partial update base on cluster config, will not enable partial update for store: " + + storeName, + e); + } + } + } + + /** + * Check if a store can support incremental pushes based on other configs. The following rules define when incremental + * push is allowed: + *

    + *
  1. If the system is running in single-region mode, the store must by hybrid
    2. + *
  2. If the system is running in multi-region mode,
    3. + *
      + *
    1. Hybrid + Active-Active + {@link DataReplicationPolicy} is {@link DataReplicationPolicy#ACTIVE_ACTIVE}
      2. + *
    2. Hybrid + {@link DataReplicationPolicy} is {@link DataReplicationPolicy#AGGREGATE}
      3. + *
    3. Hybrid + {@link DataReplicationPolicy} is {@link DataReplicationPolicy#NONE}
      4. + *
    + *
      + * @param multiRegion whether the system is running in multi-region mode + * @param hybridStoreConfig The hybrid store config after applying all updates + * @return {@code true} if incremental push is allowed, {@code false} otherwise + */ + static boolean isIncrementalPushEnabled(boolean multiRegion, HybridStoreConfig hybridStoreConfig) { + // Only hybrid stores can support incremental push + if (!AdminUtils.isHybrid(hybridStoreConfig)) { + return false; + } + + // If the system is running in multi-region mode, we need to validate the data replication policies + if (!multiRegion) { + return true; + } + + DataReplicationPolicy dataReplicationPolicy = hybridStoreConfig.getDataReplicationPolicy(); + return dataReplicationPolicy == DataReplicationPolicy.ACTIVE_ACTIVE + || dataReplicationPolicy == DataReplicationPolicy.AGGREGATE + || dataReplicationPolicy == DataReplicationPolicy.NONE; + } + + static void validateStoreConfigs(Admin admin, String clusterName, Store store) { + String storeName = store.getName(); + String errorMessagePrefix = "Store update error for " + storeName + " in cluster: " + clusterName + ": "; + + if (!store.isHybrid()) { + // Inc push + non hybrid not supported + if (store.isIncrementalPushEnabled()) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + errorMessagePrefix + "Incremental push is only supported for hybrid stores", + ErrorType.INVALID_CONFIG); + } + + // WC is only supported for hybrid stores + if (store.isWriteComputationEnabled()) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + errorMessagePrefix + "Write computation is only supported for hybrid stores", + ErrorType.INVALID_CONFIG); + } + } else { + HybridStoreConfig hybridStoreConfig = store.getHybridStoreConfig(); + // All fields of hybrid store config must have valid values + if (hybridStoreConfig.getRewindTimeInSeconds() < 0) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + errorMessagePrefix + "Rewind time cannot be negative for a hybrid store", + ErrorType.INVALID_CONFIG); + } + + if (hybridStoreConfig.getOffsetLagThresholdToGoOnline() < 0 + && hybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds() < 0) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + errorMessagePrefix + + "Both offset lag threshold and producer timestamp lag threshold cannot be negative for a hybrid store", + ErrorType.INVALID_CONFIG); + } + + VeniceControllerConfig controllerConfig = admin.getMultiClusterConfigs().getControllerConfig(clusterName); + DataReplicationPolicy dataReplicationPolicy = hybridStoreConfig.getDataReplicationPolicy(); + // Incremental push + NON_AGGREGATE DRP is not supported in multi-region mode + if (controllerConfig.isMultiRegion() && store.isIncrementalPushEnabled() + && dataReplicationPolicy == DataReplicationPolicy.NON_AGGREGATE) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + errorMessagePrefix + + "Incremental push is not supported for hybrid stores with non-aggregate data replication policy", + ErrorType.INVALID_CONFIG); + } + + // ACTIVE_ACTIVE DRP is only supported when activeActiveReplicationEnabled = true + if (dataReplicationPolicy == DataReplicationPolicy.ACTIVE_ACTIVE && !store.isActiveActiveReplicationEnabled()) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + errorMessagePrefix + + "Data replication policy ACTIVE_ACTIVE is only supported for hybrid stores with active-active replication enabled", + ErrorType.INVALID_CONFIG); + } + } + + // Storage quota can not be less than 0 + if (store.getStorageQuotaInByte() < 0 && store.getStorageQuotaInByte() != Store.UNLIMITED_STORAGE_QUOTA) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + "Storage quota can not be less than 0", + ErrorType.INVALID_CONFIG); + } + + // Read quota can not be less than 0 + if (store.getReadQuotaInCU() < 0) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + "Read quota can not be less than 0", + ErrorType.INVALID_CONFIG); + } + + // Active-active replication is only supported for stores that also have native replication + if (store.isActiveActiveReplicationEnabled() && !store.isNativeReplicationEnabled()) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + "Active/Active Replication cannot be enabled for store " + store.getName() + + " since Native Replication is not enabled on it.", + ErrorType.INVALID_CONFIG); + } + + // Active-Active and write-compute are not supported when amplification factor is more than 1 + PartitionerConfig partitionerConfig = store.getPartitionerConfig(); + if (partitionerConfig == null) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + errorMessagePrefix + "Partitioner Config cannot be null", + ErrorType.INVALID_CONFIG); + } + + if (partitionerConfig.getAmplificationFactor() > 1 + && (store.isActiveActiveReplicationEnabled() || store.isWriteComputationEnabled())) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + errorMessagePrefix + + "Active-active replication or write computation is not supported for stores with amplification factor > 1", + ErrorType.INVALID_CONFIG); + } + + // Before setting partitioner config, verify the updated partitionerConfig can be built + try { + Properties partitionerParams = new Properties(); + for (Map.Entry param: partitionerConfig.getPartitionerParams().entrySet()) { + partitionerParams.setProperty(param.getKey(), param.getValue()); + } + + PartitionUtils.getVenicePartitioner( + partitionerConfig.getPartitionerClass(), + new VeniceProperties(partitionerParams), + admin.getKeySchema(clusterName, storeName).getSchema()); + } catch (PartitionerSchemaMismatchException e) { + String errorMessage = errorMessagePrefix + e.getMessage(); + LOGGER.error(errorMessage); + throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_SCHEMA); + } catch (Exception e) { + String errorMessage = errorMessagePrefix + "Partitioner Configs invalid, please verify that partitioner " + + "configs like classpath and parameters are correct!"; + LOGGER.error(errorMessage); + throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); + } + + // Validate if the latest superset schema id is an existing value schema + int latestSupersetSchemaId = store.getLatestSuperSetValueSchemaId(); + if (latestSupersetSchemaId != SchemaData.INVALID_VALUE_SCHEMA_ID) { + if (admin.getValueSchema(clusterName, storeName, latestSupersetSchemaId) == null) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + "Unknown value schema id: " + latestSupersetSchemaId + " in store: " + storeName, + ErrorType.INVALID_CONFIG); + } + } + + if (store.getMaxCompactionLagSeconds() < store.getMinCompactionLagSeconds()) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + "Store's max compaction lag seconds: " + store.getMaxCompactionLagSeconds() + " shouldn't be smaller than " + + "store's min compaction lag seconds: " + store.getMinCompactionLagSeconds(), + ErrorType.INVALID_CONFIG); + } + + ETLStoreConfig etlStoreConfig = store.getEtlStoreConfig(); + if (etlStoreConfig != null + && (etlStoreConfig.isRegularVersionETLEnabled() || etlStoreConfig.isFutureVersionETLEnabled())) { + if (StringUtils.isEmpty(etlStoreConfig.getEtledUserProxyAccount())) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + "Cannot enable ETL for this store because etled user proxy account is not set", + ErrorType.INVALID_CONFIG); + } + } + } + + private static void validateStoreUpdate( + Admin admin, + VeniceControllerMultiClusterConfig multiClusterConfig, + String clusterName, + Store originalStore, + Store updatedStore) { + validateStorePartitionCountUpdate(admin, multiClusterConfig, clusterName, originalStore, updatedStore); + validateStorePartitionerUpdate(clusterName, originalStore, updatedStore); + + if (updatedStore.isWriteComputationEnabled() && !originalStore.isWriteComputationEnabled()) { + // Dry-run generating update schemas before sending admin messages to enable partial update because + // update schema generation may fail due to some reasons. If that happens, abort the store update process. + PrimaryControllerConfigUpdateUtils.addUpdateSchemaForStore(admin, clusterName, originalStore.getName(), true); + } + } + + private static void validateStoreViewConfigs(Map stringMap, Store store) { + Map configs = StoreViewUtils.convertStringMapViewToViewConfigMap(stringMap); + for (Map.Entry viewConfigEntry: configs.entrySet()) { + validateStoreViewConfig(store, viewConfigEntry.getValue()); + } + } + + private static void validateStoreViewConfig(Store store, ViewConfig viewConfig) { + // TODO: Pass a proper properties object here. Today this isn't used in this context + VeniceView view = + ViewUtils.getVeniceView(viewConfig.getViewClassName(), new Properties(), store, viewConfig.getViewParameters()); + view.validateConfigs(); + } + + /** + * Used by both the {@link VeniceHelixAdmin} and the {@link VeniceParentHelixAdmin} + * + * @param oldStore Existing Store that is the source for updates. This object will not be modified by this method. + * @param hybridRewindSeconds Optional is present if the returned object should include a new rewind time + * @param hybridOffsetLagThreshold Optional is present if the returned object should include a new offset lag threshold + * @param hybridTimeLagThreshold + * @param hybridDataReplicationPolicy + * @param bufferReplayPolicy + * @return null if oldStore has no hybrid configs and optionals are not present, + * otherwise a fully specified {@link HybridStoreConfig} + */ + static HybridStoreConfig mergeNewSettingsIntoOldHybridStoreConfig( + Store oldStore, + Optional hybridRewindSeconds, + Optional hybridOffsetLagThreshold, + Optional hybridTimeLagThreshold, + Optional hybridDataReplicationPolicy, + Optional bufferReplayPolicy) { + HybridStoreConfig mergedHybridStoreConfig; + if (oldStore.isHybrid()) { // for an existing hybrid store, just replace any specified values + HybridStoreConfig oldHybridConfig = oldStore.getHybridStoreConfig().clone(); + mergedHybridStoreConfig = new HybridStoreConfigImpl( + hybridRewindSeconds.orElseGet(oldHybridConfig::getRewindTimeInSeconds), + hybridOffsetLagThreshold.orElseGet(oldHybridConfig::getOffsetLagThresholdToGoOnline), + hybridTimeLagThreshold.orElseGet(oldHybridConfig::getProducerTimestampLagThresholdToGoOnlineInSeconds), + hybridDataReplicationPolicy.orElseGet(oldHybridConfig::getDataReplicationPolicy), + bufferReplayPolicy.orElseGet(oldHybridConfig::getBufferReplayPolicy)); + } else { + mergedHybridStoreConfig = new HybridStoreConfigImpl( + hybridRewindSeconds.orElse(-1L), + // If not specified, offset/time lag threshold will be -1 and will not be used to determine whether + // a partition is ready to serve + hybridOffsetLagThreshold.orElse(-1L), + hybridTimeLagThreshold.orElse(-1L), + hybridDataReplicationPolicy.orElse(DataReplicationPolicy.NON_AGGREGATE), + bufferReplayPolicy.orElse(BufferReplayPolicy.REWIND_FROM_EOP)); + } + + if (mergedHybridStoreConfig.getRewindTimeInSeconds() > 0 + && mergedHybridStoreConfig.getOffsetLagThresholdToGoOnline() < 0 + && mergedHybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds() < 0) { + throw new VeniceException( + "Both offset lag threshold and time lag threshold are negative when setting hybrid configs for store " + + oldStore.getName()); + } + + if (!AdminUtils.isHybrid(mergedHybridStoreConfig)) { + return null; + } + + return mergedHybridStoreConfig; + } + + public static void validateStorePartitionCountUpdate( + Admin admin, + VeniceControllerMultiClusterConfig multiClusterConfigs, + String clusterName, + Store originalStore, + int newPartitionCount) { + Store updatedStore = originalStore.cloneStore(); + updatedStore.setPartitionCount(newPartitionCount); + validateStorePartitionCountUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore); + } + + static void validateStorePartitionCountUpdate( + Admin admin, + VeniceControllerMultiClusterConfig multiClusterConfigs, + String clusterName, + Store originalStore, + Store updatedStore) { + String storeName = originalStore.getName(); + String errorMessagePrefix = "Store update error for " + storeName + " in cluster: " + clusterName + ": "; + VeniceControllerClusterConfig clusterConfig = admin.getHelixVeniceClusterResources(clusterName).getConfig(); + + int newPartitionCount = updatedStore.getPartitionCount(); + if (newPartitionCount < 0) { + String errorMessage = errorMessagePrefix + "Partition count: " + newPartitionCount + " should NOT be negative"; + LOGGER.error(errorMessage); + throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); + } + + if (updatedStore.isHybrid() && newPartitionCount == 0) { + String errorMessage = errorMessagePrefix + "Partition count cannot be 0 for hybrid store"; + LOGGER.error(errorMessage); + throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); + } + + if (originalStore.isHybrid() && updatedStore.isHybrid() && originalStore.getPartitionCount() != newPartitionCount) { + String errorMessage = errorMessagePrefix + "Cannot change partition count for this hybrid store"; + LOGGER.error(errorMessage); + throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); + } + + int minPartitionNum = clusterConfig.getMinNumberOfPartitions(); + if (newPartitionCount < minPartitionNum && newPartitionCount != 0) { + throw new VeniceHttpException( + HttpStatus.SC_BAD_REQUEST, + "Partition count must be at least " + minPartitionNum + " for store: " + storeName + + ". If a specific partition count is not required, set it to 0.", + ErrorType.INVALID_CONFIG); + } + + int maxPartitionNum = clusterConfig.getMaxNumberOfPartitions(); + if (newPartitionCount > maxPartitionNum) { + String errorMessage = + errorMessagePrefix + "Partition count: " + newPartitionCount + " should be less than max: " + maxPartitionNum; + LOGGER.error(errorMessage); + throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); + } + + if (updatedStore.isHybrid()) { + // Allow the update if the new partition count matches RT partition count + TopicManager topicManager; + if (admin.isParent()) { + // RT might not exist in parent colo. Get RT partition count from a child colo. + String childDatacenter = Utils.parseCommaSeparatedStringToList(clusterConfig.getChildDatacenters()).get(0); + topicManager = admin.getTopicManager(multiClusterConfigs.getChildDataCenterKafkaUrlMap().get(childDatacenter)); + } else { + topicManager = admin.getTopicManager(); + } + PubSubTopic realTimeTopic = admin.getPubSubTopicRepository().getTopic(Version.composeRealTimeTopic(storeName)); + if (!topicManager.containsTopic(realTimeTopic) + || topicManager.getPartitionCount(realTimeTopic) == newPartitionCount) { + LOGGER.info("Allow updating store " + storeName + " partition count to " + newPartitionCount); + return; + } + String errorMessage = errorMessagePrefix + "Cannot change partition count for this hybrid store"; + LOGGER.error(errorMessage); + throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); + } + } + + static void validateStorePartitionerUpdate(String clusterName, Store existingStore, Store updatedStore) { + String storeName = existingStore.getName(); + String errorMessagePrefix = "Store update error for " + storeName + " in cluster: " + clusterName + ": "; + + if (!existingStore.isHybrid() || !updatedStore.isHybrid()) { + // Allow partitioner changes for non-hybrid stores + return; + } + + PartitionerConfig existingPartitionerConfig = existingStore.getPartitionerConfig(); + PartitionerConfig updatedPartitionerConfig = updatedStore.getPartitionerConfig(); + + if (!existingPartitionerConfig.getPartitionerClass().equals(updatedPartitionerConfig.getPartitionerClass())) { + String errorMessage = errorMessagePrefix + "Partitioner class cannot be changed for hybrid store"; + LOGGER.error(errorMessage); + throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); + } + + if (!existingPartitionerConfig.getPartitionerParams().equals(updatedPartitionerConfig.getPartitionerParams())) { + String errorMessage = errorMessagePrefix + "Partitioner params cannot be changed for hybrid store"; + LOGGER.error(errorMessage); + throw new VeniceHttpException(HttpStatus.SC_BAD_REQUEST, errorMessage, ErrorType.INVALID_CONFIG); + } + } + + static void validatePersona(Admin admin, String clusterName, Store updatedStore, Optional personaName) { + String storeName = updatedStore.getName(); + StoragePersonaRepository repository = + admin.getHelixVeniceClusterResources(clusterName).getStoragePersonaRepository(); + StoragePersona personaToValidate = null; + StoragePersona existingPersona = repository.getPersonaContainingStore(storeName); + + if (personaName.isPresent()) { + personaToValidate = admin.getStoragePersona(clusterName, personaName.get()); + if (personaToValidate == null) { + String errMsg = "UpdateStore command failed for store " + storeName + ". The provided StoragePersona " + + personaName.get() + " does not exist."; + throw new VeniceException(errMsg); + } + } else if (existingPersona != null) { + personaToValidate = existingPersona; + } + + if (personaToValidate != null) { + repository.validateAddUpdatedStore(personaToValidate, Optional.of(updatedStore)); + } + } + + static PartitionerConfig mergeNewSettingsIntoOldPartitionerConfig( + Store oldStore, + Optional partitionerClass, + Optional> partitionerParams, + Optional amplificationFactor) { + + if (!partitionerClass.isPresent() && !partitionerParams.isPresent() && !amplificationFactor.isPresent()) { + return oldStore.getPartitionerConfig(); + } + + PartitionerConfig originalPartitionerConfig; + if (oldStore.getPartitionerConfig() == null) { + originalPartitionerConfig = new PartitionerConfigImpl(); + } else { + originalPartitionerConfig = oldStore.getPartitionerConfig(); + } + return new PartitionerConfigImpl( + partitionerClass.orElse(originalPartitionerConfig.getPartitionerClass()), + partitionerParams.orElse(originalPartitionerConfig.getPartitionerParams()), + amplificationFactor.orElse(originalPartitionerConfig.getAmplificationFactor())); + } + + static Map addNewViewConfigsIntoOldConfigs( + Store oldStore, + String viewClass, + ViewConfig viewConfig) throws VeniceException { + // Add new view config into the existing config map. The new configs will override existing ones which share the + // same key. + Map oldViewConfigMap = oldStore.getViewConfigs(); + if (oldViewConfigMap == null) { + oldViewConfigMap = new HashMap<>(); + } + Map mergedConfigs = new HashMap<>(oldViewConfigMap); + mergedConfigs.put(viewClass, viewConfig); + return mergedConfigs; + } + + static Map removeViewConfigFromStoreViewConfigMap(Store oldStore, String viewClass) + throws VeniceException { + Map oldViewConfigMap = oldStore.getViewConfigs(); + if (oldViewConfigMap == null) { + // TODO: We might want to return a null instead of empty map + oldViewConfigMap = new HashMap<>(); + } + Map mergedConfigs = new HashMap<>(oldViewConfigMap); + mergedConfigs.remove(viewClass); + return mergedConfigs; + } + + /** + * This function is the entry-point of all operations that are necessary after the successful execution of the store + * update. These should only be executed in the primary controller. + * @param admin The main {@link Admin} object for this component + * @param clusterName The name of the cluster where the store is being updated + * @param storeName The name of the store that was updated + */ + public static void handlePostUpdateActions(Admin admin, String clusterName, String storeName) { + PrimaryControllerConfigUpdateUtils.registerInferredSchemas(admin, clusterName, storeName); + } + + /** + * Check if direct store config updates are allowed in this controller. In multi-region mode, parent controller + * decides what store configs get applied to a store. In a single-region mode, the child controller makes this + * decision. + * In a multi-region mode, the child controller must not do any inferencing and must only apply the configs that were + * applied by the parent controller, except for child-region-only stores - i.e. participant store. + */ + static boolean isInferredStoreUpdateAllowed(Admin admin, String storeName) { + // For system stores, do not allow any inferencing + if (VeniceSystemStoreUtils.isSystemStore(storeName)) { + return false; + } + + if (!admin.isPrimary()) { + return false; + } + + // Parent controller can only apply the updates if it is processing updates in VeniceParentHelixAdmin (i.e. not via + // the Admin channel) + return !admin.isParent() || admin instanceof VeniceParentHelixAdmin; + } +} diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/UpdateStoreWrapper.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/UpdateStoreWrapper.java new file mode 100644 index 0000000000..486b17a295 --- /dev/null +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/util/UpdateStoreWrapper.java @@ -0,0 +1,18 @@ +package com.linkedin.venice.controller.util; + +import com.linkedin.venice.meta.Store; +import java.util.HashSet; +import java.util.Set; + + +public class UpdateStoreWrapper { + public final Set updatedConfigs; + public final Store originalStore; + public final Store updatedStore; + + public UpdateStoreWrapper(Store originalStore) { + this.originalStore = originalStore; + this.updatedConfigs = new HashSet<>(); + this.updatedStore = originalStore.cloneStore(); + } +} diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/AbstractTestVeniceParentHelixAdmin.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/AbstractTestVeniceParentHelixAdmin.java index aded34a940..4d2899c5c3 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/AbstractTestVeniceParentHelixAdmin.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/AbstractTestVeniceParentHelixAdmin.java @@ -4,7 +4,6 @@ import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import com.linkedin.venice.authorization.AuthorizerService; import com.linkedin.venice.authorization.DefaultIdentityParser; @@ -23,7 +22,6 @@ import com.linkedin.venice.helix.StoragePersonaRepository; import com.linkedin.venice.helix.ZkRoutersClusterManager; import com.linkedin.venice.helix.ZkStoreConfigAccessor; -import com.linkedin.venice.meta.HybridStoreConfig; import com.linkedin.venice.meta.OfflinePushStrategy; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.StoreInfo; @@ -88,7 +86,6 @@ public void setupInternalMocks() { doReturn(true).when(topicManager).containsTopicAndAllPartitionsAreOnline(pubSubTopicRepository.getTopic(topicName)); internalAdmin = mock(VeniceHelixAdmin.class); - when(internalAdmin.isHybrid((HybridStoreConfig) any())).thenCallRealMethod(); doReturn(topicManager).when(internalAdmin).getTopicManager(); SchemaEntry mockEntry = new SchemaEntry(0, TEST_SCHEMA); doReturn(mockEntry).when(internalAdmin).getKeySchema(anyString(), anyString()); @@ -127,6 +124,8 @@ public void setupInternalMocks() { .put(regionName, ControllerClient.constructClusterControllerClient(clusterName, "localhost", Optional.empty())); doReturn(controllerClients).when(internalAdmin).getControllerClientMap(any()); + doReturn(true).when(internalAdmin).isPrimary(); + resources = mockResources(config, clusterName); doReturn(storeRepository).when(resources).getStoreMetadataRepository(); ZkRoutersClusterManager manager = mock(ZkRoutersClusterManager.class); @@ -140,6 +139,8 @@ public void setupInternalMocks() { clusterLockManager = mock(ClusterLockManager.class); doReturn(clusterLockManager).when(resources).getClusterLockManager(); + doReturn(1000).when(config).getDefaultReadQuotaPerRouter(); + adminStats = mock(VeniceAdminStats.class); doReturn(adminStats).when(resources).getVeniceAdminStats(); @@ -204,6 +205,10 @@ VeniceControllerConfig mockConfig(String clusterName) { doReturn(childClusterMap).when(config).getChildDataCenterControllerUrlMap(); doReturn(MAX_PARTITION_NUM).when(config).getMaxNumberOfPartitions(); doReturn(DefaultIdentityParser.class.getName()).when(config).getIdentityParserClassName(); + doReturn(true).when(config).isMultiRegion(); + doReturn(10L).when(config).getPartitionSize(); + doReturn("dc-batch-nr").when(config).getNativeReplicationSourceFabricAsDefaultForHybrid(); + doReturn("dc-hybrid-nr").when(config).getNativeReplicationSourceFabricAsDefaultForHybrid(); return config; } diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithoutCluster.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithoutCluster.java index c052635862..6a952c653d 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithoutCluster.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceHelixAdminWithoutCluster.java @@ -10,10 +10,6 @@ import com.linkedin.venice.common.VeniceSystemStoreType; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.helix.ZkStoreConfigAccessor; -import com.linkedin.venice.meta.BufferReplayPolicy; -import com.linkedin.venice.meta.DataReplicationPolicy; -import com.linkedin.venice.meta.HybridStoreConfig; -import com.linkedin.venice.meta.HybridStoreConfigImpl; import com.linkedin.venice.meta.ReadWriteStoreRepository; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.StoreConfig; @@ -37,58 +33,6 @@ public class TestVeniceHelixAdminWithoutCluster { private final PubSubTopicRepository pubSubTopicRepository = new PubSubTopicRepository(); - @Test - public void canMergeNewHybridConfigValuesToOldStore() { - String storeName = Utils.getUniqueString("storeName"); - Store store = TestUtils.createTestStore(storeName, "owner", System.currentTimeMillis()); - Assert.assertFalse(store.isHybrid()); - - Optional rewind = Optional.of(123L); - Optional lagOffset = Optional.of(1500L); - Optional timeLag = Optional.of(300L); - Optional dataReplicationPolicy = Optional.of(DataReplicationPolicy.AGGREGATE); - Optional bufferReplayPolicy = Optional.of(BufferReplayPolicy.REWIND_FROM_EOP); - HybridStoreConfig hybridStoreConfig = VeniceHelixAdmin.mergeNewSettingsIntoOldHybridStoreConfig( - store, - Optional.empty(), - Optional.empty(), - Optional.empty(), - Optional.empty(), - Optional.empty()); - Assert.assertNull( - hybridStoreConfig, - "passing empty optionals and a non-hybrid store should generate a null hybrid config"); - - hybridStoreConfig = VeniceHelixAdmin.mergeNewSettingsIntoOldHybridStoreConfig( - store, - rewind, - lagOffset, - timeLag, - dataReplicationPolicy, - bufferReplayPolicy); - Assert.assertNotNull(hybridStoreConfig, "specifying rewind and lagOffset should generate a valid hybrid config"); - Assert.assertEquals(hybridStoreConfig.getRewindTimeInSeconds(), 123L); - Assert.assertEquals(hybridStoreConfig.getOffsetLagThresholdToGoOnline(), 1500L); - Assert.assertEquals(hybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds(), 300L); - Assert.assertEquals(hybridStoreConfig.getDataReplicationPolicy(), DataReplicationPolicy.AGGREGATE); - - // It's okay that time lag threshold or data replication policy is not specified - hybridStoreConfig = VeniceHelixAdmin.mergeNewSettingsIntoOldHybridStoreConfig( - store, - rewind, - lagOffset, - Optional.empty(), - Optional.empty(), - Optional.empty()); - Assert.assertNotNull(hybridStoreConfig, "specifying rewind and lagOffset should generate a valid hybrid config"); - Assert.assertEquals(hybridStoreConfig.getRewindTimeInSeconds(), 123L); - Assert.assertEquals(hybridStoreConfig.getOffsetLagThresholdToGoOnline(), 1500L); - Assert.assertEquals( - hybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds(), - HybridStoreConfigImpl.DEFAULT_HYBRID_TIME_LAG_THRESHOLD); - Assert.assertEquals(hybridStoreConfig.getDataReplicationPolicy(), DataReplicationPolicy.NON_AGGREGATE); - } - @Test(expectedExceptions = VeniceException.class, expectedExceptionsMessageRegExp = ".*still exists in cluster.*") public void testCheckResourceCleanupBeforeStoreCreationWhenExistsInOtherCluster() { String clusterName = "cluster1"; diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceParentHelixAdmin.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceParentHelixAdmin.java index 5fc9737708..e35248e39c 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceParentHelixAdmin.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/TestVeniceParentHelixAdmin.java @@ -2,7 +2,6 @@ import static com.linkedin.venice.controller.VeniceHelixAdmin.VERSION_ID_UNSET; import static com.linkedin.venice.meta.BufferReplayPolicy.REWIND_FROM_SOP; -import static com.linkedin.venice.meta.HybridStoreConfigImpl.DEFAULT_HYBRID_TIME_LAG_THRESHOLD; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.anyInt; @@ -21,7 +20,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertThrows; -import com.linkedin.venice.common.VeniceSystemStoreUtils; import com.linkedin.venice.compression.CompressionStrategy; import com.linkedin.venice.controller.kafka.AdminTopicUtils; import com.linkedin.venice.controller.kafka.consumer.AdminConsumptionTask; @@ -54,9 +52,6 @@ import com.linkedin.venice.exceptions.VeniceStoreAlreadyExistsException; import com.linkedin.venice.exceptions.VeniceUnsupportedOperationException; import com.linkedin.venice.helix.HelixReadWriteStoreRepository; -import com.linkedin.venice.meta.BufferReplayPolicy; -import com.linkedin.venice.meta.DataReplicationPolicy; -import com.linkedin.venice.meta.HybridStoreConfigImpl; import com.linkedin.venice.meta.OfflinePushStrategy; import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.meta.ReadStrategy; @@ -89,7 +84,6 @@ import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; import com.linkedin.venice.utils.Utils; -import com.linkedin.venice.utils.concurrent.VeniceConcurrentHashMap; import com.linkedin.venice.utils.locks.ClusterLockManager; import com.linkedin.venice.views.ChangeCaptureView; import com.linkedin.venice.writer.VeniceWriter; @@ -157,102 +151,6 @@ public void testStartWhenTopicNotExists() { Optional.empty()); } - /** - * Partially stubbed class to verify async setup behavior. - */ - private static class AsyncSetupMockVeniceParentHelixAdmin extends VeniceParentHelixAdmin { - private Map systemStores = new VeniceConcurrentHashMap<>(); - - public AsyncSetupMockVeniceParentHelixAdmin(VeniceHelixAdmin veniceHelixAdmin, VeniceControllerConfig config) { - super(veniceHelixAdmin, TestUtils.getMultiClusterConfigFromOneCluster(config)); - } - - public boolean isAsyncSetupRunning(String clusterName) { - return asyncSetupEnabledMap.get(clusterName); - } - - @Override - public void createStore( - String clusterName, - String storeName, - String owner, - String keySchema, - String valueSchema, - boolean isSystemStore) { - if (!(VeniceSystemStoreUtils.isSystemStore(storeName) && isSystemStore)) { - throw new VeniceException("Invalid store name and isSystemStore combination. Got store name: " + storeName); - } - if (systemStores.containsKey(storeName)) { - // no op - return; - } - Store newStore = new ZKStore( - storeName, - owner, - System.currentTimeMillis(), - PersistenceType.IN_MEMORY, - RoutingStrategy.HASH, - ReadStrategy.ANY_OF_ONLINE, - OfflinePushStrategy.WAIT_N_MINUS_ONE_REPLCIA_PER_PARTITION, - 1); - systemStores.put(storeName, newStore); - } - - @Override - public Store getStore(String clusterName, String storeName) { - if (!systemStores.containsKey(storeName)) { - return null; - } - return systemStores.get(storeName).cloneStore(); - } - - @Override - public void updateStore(String clusterName, String storeName, UpdateStoreQueryParams params) { - Optional hybridRewindSeconds = params.getHybridRewindSeconds(); - Optional hybridOffsetLagThreshold = params.getHybridOffsetLagThreshold(); - Optional hybridTimeLagThreshold = params.getHybridTimeLagThreshold(); - Optional hybridDataReplicationPolicy = params.getHybridDataReplicationPolicy(); - Optional hybridBufferReplayPolicy = params.getHybridBufferReplayPolicy(); - - if (!systemStores.containsKey(storeName)) { - throw new VeniceNoStoreException("Cannot update store " + storeName + " because it's missing."); - } - if (hybridRewindSeconds.isPresent() && hybridOffsetLagThreshold.isPresent()) { - final long finalHybridTimeLagThreshold = hybridTimeLagThreshold.orElse(DEFAULT_HYBRID_TIME_LAG_THRESHOLD); - final DataReplicationPolicy finalHybridDataReplicationPolicy = - hybridDataReplicationPolicy.orElse(DataReplicationPolicy.NON_AGGREGATE); - final BufferReplayPolicy finalHybridBufferReplayPolicy = - hybridBufferReplayPolicy.orElse(BufferReplayPolicy.REWIND_FROM_EOP); - systemStores.get(storeName) - .setHybridStoreConfig( - new HybridStoreConfigImpl( - hybridRewindSeconds.get(), - hybridOffsetLagThreshold.get(), - finalHybridTimeLagThreshold, - finalHybridDataReplicationPolicy, - finalHybridBufferReplayPolicy)); - } - } - - @Override - public Version incrementVersionIdempotent( - String clusterName, - String storeName, - String pushJobId, - int numberOfPartition, - int replicationFactor) { - if (!systemStores.containsKey(storeName)) { - throw new VeniceNoStoreException("Cannot add version to store " + storeName + " because it's missing."); - } - Version version = new VersionImpl(storeName, 1, "test-id"); - version.setReplicationFactor(replicationFactor); - List versions = new ArrayList<>(); - versions.add(version); - systemStores.get(storeName).setVersions(versions); - return version; - } - } - @Test public void testAddStore() { doReturn(CompletableFuture.completedFuture(new SimplePubSubProduceResultImpl(topicName, partitionId, 1, -1))) @@ -1745,7 +1643,7 @@ public void testUpdateStore() { .thenReturn(AdminTopicMetadataAccessor.generateMetadataMap(1, -1, 1)); UpdateStoreQueryParams storeQueryParams1 = - new UpdateStoreQueryParams().setIncrementalPushEnabled(true).setBlobTransferEnabled(true); + new UpdateStoreQueryParams().setChunkingEnabled(true).setBlobTransferEnabled(true); parentAdmin.initStorageCluster(clusterName); parentAdmin.updateStore(clusterName, storeName, storeQueryParams1); @@ -1765,7 +1663,7 @@ public void testUpdateStore() { assertEquals(adminMessage.operationType, AdminMessageType.UPDATE_STORE.getValue()); UpdateStore updateStore = (UpdateStore) adminMessage.payloadUnion; - assertEquals(updateStore.incrementalPushEnabled, true); + Assert.assertTrue(updateStore.chunkingEnabled); Assert.assertTrue(updateStore.blobTransferEnabled); long readQuota = 100L; @@ -1774,7 +1672,6 @@ public void testUpdateStore() { Map testPartitionerParams = new HashMap<>(); UpdateStoreQueryParams updateStoreQueryParams = new UpdateStoreQueryParams().setEnableReads(readability) - .setIncrementalPushEnabled(false) .setPartitionCount(64) .setPartitionerClass("com.linkedin.venice.partitioner.DefaultVenicePartitioner") .setPartitionerParams(testPartitionerParams) @@ -1893,68 +1790,6 @@ public void testUpdateStoreNativeReplicationSourceFabric() { "Native replication source fabric does not match after updating the store!"); } - @Test - public void testDisableHybridConfigWhenActiveActiveOrIncPushConfigIsEnabled() { - String storeName = Utils.getUniqueString("testUpdateStore"); - Store store = TestUtils.createTestStore(storeName, "test", System.currentTimeMillis()); - - store.setHybridStoreConfig( - new HybridStoreConfigImpl( - 1000, - 100, - -1, - DataReplicationPolicy.NON_AGGREGATE, - BufferReplayPolicy.REWIND_FROM_EOP)); - store.setActiveActiveReplicationEnabled(true); - store.setIncrementalPushEnabled(true); - store.setNativeReplicationEnabled(true); - store.setChunkingEnabled(true); - doReturn(store).when(internalAdmin).getStore(clusterName, storeName); - - doReturn(CompletableFuture.completedFuture(new SimplePubSubProduceResultImpl(topicName, partitionId, 1, -1))) - .when(veniceWriter) - .put(any(), any(), anyInt()); - - when(zkClient.readData(zkMetadataNodePath, null)).thenReturn(null) - .thenReturn(AdminTopicMetadataAccessor.generateMetadataMap(1, -1, 1)); - - parentAdmin.initStorageCluster(clusterName); - // When user disable hybrid but also try to manually turn on A/A or Incremental Push, update operation should fail - // loudly. - assertThrows( - () -> parentAdmin.updateStore( - clusterName, - storeName, - new UpdateStoreQueryParams().setHybridRewindSeconds(-1) - .setHybridOffsetLagThreshold(-1) - .setActiveActiveReplicationEnabled(true))); - assertThrows( - () -> parentAdmin.updateStore( - clusterName, - storeName, - new UpdateStoreQueryParams().setHybridRewindSeconds(-1) - .setHybridOffsetLagThreshold(-1) - .setIncrementalPushEnabled(true))); - - parentAdmin.updateStore( - clusterName, - storeName, - new UpdateStoreQueryParams().setHybridOffsetLagThreshold(-1).setHybridRewindSeconds(-1)); - - ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(byte[].class); - ArgumentCaptor valueCaptor = ArgumentCaptor.forClass(byte[].class); - ArgumentCaptor schemaCaptor = ArgumentCaptor.forClass(Integer.class); - - verify(veniceWriter, times(1)).put(keyCaptor.capture(), valueCaptor.capture(), schemaCaptor.capture()); - byte[] valueBytes = valueCaptor.getValue(); - int schemaId = schemaCaptor.getValue(); - AdminOperation adminMessage = adminOperationSerializer.deserialize(ByteBuffer.wrap(valueBytes), schemaId); - UpdateStore updateStore = (UpdateStore) adminMessage.payloadUnion; - Assert.assertFalse(internalAdmin.isHybrid(updateStore.getHybridStoreConfig())); - Assert.assertFalse(updateStore.incrementalPushEnabled); - Assert.assertFalse(updateStore.activeActiveReplicationEnabled); - } - @Test public void testSetStoreViewConfig() { String storeName = Utils.getUniqueString("testUpdateStore"); @@ -2599,7 +2434,7 @@ public void testAclException() { } @Test - public void testHybridAndIncrementalUpdateStoreCommands() { + public void testHybridUpdatesPartitionCount() { String storeName = Utils.getUniqueString("testUpdateStore"); Store store = TestUtils.createTestStore(storeName, "test", System.currentTimeMillis()); doReturn(store).when(internalAdmin).getStore(clusterName, storeName); @@ -2635,19 +2470,8 @@ public void testHybridAndIncrementalUpdateStoreCommands() { UpdateStore updateStore = (UpdateStore) adminMessage.payloadUnion; assertEquals(updateStore.hybridStoreConfig.offsetLagThresholdToGoOnline, 20000); assertEquals(updateStore.hybridStoreConfig.rewindTimeInSeconds, 60); - - store.setHybridStoreConfig( - new HybridStoreConfigImpl( - 60, - 20000, - 0, - DataReplicationPolicy.NON_AGGREGATE, - BufferReplayPolicy.REWIND_FROM_EOP)); - // Incremental push can be enabled on a hybrid store, default inc push policy is inc push to RT now - parentAdmin.updateStore(clusterName, storeName, new UpdateStoreQueryParams().setIncrementalPushEnabled(true)); - - // veniceWriter.put will be called again for the second update store command - verify(veniceWriter, times(2)).put(keyCaptor.capture(), valueCaptor.capture(), schemaCaptor.capture()); + assertEquals(updateStore.nativeReplicationSourceFabric.toString(), "dc-hybrid-nr"); + assertEquals(updateStore.partitionNum, 1024); } @Test diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/init/SystemStoreInitializationHelperTest.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/init/SystemStoreInitializationHelperTest.java index a38e02d3bd..c999901e38 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/init/SystemStoreInitializationHelperTest.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/init/SystemStoreInitializationHelperTest.java @@ -64,14 +64,17 @@ public void testInitialSystemStoreSetup(boolean explicitlyProvidedKeySchema) { int partitionCount = 10; int replicationFactor = 3; doReturn(1).when(firstVersion).getNumber(); + doReturn(partitionCount).when(firstVersion).getPartitionCount(); Store storeForTest = mock(Store.class); Store storeForTestAfterUpdateStore = mock(Store.class); doReturn(true).when(storeForTestAfterUpdateStore).isHybrid(); + doReturn(partitionCount).when(storeForTestAfterUpdateStore).getPartitionCount(); Store storeForTestAfterCreatingVersion = mock(Store.class); doReturn(true).when(storeForTestAfterCreatingVersion).isHybrid(); + doReturn(partitionCount).when(storeForTestAfterUpdateStore).getPartitionCount(); doReturn(versionNumber).when(storeForTestAfterCreatingVersion).getCurrentVersion(); doReturn(firstVersion).when(storeForTestAfterCreatingVersion).getVersion(versionNumber); doReturn(Collections.singletonList(firstVersion)).when(storeForTestAfterCreatingVersion).getVersions(); diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/consumer/TestAdminConsumerService.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/consumer/TestAdminConsumerService.java index 9629005f65..0a2c6bc440 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/consumer/TestAdminConsumerService.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/kafka/consumer/TestAdminConsumerService.java @@ -1,6 +1,5 @@ package com.linkedin.venice.controller.kafka.consumer; -import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED; import static com.linkedin.venice.ConfigKeys.ADMIN_TOPIC_SOURCE_REGION; import static com.linkedin.venice.ConfigKeys.CHILD_CLUSTER_ALLOWLIST; import static com.linkedin.venice.ConfigKeys.CHILD_DATA_CENTER_KAFKA_URL_PREFIX; @@ -52,7 +51,6 @@ public void testMultipleAdminConsumerServiceWithSameMetricsRepo() { .put( CLUSTER_TO_SERVER_D2, TestUtils.getClusterToD2String(Collections.singletonMap(someClusterName, "dummy_server_d2"))) - .put(ADMIN_TOPIC_REMOTE_CONSUMPTION_ENABLED, true) .put(ADMIN_TOPIC_SOURCE_REGION, adminTopicSourceRegion) .put(NATIVE_REPLICATION_FABRIC_ALLOWLIST, adminTopicSourceRegion) .put(CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + adminTopicSourceRegion, "blah") diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/util/AdminUtilsTest.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/util/AdminUtilsTest.java new file mode 100644 index 0000000000..7146786ed3 --- /dev/null +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/util/AdminUtilsTest.java @@ -0,0 +1,134 @@ +package com.linkedin.venice.controller.util; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import com.linkedin.venice.ConfigConstants; +import com.linkedin.venice.controller.Admin; +import com.linkedin.venice.controller.VeniceControllerConfig; +import com.linkedin.venice.controller.VeniceControllerMultiClusterConfig; +import com.linkedin.venice.controller.kafka.protocol.admin.HybridStoreConfigRecord; +import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.meta.BufferReplayPolicy; +import com.linkedin.venice.meta.DataReplicationPolicy; +import com.linkedin.venice.meta.HybridStoreConfig; +import com.linkedin.venice.meta.HybridStoreConfigImpl; +import com.linkedin.venice.meta.Store; +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class AdminUtilsTest { + @Test + public void testIsHybrid() { + Assert.assertFalse(AdminUtils.isHybrid((HybridStoreConfig) null)); + Assert.assertFalse(AdminUtils.isHybrid((HybridStoreConfigRecord) null)); + + HybridStoreConfig hybridStoreConfig; + hybridStoreConfig = new HybridStoreConfigImpl(-1, -1, -1, null, null); + Assert.assertFalse(AdminUtils.isHybrid(hybridStoreConfig)); + + hybridStoreConfig = new HybridStoreConfigImpl(100, -1, -1, null, null); + Assert.assertFalse(AdminUtils.isHybrid(hybridStoreConfig)); + + hybridStoreConfig = new HybridStoreConfigImpl(100, 100, -1, null, null); + assertTrue(AdminUtils.isHybrid(hybridStoreConfig)); + + hybridStoreConfig = new HybridStoreConfigImpl(100, 100, 100, null, null); + assertTrue(AdminUtils.isHybrid(hybridStoreConfig)); + + hybridStoreConfig = new HybridStoreConfigImpl(100, -1, 100, null, null); + assertTrue(AdminUtils.isHybrid(hybridStoreConfig)); + + hybridStoreConfig = new HybridStoreConfigImpl(-1, -1, 100, null, null); + Assert.assertFalse(AdminUtils.isHybrid(hybridStoreConfig)); + + HybridStoreConfigRecord hybridStoreConfigRecord = new HybridStoreConfigRecord(); + hybridStoreConfigRecord.rewindTimeInSeconds = 100; + hybridStoreConfigRecord.offsetLagThresholdToGoOnline = 100; + hybridStoreConfigRecord.producerTimestampLagThresholdToGoOnlineInSeconds = -1; + hybridStoreConfigRecord.dataReplicationPolicy = DataReplicationPolicy.ACTIVE_ACTIVE.getValue(); + hybridStoreConfigRecord.bufferReplayPolicy = BufferReplayPolicy.REWIND_FROM_SOP.getValue(); + assertTrue(AdminUtils.isHybrid(hybridStoreConfigRecord)); + } + + @Test + public void testGetRmdVersionID() { + String storeName = "storeName"; + String clusterName = "clusterName"; + + Admin mockAdmin = mock(Admin.class); + VeniceControllerMultiClusterConfig multiClusterConfig = mock(VeniceControllerMultiClusterConfig.class); + VeniceControllerConfig controllerConfig = mock(VeniceControllerConfig.class); + Store mockStore = mock(Store.class); + + // Store null + cluster config not set + doReturn(null).when(mockAdmin).getStore(clusterName, storeName); + doReturn(multiClusterConfig).when(mockAdmin).getMultiClusterConfigs(); + doReturn(null).when(multiClusterConfig).getControllerConfig(clusterName); + VeniceException e1 = + Assert.expectThrows(VeniceException.class, () -> AdminUtils.getRmdVersionID(mockAdmin, storeName, clusterName)); + assertTrue(e1.getMessage().contains("No controller cluster config found for cluster clusterName")); + + reset(mockAdmin); + reset(multiClusterConfig); + reset(controllerConfig); + reset(mockStore); + + // Store null + cluster config set + doReturn(null).when(mockAdmin).getStore(clusterName, storeName); + doReturn(multiClusterConfig).when(mockAdmin).getMultiClusterConfigs(); + doReturn(controllerConfig).when(multiClusterConfig).getControllerConfig(clusterName); + doReturn(10).when(controllerConfig).getReplicationMetadataVersion(); + assertEquals(AdminUtils.getRmdVersionID(mockAdmin, storeName, clusterName), 10); + + reset(mockAdmin); + reset(multiClusterConfig); + reset(controllerConfig); + reset(mockStore); + + // Store-level RMD version ID not found + cluster config not set + doReturn(mockStore).when(mockAdmin).getStore(clusterName, storeName); + doReturn(ConfigConstants.UNSPECIFIED_REPLICATION_METADATA_VERSION).when(mockStore).getRmdVersion(); + doReturn(multiClusterConfig).when(mockAdmin).getMultiClusterConfigs(); + doReturn(null).when(multiClusterConfig).getControllerConfig(clusterName); + VeniceException e2 = + Assert.expectThrows(VeniceException.class, () -> AdminUtils.getRmdVersionID(mockAdmin, storeName, clusterName)); + assertTrue(e2.getMessage().contains("No controller cluster config found for cluster clusterName")); + + reset(mockAdmin); + reset(multiClusterConfig); + reset(controllerConfig); + reset(mockStore); + + // Store-level RMD version ID not found + cluster config set + doReturn(mockStore).when(mockAdmin).getStore(clusterName, storeName); + doReturn(ConfigConstants.UNSPECIFIED_REPLICATION_METADATA_VERSION).when(mockStore).getRmdVersion(); + doReturn(multiClusterConfig).when(mockAdmin).getMultiClusterConfigs(); + doReturn(controllerConfig).when(multiClusterConfig).getControllerConfig(clusterName); + doReturn(10).when(controllerConfig).getReplicationMetadataVersion(); + assertEquals(AdminUtils.getRmdVersionID(mockAdmin, storeName, clusterName), 10); + + reset(mockAdmin); + reset(multiClusterConfig); + reset(controllerConfig); + reset(mockStore); + + // Store-level RMD version ID found + doReturn(mockStore).when(mockAdmin).getStore(clusterName, storeName); + doReturn(5).when(mockStore).getRmdVersion(); + doReturn(multiClusterConfig).when(mockAdmin).getMultiClusterConfigs(); + doReturn(controllerConfig).when(multiClusterConfig).getControllerConfig(clusterName); + doReturn(10).when(controllerConfig).getReplicationMetadataVersion(); + assertEquals(AdminUtils.getRmdVersionID(mockAdmin, storeName, clusterName), 5); + verify(mockAdmin, never()).getMultiClusterConfigs(); + verify(multiClusterConfig, never()).getControllerConfig(any()); + verify(controllerConfig, never()).getReplicationMetadataVersion(); + } +} diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/util/PrimaryControllerConfigUpdateUtilsTest.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/util/PrimaryControllerConfigUpdateUtilsTest.java new file mode 100644 index 0000000000..6985ba1298 --- /dev/null +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/util/PrimaryControllerConfigUpdateUtilsTest.java @@ -0,0 +1,154 @@ +package com.linkedin.venice.controller.util; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.linkedin.venice.controller.Admin; +import com.linkedin.venice.controller.HelixVeniceClusterResources; +import com.linkedin.venice.controller.supersetschema.SupersetSchemaGenerator; +import com.linkedin.venice.meta.ReadWriteSchemaRepository; +import com.linkedin.venice.meta.Store; +import com.linkedin.venice.schema.SchemaData; +import com.linkedin.venice.schema.SchemaEntry; +import com.linkedin.venice.schema.rmd.RmdSchemaGenerator; +import com.linkedin.venice.schema.writecompute.WriteComputeSchemaConverter; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import org.apache.avro.Schema; +import org.testng.annotations.Test; + + +public class PrimaryControllerConfigUpdateUtilsTest { + private static final String VALUE_FIELD_NAME = "int_field"; + private static final String SECOND_VALUE_FIELD_NAME = "opt_int_field"; + private static final String VALUE_SCHEMA_V1_STR = "{\n" + "\"type\": \"record\",\n" + + "\"name\": \"TestValueSchema\",\n" + "\"namespace\": \"com.linkedin.venice.fastclient.schema\",\n" + + "\"fields\": [\n" + " {\"name\": \"" + VALUE_FIELD_NAME + "\", \"type\": \"int\", \"default\": 10}]\n" + "}"; + private static final String VALUE_SCHEMA_V2_STR = + "{\n" + "\"type\": \"record\",\n" + "\"name\": \"TestValueSchema\",\n" + + "\"namespace\": \"com.linkedin.venice.fastclient.schema\",\n" + "\"fields\": [\n" + "{\"name\": \"" + + SECOND_VALUE_FIELD_NAME + "\", \"type\": [\"null\", \"int\"], \"default\": null}]\n" + "}"; + private static final String SUPERSET_VALUE_SCHEMA_STR = "{\n" + "\"type\": \"record\",\n" + + "\"name\": \"TestValueSchema\",\n" + "\"namespace\": \"com.linkedin.venice.fastclient.schema\",\n" + + "\"fields\": [\n" + " {\"name\": \"" + VALUE_FIELD_NAME + "\", \"type\": \"int\", \"default\": 10},\n" + + "{\"name\": \"" + SECOND_VALUE_FIELD_NAME + "\", \"type\": [\"null\", \"int\"], \"default\": null}]\n" + "}"; + + @Test + public void testRegisterInferredSchemas() { + String clusterName = "clusterName"; + String storeName = "storeName"; + Collection storeValueSchemas = + Arrays.asList(new SchemaEntry(1, VALUE_SCHEMA_V1_STR), new SchemaEntry(2, VALUE_SCHEMA_V2_STR)); + SchemaEntry supersetSchemaEntry = new SchemaEntry(3, SUPERSET_VALUE_SCHEMA_STR); + + Admin mockAdmin = mock(Admin.class); + Store store = mock(Store.class); + + reset(mockAdmin); + reset(store); + setupMocks(mockAdmin, store, clusterName, storeName, storeValueSchemas, supersetSchemaEntry); + + doReturn(SchemaData.INVALID_VALUE_SCHEMA_ID).when(store).getLatestSuperSetValueSchemaId(); + doReturn(true).when(store).isReadComputationEnabled(); + PrimaryControllerConfigUpdateUtils.registerInferredSchemas(mockAdmin, clusterName, storeName); + validateSuperSetSchemaGenerated(mockAdmin, clusterName, storeName); + + reset(mockAdmin); + reset(store); + setupMocks(mockAdmin, store, clusterName, storeName, storeValueSchemas, supersetSchemaEntry); + + doReturn(SchemaData.INVALID_VALUE_SCHEMA_ID).when(store).getLatestSuperSetValueSchemaId(); + doReturn(true).when(store).isWriteComputationEnabled(); + PrimaryControllerConfigUpdateUtils.registerInferredSchemas(mockAdmin, clusterName, storeName); + validateSuperSetSchemaGenerated(mockAdmin, clusterName, storeName); + validateUpdateSchemaGenerated(mockAdmin, clusterName, storeName); + + reset(mockAdmin); + reset(store); + setupMocks(mockAdmin, store, clusterName, storeName, storeValueSchemas, supersetSchemaEntry); + + doReturn(1).when(store).getLatestSuperSetValueSchemaId(); + PrimaryControllerConfigUpdateUtils.registerInferredSchemas(mockAdmin, clusterName, storeName); + validateSuperSetSchemaGenerated(mockAdmin, clusterName, storeName); + + reset(mockAdmin); + reset(store); + setupMocks(mockAdmin, store, clusterName, storeName, storeValueSchemas, supersetSchemaEntry); + + doReturn(true).when(store).isActiveActiveReplicationEnabled(); + doReturn(1).when(store).getRmdVersion(); + PrimaryControllerConfigUpdateUtils.registerInferredSchemas(mockAdmin, clusterName, storeName); + validateRmdSchemaGenerated(mockAdmin, clusterName, storeName); + } + + private void setupMocks( + Admin mockAdmin, + Store store, + String clusterName, + String storeName, + Collection storeValueSchemas, + SchemaEntry supersetSchemaEntry) { + doReturn(storeName).when(store).getName(); + + SupersetSchemaGenerator supersetSchemaGenerator = mock(SupersetSchemaGenerator.class); + + doReturn(true).when(mockAdmin).isPrimary(); + doReturn(false).when(mockAdmin).isParent(); + doReturn(store).when(mockAdmin).getStore(clusterName, storeName); + doReturn(supersetSchemaGenerator).when(mockAdmin).getSupersetSchemaGenerator(clusterName); + doReturn(storeValueSchemas).when(mockAdmin).getValueSchemas(clusterName, storeName); + + doReturn(supersetSchemaEntry).when(supersetSchemaGenerator).generateSupersetSchemaFromSchemas(storeValueSchemas); + + HelixVeniceClusterResources clusterResources = mock(HelixVeniceClusterResources.class); + doReturn(clusterResources).when(mockAdmin).getHelixVeniceClusterResources(clusterName); + + ReadWriteSchemaRepository schemaRepository = mock(ReadWriteSchemaRepository.class); + doReturn(schemaRepository).when(clusterResources).getSchemaRepository(); + + doReturn(Collections.emptyList()).when(schemaRepository).getReplicationMetadataSchemas(storeName); + } + + private void validateSuperSetSchemaGenerated(Admin mockAdmin, String clusterName, String storeName) { + verify(mockAdmin).addSupersetSchema( + clusterName, + storeName, + null, + SchemaData.INVALID_VALUE_SCHEMA_ID, + SUPERSET_VALUE_SCHEMA_STR, + 3); + } + + private void validateUpdateSchemaGenerated(Admin mockAdmin, String clusterName, String storeName) { + SchemaEntry updateSchemaEntry1 = new SchemaEntry(1, VALUE_SCHEMA_V1_STR); + Schema updateSchema1 = + WriteComputeSchemaConverter.getInstance().convertFromValueRecordSchema(updateSchemaEntry1.getSchema()); + + SchemaEntry updateSchemaEntry2 = new SchemaEntry(1, VALUE_SCHEMA_V2_STR); + Schema updateSchema2 = + WriteComputeSchemaConverter.getInstance().convertFromValueRecordSchema(updateSchemaEntry2.getSchema()); + + // Ideally, we should have seen the superset schema also, but due to the static-ness of mocks, we don't see it now + verify(mockAdmin).addDerivedSchema(clusterName, storeName, 1, updateSchema1.toString()); + verify(mockAdmin).addDerivedSchema(clusterName, storeName, 2, updateSchema2.toString()); + verify(mockAdmin, times(2)).addDerivedSchema(eq(clusterName), eq(storeName), anyInt(), anyString()); + } + + private void validateRmdSchemaGenerated(Admin mockAdmin, String clusterName, String storeName) { + Schema rmdSchema1 = RmdSchemaGenerator.generateMetadataSchema(VALUE_SCHEMA_V1_STR, 1); + Schema rmdSchema2 = RmdSchemaGenerator.generateMetadataSchema(VALUE_SCHEMA_V2_STR, 1); + + // Ideally, we should have seen the superset schema also, but due to the static-ness of mocks, we don't see it now + verify(mockAdmin).addReplicationMetadataSchema(clusterName, storeName, 1, 1, rmdSchema1.toString()); + verify(mockAdmin).addReplicationMetadataSchema(clusterName, storeName, 2, 1, rmdSchema2.toString()); + verify(mockAdmin, times(2)) + .addReplicationMetadataSchema(eq(clusterName), eq(storeName), anyInt(), eq(1), anyString()); + } +} diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/util/UpdateStoreUtilsTest.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/util/UpdateStoreUtilsTest.java new file mode 100644 index 0000000000..919f35a874 --- /dev/null +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/util/UpdateStoreUtilsTest.java @@ -0,0 +1,1041 @@ +package com.linkedin.venice.controller.util; + +import static com.linkedin.venice.controllerapi.ControllerApiConstants.ACTIVE_ACTIVE_REPLICATION_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.INCREMENTAL_PUSH_ENABLED; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.NATIVE_REPLICATION_SOURCE_FABRIC; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.PARTITION_COUNT; +import static com.linkedin.venice.controllerapi.ControllerApiConstants.WRITE_COMPUTATION_ENABLED; +import static com.linkedin.venice.utils.ByteUtils.BYTES_PER_GB; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotSame; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import com.linkedin.venice.common.VeniceSystemStoreType; +import com.linkedin.venice.common.VeniceSystemStoreUtils; +import com.linkedin.venice.controller.Admin; +import com.linkedin.venice.controller.HelixVeniceClusterResources; +import com.linkedin.venice.controller.VeniceControllerClusterConfig; +import com.linkedin.venice.controller.VeniceControllerConfig; +import com.linkedin.venice.controller.VeniceControllerMultiClusterConfig; +import com.linkedin.venice.controller.VeniceHelixAdmin; +import com.linkedin.venice.controller.VeniceParentHelixAdmin; +import com.linkedin.venice.exceptions.ErrorType; +import com.linkedin.venice.exceptions.PartitionerSchemaMismatchException; +import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.exceptions.VeniceHttpException; +import com.linkedin.venice.helix.StoragePersonaRepository; +import com.linkedin.venice.meta.BufferReplayPolicy; +import com.linkedin.venice.meta.DataReplicationPolicy; +import com.linkedin.venice.meta.ETLStoreConfigImpl; +import com.linkedin.venice.meta.HybridStoreConfig; +import com.linkedin.venice.meta.HybridStoreConfigImpl; +import com.linkedin.venice.meta.PartitionerConfig; +import com.linkedin.venice.meta.PartitionerConfigImpl; +import com.linkedin.venice.meta.Store; +import com.linkedin.venice.meta.Version; +import com.linkedin.venice.meta.ViewConfig; +import com.linkedin.venice.partitioner.DefaultVenicePartitioner; +import com.linkedin.venice.partitioner.VenicePartitioner; +import com.linkedin.venice.persona.StoragePersona; +import com.linkedin.venice.pubsub.PubSubTopicRepository; +import com.linkedin.venice.pubsub.api.PubSubTopic; +import com.linkedin.venice.pubsub.manager.TopicManager; +import com.linkedin.venice.schema.SchemaData; +import com.linkedin.venice.schema.SchemaEntry; +import com.linkedin.venice.utils.TestUtils; +import com.linkedin.venice.utils.Utils; +import com.linkedin.venice.utils.VeniceProperties; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nonnull; +import org.apache.avro.Schema; +import org.apache.http.HttpStatus; +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class UpdateStoreUtilsTest { + private static final String VALUE_FIELD_NAME = "int_field"; + private static final String SECOND_VALUE_FIELD_NAME = "opt_int_field"; + private static final String VALUE_SCHEMA_V1_STR = "{\n" + "\"type\": \"record\",\n" + + "\"name\": \"TestValueSchema\",\n" + "\"namespace\": \"com.linkedin.venice.fastclient.schema\",\n" + + "\"fields\": [\n" + " {\"name\": \"" + VALUE_FIELD_NAME + "\", \"type\": \"int\", \"default\": 10}]\n" + "}"; + private static final String VALUE_SCHEMA_V2_STR = + "{\n" + "\"type\": \"record\",\n" + "\"name\": \"TestValueSchema\",\n" + + "\"namespace\": \"com.linkedin.venice.fastclient.schema\",\n" + "\"fields\": [\n" + "{\"name\": \"" + + SECOND_VALUE_FIELD_NAME + "\", \"type\": [\"null\", \"int\"], \"default\": null}]\n" + "}"; + + @Test + public void testMergeNewHybridConfigValuesToOldStore() { + String storeName = Utils.getUniqueString("storeName"); + Store store = TestUtils.createTestStore(storeName, "owner", System.currentTimeMillis()); + assertFalse(store.isHybrid()); + + Optional rewind = Optional.of(123L); + Optional lagOffset = Optional.of(1500L); + Optional timeLag = Optional.of(300L); + Optional dataReplicationPolicy = Optional.of(DataReplicationPolicy.AGGREGATE); + Optional bufferReplayPolicy = Optional.of(BufferReplayPolicy.REWIND_FROM_EOP); + HybridStoreConfig hybridStoreConfig = UpdateStoreUtils.mergeNewSettingsIntoOldHybridStoreConfig( + store, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + Assert.assertNull( + hybridStoreConfig, + "passing empty optionals and a non-hybrid store should generate a null hybrid config"); + + hybridStoreConfig = UpdateStoreUtils.mergeNewSettingsIntoOldHybridStoreConfig( + store, + rewind, + lagOffset, + timeLag, + dataReplicationPolicy, + bufferReplayPolicy); + Assert.assertNotNull(hybridStoreConfig, "specifying rewind and lagOffset should generate a valid hybrid config"); + Assert.assertEquals(hybridStoreConfig.getRewindTimeInSeconds(), 123L); + Assert.assertEquals(hybridStoreConfig.getOffsetLagThresholdToGoOnline(), 1500L); + Assert.assertEquals(hybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds(), 300L); + Assert.assertEquals(hybridStoreConfig.getDataReplicationPolicy(), DataReplicationPolicy.AGGREGATE); + + // It's okay that time lag threshold or data replication policy is not specified + hybridStoreConfig = UpdateStoreUtils.mergeNewSettingsIntoOldHybridStoreConfig( + store, + rewind, + lagOffset, + Optional.empty(), + Optional.empty(), + Optional.empty()); + Assert.assertNotNull(hybridStoreConfig, "specifying rewind and lagOffset should generate a valid hybrid config"); + Assert.assertEquals(hybridStoreConfig.getRewindTimeInSeconds(), 123L); + Assert.assertEquals(hybridStoreConfig.getOffsetLagThresholdToGoOnline(), 1500L); + Assert.assertEquals( + hybridStoreConfig.getProducerTimestampLagThresholdToGoOnlineInSeconds(), + HybridStoreConfigImpl.DEFAULT_HYBRID_TIME_LAG_THRESHOLD); + Assert.assertEquals(hybridStoreConfig.getDataReplicationPolicy(), DataReplicationPolicy.NON_AGGREGATE); + } + + @Test + public void testIsInferredStoreUpdateAllowed() { + String clusterName = "clusterName"; + String storeName = "storeName"; + Admin mockAdmin = mock(Admin.class); + + assertFalse( + UpdateStoreUtils.isInferredStoreUpdateAllowed(mockAdmin, VeniceSystemStoreUtils.getMetaStoreName(storeName))); + assertFalse( + UpdateStoreUtils + .isInferredStoreUpdateAllowed(mockAdmin, VeniceSystemStoreUtils.getDaVinciPushStatusStoreName(storeName))); + assertFalse( + UpdateStoreUtils.isInferredStoreUpdateAllowed( + mockAdmin, + VeniceSystemStoreUtils.getParticipantStoreNameForCluster(clusterName))); + assertFalse( + UpdateStoreUtils.isInferredStoreUpdateAllowed( + mockAdmin, + VeniceSystemStoreType.BATCH_JOB_HEARTBEAT_STORE.getZkSharedStoreName())); + + doReturn(false).when(mockAdmin).isPrimary(); + assertFalse(UpdateStoreUtils.isInferredStoreUpdateAllowed(mockAdmin, storeName)); + + Admin mockChildAdmin = mock(VeniceHelixAdmin.class); + doReturn(true).when(mockChildAdmin).isPrimary(); + doReturn(false).when(mockChildAdmin).isParent(); + assertTrue(UpdateStoreUtils.isInferredStoreUpdateAllowed(mockChildAdmin, storeName)); + + Admin mockParentAdmin = mock(VeniceParentHelixAdmin.class); + doReturn(true).when(mockParentAdmin).isPrimary(); + doReturn(true).when(mockParentAdmin).isParent(); + assertTrue(UpdateStoreUtils.isInferredStoreUpdateAllowed(mockParentAdmin, storeName)); + } + + @Test + public void testUpdateInferredConfig() { + String storeName = "storeName"; + Admin admin = mock(Admin.class); + Store store = mock(Store.class); + Set updatedConfigSet = new HashSet<>(); + final AtomicBoolean updaterInvoked = new AtomicBoolean(false); + + doReturn(storeName).when(store).getName(); + doReturn(true).when(admin).isPrimary(); + doReturn(false).when(admin).isParent(); + + // Config previously updated. Will not update again. + updatedConfigSet.add("key1"); + updaterInvoked.set(false); + UpdateStoreUtils.updateInferredConfig(admin, store, "key1", updatedConfigSet, () -> updaterInvoked.set(true)); + assertFalse(updaterInvoked.get()); + assertTrue(updatedConfigSet.contains("key1")); + assertEquals(updatedConfigSet.size(), 1); + + // Config not updated previously. Will update it. + updatedConfigSet.clear(); + updaterInvoked.set(false); + UpdateStoreUtils.updateInferredConfig(admin, store, "key1", updatedConfigSet, () -> updaterInvoked.set(true)); + assertTrue(updaterInvoked.get()); + assertTrue(updatedConfigSet.contains("key1")); + assertEquals(updatedConfigSet.size(), 1); + + // Config not updated previously. Will not update it for system stores. + updatedConfigSet.clear(); + updaterInvoked.set(false); + doReturn(VeniceSystemStoreUtils.getParticipantStoreNameForCluster(storeName)).when(store).getName(); + UpdateStoreUtils.updateInferredConfig(admin, store, "key1", updatedConfigSet, () -> updaterInvoked.set(true)); + assertFalse(updaterInvoked.get()); + assertTrue(updatedConfigSet.isEmpty()); + } + + @Test + public void testUpdateInferredConfigsForHybridToBatch() { + String storeName = "storeName"; + Admin admin = mock(Admin.class); + Store store = mock(Store.class); + VeniceControllerClusterConfig clusterConfig = mock(VeniceControllerClusterConfig.class); + + doReturn(true).when(admin).isPrimary(); + doReturn(false).when(admin).isParent(); + + Set updatedConfigSet = new HashSet<>(); + doReturn(storeName).when(store).getName(); + doReturn("dc-batch").when(clusterConfig).getNativeReplicationSourceFabricAsDefaultForBatchOnly(); + doReturn("dc-hybrid").when(clusterConfig).getNativeReplicationSourceFabricAsDefaultForHybrid(); + + UpdateStoreUtils.updateInferredConfigsForHybridToBatch(admin, clusterConfig, store, updatedConfigSet); + + verify(store).setIncrementalPushEnabled(false); + verify(store).setNativeReplicationSourceFabric("dc-batch"); + verify(store).setActiveActiveReplicationEnabled(false); + + assertEquals(updatedConfigSet.size(), 3); + assertTrue(updatedConfigSet.contains(INCREMENTAL_PUSH_ENABLED)); + assertTrue(updatedConfigSet.contains(NATIVE_REPLICATION_SOURCE_FABRIC)); + assertTrue(updatedConfigSet.contains(ACTIVE_ACTIVE_REPLICATION_ENABLED)); + } + + @Test + public void testUpdateInferredConfigsForBatchToHybrid() { + String clusterName = "clusterName"; + String storeName = "storeName"; + Admin admin = mock(Admin.class); + Store store = mock(Store.class); + VeniceControllerClusterConfig clusterConfig = mock(VeniceControllerClusterConfig.class); + + doReturn(true).when(admin).isPrimary(); + doReturn(false).when(admin).isParent(); + + doReturn(storeName).when(store).getName(); + doReturn(false).when(store).isSystemStore(); + doReturn(0).when(store).getPartitionCount(); + doReturn(10 * BYTES_PER_GB).when(store).getStorageQuotaInByte(); + + Set updatedConfigSet = new HashSet<>(); + doReturn(clusterName).when(clusterConfig).getClusterName(); + doReturn("dc-batch").when(clusterConfig).getNativeReplicationSourceFabricAsDefaultForBatchOnly(); + doReturn("dc-hybrid").when(clusterConfig).getNativeReplicationSourceFabricAsDefaultForHybrid(); + doReturn(false).when(clusterConfig).isActiveActiveReplicationEnabledAsDefaultForHybrid(); + doReturn(1 * BYTES_PER_GB).when(clusterConfig).getPartitionSize(); + doReturn(3).when(clusterConfig).getMinNumberOfPartitionsForHybrid(); + doReturn(100).when(clusterConfig).getMaxNumberOfPartitions(); + doReturn(1).when(clusterConfig).getPartitionCountRoundUpSize(); + doReturn(true).when(clusterConfig).isEnablePartialUpdateForHybridNonActiveActiveUserStores(); + + doReturn(Arrays.asList(new SchemaEntry(1, VALUE_SCHEMA_V1_STR), new SchemaEntry(2, VALUE_SCHEMA_V2_STR))) + .when(admin) + .getValueSchemas(clusterName, storeName); + + UpdateStoreUtils.updateInferredConfigsForBatchToHybrid(admin, clusterConfig, store, updatedConfigSet); + + verify(store).setNativeReplicationSourceFabric("dc-hybrid"); + verify(store).setActiveActiveReplicationEnabled(false); + verify(store).setPartitionCount(10); + verify(store).setWriteComputationEnabled(true); + + assertEquals(updatedConfigSet.size(), 4); + assertTrue(updatedConfigSet.contains(NATIVE_REPLICATION_SOURCE_FABRIC)); + assertTrue(updatedConfigSet.contains(ACTIVE_ACTIVE_REPLICATION_ENABLED)); + assertTrue(updatedConfigSet.contains(PARTITION_COUNT)); + assertTrue(updatedConfigSet.contains(WRITE_COMPUTATION_ENABLED)); + + // Update schemas should only be generated in dry-run mode and not registered yet. + verify(admin, never()).addDerivedSchema(eq(clusterName), eq(storeName), anyInt(), anyString()); + } + + @Test + public void testIsIncrementalPushEnabled() { + HybridStoreConfig nonHybridConfig = + new HybridStoreConfigImpl(-1, -1, -1, DataReplicationPolicy.AGGREGATE, BufferReplayPolicy.REWIND_FROM_EOP); + HybridStoreConfig hybridConfigWithNonAggregateDRP = new HybridStoreConfigImpl( + 100, + 1000, + -1, + DataReplicationPolicy.NON_AGGREGATE, + BufferReplayPolicy.REWIND_FROM_EOP); + HybridStoreConfig hybridConfigWithAggregateDRP = + new HybridStoreConfigImpl(100, 1000, -1, DataReplicationPolicy.AGGREGATE, BufferReplayPolicy.REWIND_FROM_EOP); + HybridStoreConfig hybridConfigWithActiveActiveDRP = new HybridStoreConfigImpl( + 100, + 1000, + -1, + DataReplicationPolicy.ACTIVE_ACTIVE, + BufferReplayPolicy.REWIND_FROM_EOP); + HybridStoreConfig hybridConfigWithNoneDRP = + new HybridStoreConfigImpl(100, 1000, -1, DataReplicationPolicy.NONE, BufferReplayPolicy.REWIND_FROM_EOP); + + // In single-region mode, any hybrid store should have incremental push enabled. + assertFalse(UpdateStoreUtils.isIncrementalPushEnabled(false, null)); + assertFalse(UpdateStoreUtils.isIncrementalPushEnabled(false, nonHybridConfig)); + assertTrue(UpdateStoreUtils.isIncrementalPushEnabled(false, hybridConfigWithNonAggregateDRP)); + assertTrue(UpdateStoreUtils.isIncrementalPushEnabled(false, hybridConfigWithAggregateDRP)); + assertTrue(UpdateStoreUtils.isIncrementalPushEnabled(false, hybridConfigWithActiveActiveDRP)); + assertTrue(UpdateStoreUtils.isIncrementalPushEnabled(false, hybridConfigWithNoneDRP)); + + // In multi-region mode, hybrid stores with NON_AGGREGATE DataReplicationPolicy should not have incremental push + // enabled. + assertFalse(UpdateStoreUtils.isIncrementalPushEnabled(true, null)); + assertFalse(UpdateStoreUtils.isIncrementalPushEnabled(true, nonHybridConfig)); + assertFalse(UpdateStoreUtils.isIncrementalPushEnabled(true, hybridConfigWithNonAggregateDRP)); + assertTrue(UpdateStoreUtils.isIncrementalPushEnabled(true, hybridConfigWithAggregateDRP)); + assertTrue(UpdateStoreUtils.isIncrementalPushEnabled(true, hybridConfigWithActiveActiveDRP)); + assertTrue(UpdateStoreUtils.isIncrementalPushEnabled(true, hybridConfigWithNoneDRP)); + } + + @Test + public void testValidateStoreConfigs() { + String clusterName = "clusterName"; + String storeName = "storeName"; + Admin admin = mock(Admin.class); + Store store = mock(Store.class); + VeniceControllerMultiClusterConfig multiClusterConfigs = mock(VeniceControllerMultiClusterConfig.class); + VeniceControllerConfig controllerConfig = mock(VeniceControllerConfig.class); + + doReturn(multiClusterConfigs).when(admin).getMultiClusterConfigs(); + doReturn(controllerConfig).when(multiClusterConfigs).getControllerConfig(clusterName); + + // Batch-only + incremental push is not allowed + doReturn(storeName).when(store).getName(); + doReturn(false).when(store).isHybrid(); + doReturn(true).when(store).isIncrementalPushEnabled(); + VeniceHttpException e1 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e1.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e1.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue(e1.getMessage().contains("Incremental push is only supported for hybrid stores")); + + reset(store); + + // Batch-only + write compute is not allowed + doReturn(storeName).when(store).getName(); + doReturn(false).when(store).isHybrid(); + doReturn(true).when(store).isWriteComputationEnabled(); + VeniceHttpException e2 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e2.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e2.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue(e2.getMessage().contains("Write computation is only supported for hybrid stores")); + + reset(store); + + // Hybrid store cannot have negative rewind time config + doReturn(storeName).when(store).getName(); + doReturn(true).when(store).isHybrid(); + doReturn( + new HybridStoreConfigImpl(-1, 100, -1, DataReplicationPolicy.NON_AGGREGATE, BufferReplayPolicy.REWIND_FROM_EOP)) + .when(store) + .getHybridStoreConfig(); + VeniceHttpException e3 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e3.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e3.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue(e3.getMessage().contains("Rewind time cannot be negative for a hybrid store")); + + reset(store); + + // Hybrid store cannot have negative offset lag and negative producer time lag thresholds + doReturn(storeName).when(store).getName(); + doReturn(true).when(store).isHybrid(); + doReturn( + new HybridStoreConfigImpl(100, -1, -1, DataReplicationPolicy.NON_AGGREGATE, BufferReplayPolicy.REWIND_FROM_EOP)) + .when(store) + .getHybridStoreConfig(); + VeniceHttpException e4 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e4.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e4.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue( + e4.getMessage() + .contains( + "Both offset lag threshold and producer timestamp lag threshold cannot be negative for a hybrid store")); + + reset(store); + + // Incremental push + NON_AGGREGATE DRP is not supported in multi-region mode + doReturn(true).when(controllerConfig).isMultiRegion(); + doReturn(storeName).when(store).getName(); + doReturn(true).when(store).isHybrid(); + doReturn(true).when(store).isIncrementalPushEnabled(); + doReturn( + new HybridStoreConfigImpl( + 100, + 100, + -1, + DataReplicationPolicy.NON_AGGREGATE, + BufferReplayPolicy.REWIND_FROM_EOP)).when(store).getHybridStoreConfig(); + VeniceHttpException e5 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e5.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e5.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue( + e5.getMessage() + .contains( + "Incremental push is not supported for hybrid stores with non-aggregate data replication policy")); + + reset(controllerConfig); + reset(store); + + // Incremental push + NON_AGGREGATE DRP is supported in single-region mode + doReturn(false).when(controllerConfig).isMultiRegion(); + doReturn(storeName).when(store).getName(); + doReturn(true).when(store).isHybrid(); + doReturn(true).when(store).isIncrementalPushEnabled(); + doReturn( + new HybridStoreConfigImpl( + 100, + 100, + -1, + DataReplicationPolicy.NON_AGGREGATE, + BufferReplayPolicy.REWIND_FROM_EOP)).when(store).getHybridStoreConfig(); + doReturn(new PartitionerConfigImpl()).when(store).getPartitionerConfig(); + doReturn(new SchemaEntry(1, VALUE_SCHEMA_V1_STR)).when(admin).getKeySchema(clusterName, storeName); + doReturn(SchemaData.INVALID_VALUE_SCHEMA_ID).when(store).getLatestSuperSetValueSchemaId(); + UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store); + + reset(controllerConfig); + reset(store); + + // ACTIVE_ACTIVE DRP is only supported when activeActiveReplicationEnabled = true + doReturn(storeName).when(store).getName(); + doReturn(true).when(store).isHybrid(); + doReturn(false).when(store).isActiveActiveReplicationEnabled(); + doReturn( + new HybridStoreConfigImpl( + 100, + -1, + 100, + DataReplicationPolicy.ACTIVE_ACTIVE, + BufferReplayPolicy.REWIND_FROM_EOP)).when(store).getHybridStoreConfig(); + VeniceHttpException e6 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e6.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e6.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue( + e6.getMessage() + .contains( + "Data replication policy ACTIVE_ACTIVE is only supported for hybrid stores with active-active replication enabled")); + + reset(controllerConfig); + reset(store); + + // Storage quota can not be less than 0 + doReturn(storeName).when(store).getName(); + doReturn(-5L).when(store).getStorageQuotaInByte(); + VeniceHttpException e7 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e7.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e7.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue(e7.getMessage().contains("Storage quota can not be less than 0")); + + reset(controllerConfig); + reset(store); + + // Storage quota can be -1. Special value for unlimited quota + doReturn(storeName).when(store).getName(); + doReturn(-1L).when(store).getStorageQuotaInByte(); + doReturn(new PartitionerConfigImpl()).when(store).getPartitionerConfig(); + doReturn(new SchemaEntry(1, VALUE_SCHEMA_V1_STR)).when(admin).getKeySchema(clusterName, storeName); + doReturn(SchemaData.INVALID_VALUE_SCHEMA_ID).when(store).getLatestSuperSetValueSchemaId(); + UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store); + + reset(controllerConfig); + reset(store); + + // Read quota can not be less than 0 + doReturn(storeName).when(store).getName(); + doReturn(-5L).when(store).getReadQuotaInCU(); + VeniceHttpException e8 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e8.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e8.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue(e8.getMessage().contains("Read quota can not be less than 0")); + + reset(controllerConfig); + reset(store); + + // Active-active replication is only supported for stores that also have native replication + doReturn(storeName).when(store).getName(); + doReturn(false).when(store).isNativeReplicationEnabled(); + doReturn(true).when(store).isActiveActiveReplicationEnabled(); + VeniceHttpException e9 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e9.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e9.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue( + e9.getMessage() + .contains( + "Active/Active Replication cannot be enabled for store " + store.getName() + + " since Native Replication is not enabled on it.")); + + reset(controllerConfig); + reset(store); + + // Partitioner Config cannot be null + doReturn(storeName).when(store).getName(); + doReturn(null).when(store).getPartitionerConfig(); + VeniceHttpException e10 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e10.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e10.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue(e10.getMessage().contains("Partitioner Config cannot be null")); + + reset(controllerConfig); + reset(store); + + // Active-Active is not supported when amplification factor is more than 1 + doReturn(storeName).when(store).getName(); + doReturn(new PartitionerConfigImpl(DefaultVenicePartitioner.class.getName(), new HashMap<>(), 10)).when(store) + .getPartitionerConfig(); + doReturn(true).when(store).isNativeReplicationEnabled(); + doReturn(true).when(store).isActiveActiveReplicationEnabled(); + VeniceHttpException e11 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e11.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e11.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue( + e11.getMessage() + .contains( + "Active-active replication or write computation is not supported for stores with amplification factor > 1")); + + reset(controllerConfig); + reset(store); + + // Write-compute is not supported when amplification factor is more than 1 + doReturn(storeName).when(store).getName(); + doReturn(true).when(store).isHybrid(); + doReturn( + new HybridStoreConfigImpl( + 100, + 100, + -1, + DataReplicationPolicy.NON_AGGREGATE, + BufferReplayPolicy.REWIND_FROM_EOP)).when(store).getHybridStoreConfig(); + doReturn(new PartitionerConfigImpl(DefaultVenicePartitioner.class.getName(), new HashMap<>(), 10)).when(store) + .getPartitionerConfig(); + doReturn(true).when(store).isWriteComputationEnabled(); + VeniceHttpException e12 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e12.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e12.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue( + e12.getMessage() + .contains( + "Active-active replication or write computation is not supported for stores with amplification factor > 1")); + + reset(controllerConfig); + reset(store); + + // Verify the updated partitionerConfig can be built - partitioner doesn't exist + doReturn(storeName).when(store).getName(); + doReturn(new PartitionerConfigImpl("com.linkedin.venice.InvalidPartitioner", new HashMap<>(), 10)).when(store) + .getPartitionerConfig(); + VeniceHttpException e13 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e13.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e13.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue( + e13.getMessage() + .contains( + "Partitioner Configs invalid, please verify that partitioner configs like classpath and parameters are correct!")); + + reset(controllerConfig); + reset(store); + + // Verify the updated partitionerConfig can be built - schema is not supported by partitioner + doReturn(storeName).when(store).getName(); + doReturn( + new PartitionerConfigImpl( + PickyVenicePartitioner.class.getName(), + Collections.singletonMap(PickyVenicePartitioner.SCHEMA_VALID, "false"), + 10)).when(store).getPartitionerConfig(); + VeniceHttpException e14 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e14.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e14.getErrorType(), ErrorType.INVALID_SCHEMA); + assertTrue(e14.getMessage().contains("Schema is not valid")); + + reset(controllerConfig); + reset(store); + + // Validate if the latest superset schema id is an existing value schema + doReturn(storeName).when(store).getName(); + doReturn(new PartitionerConfigImpl()).when(store).getPartitionerConfig(); + doReturn(new SchemaEntry(1, VALUE_SCHEMA_V1_STR)).when(admin).getKeySchema(clusterName, storeName); + doReturn(null).when(admin).getValueSchema(clusterName, storeName, 10); + doReturn(10).when(store).getLatestSuperSetValueSchemaId(); + VeniceHttpException e15 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e15.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e15.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue(e15.getMessage().contains("Unknown value schema id: 10 in store: storeName")); + + reset(controllerConfig); + reset(store); + + // Max compaction lag >= Min compaction lag + doReturn(storeName).when(store).getName(); + doReturn(new PartitionerConfigImpl()).when(store).getPartitionerConfig(); + doReturn(SchemaData.INVALID_VALUE_SCHEMA_ID).when(store).getLatestSuperSetValueSchemaId(); + doReturn(10L).when(store).getMaxCompactionLagSeconds(); + doReturn(100L).when(store).getMinCompactionLagSeconds(); + VeniceHttpException e16 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e16.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e16.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue( + e16.getMessage() + .contains( + "Store's max compaction lag seconds: 10 shouldn't be smaller than store's min compaction lag seconds: 100")); + + reset(controllerConfig); + reset(store); + + // ETL Proxy user must be set if ETL is enabled for current or future version + doReturn(storeName).when(store).getName(); + doReturn(new PartitionerConfigImpl()).when(store).getPartitionerConfig(); + doReturn(SchemaData.INVALID_VALUE_SCHEMA_ID).when(store).getLatestSuperSetValueSchemaId(); + doReturn(new ETLStoreConfigImpl("", true, false)).when(store).getEtlStoreConfig(); + VeniceHttpException e17 = + expectThrows(VeniceHttpException.class, () -> UpdateStoreUtils.validateStoreConfigs(admin, clusterName, store)); + assertEquals(e17.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e17.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue( + e17.getMessage().contains("Cannot enable ETL for this store because etled user proxy account is not set")); + + reset(controllerConfig); + reset(store); + } + + @Test + public void testValidateStorePartitionCountUpdate() { + String clusterName = "clusterName"; + String storeName = "storeName"; + + Admin admin = mock(Admin.class); + VeniceControllerMultiClusterConfig multiClusterConfigs = mock(VeniceControllerMultiClusterConfig.class); + VeniceControllerClusterConfig clusterConfig = mock(VeniceControllerClusterConfig.class); + HelixVeniceClusterResources clusterResources = mock(HelixVeniceClusterResources.class); + TopicManager topicManager = mock(TopicManager.class); + PubSubTopicRepository topicRepository = mock(PubSubTopicRepository.class); + PubSubTopic rtTopic = mock(PubSubTopic.class); + + doReturn(false).when(admin).isParent(); + doReturn(topicManager).when(admin).getTopicManager(); + doReturn(topicRepository).when(admin).getPubSubTopicRepository(); + doReturn(rtTopic).when(topicRepository).getTopic(Version.composeRealTimeTopic(storeName)); + + Store originalStore = mock(Store.class); + Store updatedStore = mock(Store.class); + + doReturn(3).when(clusterConfig).getMinNumberOfPartitions(); + doReturn(100).when(clusterConfig).getMaxNumberOfPartitions(); + + doReturn(clusterResources).when(admin).getHelixVeniceClusterResources(clusterName); + doReturn(clusterConfig).when(clusterResources).getConfig(); + + doReturn(storeName).when(originalStore).getName(); + doReturn(storeName).when(updatedStore).getName(); + + // Negative partition count is not allowed + doReturn(-1).when(updatedStore).getPartitionCount(); + VeniceHttpException e1 = expectThrows( + VeniceHttpException.class, + () -> UpdateStoreUtils + .validateStorePartitionCountUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore)); + assertEquals(e1.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e1.getErrorType(), ErrorType.INVALID_CONFIG); + + // Hybrid store with partition count = 0 + doReturn(true).when(updatedStore).isHybrid(); + doReturn(0).when(updatedStore).getPartitionCount(); + VeniceHttpException e2 = expectThrows( + VeniceHttpException.class, + () -> UpdateStoreUtils + .validateStorePartitionCountUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore)); + assertEquals(e2.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e2.getErrorType(), ErrorType.INVALID_CONFIG); + + // Partition count cannot be less than min partition count + doReturn(true).when(updatedStore).isHybrid(); + doReturn(1).when(updatedStore).getPartitionCount(); + VeniceHttpException e3 = expectThrows( + VeniceHttpException.class, + () -> UpdateStoreUtils + .validateStorePartitionCountUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore)); + assertEquals(e3.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e3.getErrorType(), ErrorType.INVALID_CONFIG); + + // Partition count cannot be greater than max partition count + doReturn(false).when(updatedStore).isHybrid(); + doReturn(1000).when(updatedStore).getPartitionCount(); + VeniceHttpException e4 = expectThrows( + VeniceHttpException.class, + () -> UpdateStoreUtils + .validateStorePartitionCountUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore)); + assertEquals(e4.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e4.getErrorType(), ErrorType.INVALID_CONFIG); + + // Partition count change for hybrid stores is not allowed + doReturn(true).when(originalStore).isHybrid(); + doReturn(true).when(updatedStore).isHybrid(); + doReturn(10).when(originalStore).getPartitionCount(); + doReturn(20).when(updatedStore).getPartitionCount(); + VeniceHttpException e5 = expectThrows( + VeniceHttpException.class, + () -> UpdateStoreUtils + .validateStorePartitionCountUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore)); + assertEquals(e5.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e5.getErrorType(), ErrorType.INVALID_CONFIG); + + // Partition count update is allowed if RT topic doesn't exist + doReturn(true).when(originalStore).isHybrid(); + doReturn(true).when(updatedStore).isHybrid(); + doReturn(10).when(originalStore).getPartitionCount(); + doReturn(10).when(updatedStore).getPartitionCount(); + doReturn(false).when(topicManager).containsTopic(rtTopic); + UpdateStoreUtils + .validateStorePartitionCountUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore); + + // Partition count update is allowed if RT topic exists and partition count matches the store's partition count + doReturn(true).when(originalStore).isHybrid(); + doReturn(true).when(updatedStore).isHybrid(); + doReturn(10).when(originalStore).getPartitionCount(); + doReturn(10).when(updatedStore).getPartitionCount(); + doReturn(true).when(topicManager).containsTopic(rtTopic); + doReturn(10).when(topicManager).getPartitionCount(rtTopic); + UpdateStoreUtils + .validateStorePartitionCountUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore); + + // Partition count update is not allowed if RT topic exists and partition count is different from the store's + // partition count + doReturn(true).when(originalStore).isHybrid(); + doReturn(true).when(updatedStore).isHybrid(); + doReturn(10).when(originalStore).getPartitionCount(); + doReturn(10).when(updatedStore).getPartitionCount(); + doReturn(true).when(topicManager).containsTopic(rtTopic); + doReturn(20).when(topicManager).getPartitionCount(rtTopic); + VeniceHttpException e6 = expectThrows( + VeniceHttpException.class, + () -> UpdateStoreUtils + .validateStorePartitionCountUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore)); + assertEquals(e6.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e6.getErrorType(), ErrorType.INVALID_CONFIG); + + // Partition count change for batch stores is allowed + doReturn(true).when(originalStore).isHybrid(); + doReturn(false).when(updatedStore).isHybrid(); + doReturn(10).when(originalStore).getPartitionCount(); + doReturn(20).when(updatedStore).getPartitionCount(); + // No exception is thrown + UpdateStoreUtils + .validateStorePartitionCountUpdate(admin, multiClusterConfigs, clusterName, originalStore, updatedStore); + } + + @Test + public void testValidateStorePartitionerUpdate() { + String clusterName = "clusterName"; + String storeName = "storeName"; + + Store originalStore = mock(Store.class); + Store updatedStore = mock(Store.class); + + doReturn(storeName).when(originalStore).getName(); + doReturn(storeName).when(updatedStore).getName(); + + // Partitioner param update is allowed for batch-only stores + doReturn(false).when(originalStore).isHybrid(); + doReturn(false).when(updatedStore).isHybrid(); + UpdateStoreUtils.validateStorePartitionerUpdate(clusterName, originalStore, updatedStore); + + // Partitioner param update is allowed during hybrid to batch conversion + doReturn(true).when(originalStore).isHybrid(); + doReturn(false).when(updatedStore).isHybrid(); + UpdateStoreUtils.validateStorePartitionerUpdate(clusterName, originalStore, updatedStore); + + // Partitioner param update is allowed during batch to hybrid conversion + doReturn(false).when(originalStore).isHybrid(); + doReturn(true).when(updatedStore).isHybrid(); + UpdateStoreUtils.validateStorePartitionerUpdate(clusterName, originalStore, updatedStore); + + PartitionerConfig originalPartitionerConfig; + PartitionerConfig updatedPartitionerConfig; + + // Partitioner class update is not allowed for hybrid stores + doReturn(true).when(originalStore).isHybrid(); + doReturn(true).when(updatedStore).isHybrid(); + originalPartitionerConfig = new PartitionerConfigImpl("ClassA", Collections.singletonMap("key1", "value1"), 1); + updatedPartitionerConfig = new PartitionerConfigImpl("ClassB", Collections.singletonMap("key1", "value1"), 1); + doReturn(originalPartitionerConfig).when(originalStore).getPartitionerConfig(); + doReturn(updatedPartitionerConfig).when(updatedStore).getPartitionerConfig(); + VeniceHttpException e1 = expectThrows( + VeniceHttpException.class, + () -> UpdateStoreUtils.validateStorePartitionerUpdate(clusterName, originalStore, updatedStore)); + assertEquals(e1.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e1.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue(e1.getMessage().contains("Partitioner class cannot be changed for hybrid store")); + + // Partitioner param update is not allowed for hybrid stores + doReturn(true).when(originalStore).isHybrid(); + doReturn(true).when(updatedStore).isHybrid(); + originalPartitionerConfig = new PartitionerConfigImpl("ClassA", Collections.singletonMap("key1", "value1"), 1); + updatedPartitionerConfig = new PartitionerConfigImpl("ClassA", Collections.singletonMap("key2", "value2"), 1); + doReturn(originalPartitionerConfig).when(originalStore).getPartitionerConfig(); + doReturn(updatedPartitionerConfig).when(updatedStore).getPartitionerConfig(); + VeniceHttpException e2 = expectThrows( + VeniceHttpException.class, + () -> UpdateStoreUtils.validateStorePartitionerUpdate(clusterName, originalStore, updatedStore)); + assertEquals(e2.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST); + assertEquals(e2.getErrorType(), ErrorType.INVALID_CONFIG); + assertTrue(e2.getMessage().contains("Partitioner params cannot be changed for hybrid store")); + + // Amplification factor changes are allowed for hybrid stores + doReturn(true).when(originalStore).isHybrid(); + doReturn(true).when(updatedStore).isHybrid(); + originalPartitionerConfig = new PartitionerConfigImpl("ClassA", Collections.singletonMap("key1", "value1"), 1); + updatedPartitionerConfig = new PartitionerConfigImpl("ClassA", Collections.singletonMap("key1", "value1"), 10); + doReturn(originalPartitionerConfig).when(originalStore).getPartitionerConfig(); + doReturn(updatedPartitionerConfig).when(updatedStore).getPartitionerConfig(); + UpdateStoreUtils.validateStorePartitionerUpdate(clusterName, originalStore, updatedStore); + } + + @Test + public void testValidatePersona() { + String clusterName = "clusterName"; + String storeName = "storeName"; + + Store store = mock(Store.class); + Admin admin = mock(Admin.class); + HelixVeniceClusterResources clusterResources = mock(HelixVeniceClusterResources.class); + StoragePersonaRepository personaRepository = mock(StoragePersonaRepository.class); + StoragePersona persona = mock(StoragePersona.class); + + doReturn(storeName).when(store).getName(); + + doReturn(clusterResources).when(admin).getHelixVeniceClusterResources(clusterName); + doReturn(personaRepository).when(clusterResources).getStoragePersonaRepository(); + + // Persona not updated. Store doesn't have an existing persona. Update is allowed. + doReturn(null).when(personaRepository).getPersonaContainingStore(storeName); + UpdateStoreUtils.validatePersona(admin, clusterName, store, Optional.empty()); + + // Persona not updated. Store has an existing persona. Update is allowed if persona repo allows. + doReturn(persona).when(personaRepository).getPersonaContainingStore(storeName); + // Validation doesn't throw exception -> update is allowed + doNothing().when(personaRepository).validateAddUpdatedStore(any(), any()); + UpdateStoreUtils.validatePersona(admin, clusterName, store, Optional.empty()); + // Validation throws exception -> update is not allowed + doThrow(new VeniceException()).when(personaRepository).validateAddUpdatedStore(any(), any()); + assertThrows( + VeniceException.class, + () -> UpdateStoreUtils.validatePersona(admin, clusterName, store, Optional.empty())); + + String updatedPersona = "persona2"; + // Persona updated. New persona doesn't exist. Update is not allowed. + doReturn(null).when(personaRepository).getPersonaContainingStore(storeName); + doReturn(null).when(admin).getStoragePersona(clusterName, updatedPersona); + assertThrows( + VeniceException.class, + () -> UpdateStoreUtils.validatePersona(admin, clusterName, store, Optional.of(updatedPersona))); + + // Persona updated. New persona exists. Update is allowed if persona repo allows. + doReturn(null).when(personaRepository).getPersonaContainingStore(storeName); + doReturn(persona).when(admin).getStoragePersona(clusterName, updatedPersona); + // Validation doesn't throw exception -> update is allowed + doNothing().when(personaRepository).validateAddUpdatedStore(any(), any()); + UpdateStoreUtils.validatePersona(admin, clusterName, store, Optional.of(updatedPersona)); + // Validation throws exception -> update is not allowed + doThrow(new VeniceException()).when(personaRepository).validateAddUpdatedStore(any(), any()); + assertThrows( + VeniceException.class, + () -> UpdateStoreUtils.validatePersona(admin, clusterName, store, Optional.of(updatedPersona))); + } + + @Test + public void testMergeNewSettingsIntoOldPartitionerConfig() { + String storeName = "storeName"; + Store store = mock(Store.class); + + PartitionerConfig oldPartitionerConfig = new PartitionerConfigImpl(); + + doReturn(storeName).when(store).getName(); + doReturn(oldPartitionerConfig).when(store).getPartitionerConfig(); + + // No updates to the store partitioner configs should return the same partitioner configs + assertSame( + UpdateStoreUtils + .mergeNewSettingsIntoOldPartitionerConfig(store, Optional.empty(), Optional.empty(), Optional.empty()), + oldPartitionerConfig); + + String updatedPartitionerClass = "Class B"; + Map updatedPartitionerParams = Collections.singletonMap("key1", "value1"); + int updatedAmpFactor = 10; + + PartitionerConfig newPartitionerConfig = UpdateStoreUtils.mergeNewSettingsIntoOldPartitionerConfig( + store, + Optional.of(updatedPartitionerClass), + Optional.of(updatedPartitionerParams), + Optional.of(updatedAmpFactor)); + assertNotSame(newPartitionerConfig, oldPartitionerConfig); // Should be a new object + assertEquals(newPartitionerConfig.getPartitionerClass(), updatedPartitionerClass); + assertEquals(newPartitionerConfig.getPartitionerParams(), updatedPartitionerParams); + assertEquals(newPartitionerConfig.getAmplificationFactor(), updatedAmpFactor); + + // Even if the store doesn't have a partitioner config, the new partitioner config should be returned + doReturn(null).when(store).getPartitionerConfig(); + PartitionerConfig newPartitionerConfig2 = UpdateStoreUtils.mergeNewSettingsIntoOldPartitionerConfig( + store, + Optional.of(updatedPartitionerClass), + Optional.of(updatedPartitionerParams), + Optional.of(updatedAmpFactor)); + assertNotSame(newPartitionerConfig2, oldPartitionerConfig); // Should be a new object + assertEquals(newPartitionerConfig2.getPartitionerClass(), updatedPartitionerClass); + assertEquals(newPartitionerConfig2.getPartitionerParams(), updatedPartitionerParams); + assertEquals(newPartitionerConfig2.getAmplificationFactor(), updatedAmpFactor); + } + + @Test + public void testAddNewViewConfigsIntoOldConfigs() { + String storeName = "storeName"; + Store store = mock(Store.class); + String classA = "ClassA"; + String classB = "ClassB"; + String classC = "ClassC"; + + ViewConfig viewConfigA = mock(ViewConfig.class); + ViewConfig viewConfigB = mock(ViewConfig.class); + ViewConfig viewConfigC = mock(ViewConfig.class); + + Map viewConfigMap = new HashMap() { + { + put(classA, viewConfigA); + put(classB, viewConfigB); + } + }; + + doReturn(storeName).when(store).getName(); + doReturn(viewConfigMap).when(store).getViewConfigs(); + + Map mergedViewConfig1 = + UpdateStoreUtils.addNewViewConfigsIntoOldConfigs(store, classC, viewConfigC); + assertEquals(mergedViewConfig1.size(), 3); + assertEquals(mergedViewConfig1.get(classA), viewConfigA); + assertEquals(mergedViewConfig1.get(classB), viewConfigB); + assertEquals(mergedViewConfig1.get(classC), viewConfigC); + + Map mergedViewConfig2 = + UpdateStoreUtils.addNewViewConfigsIntoOldConfigs(store, classB, viewConfigC); + assertEquals(mergedViewConfig2.size(), 2); + assertEquals(mergedViewConfig2.get(classA), viewConfigA); + assertEquals(mergedViewConfig2.get(classB), viewConfigC); + + doReturn(null).when(store).getViewConfigs(); + Map mergedViewConfig3 = + UpdateStoreUtils.addNewViewConfigsIntoOldConfigs(store, classA, viewConfigA); + assertEquals(mergedViewConfig3.size(), 1); + assertEquals(mergedViewConfig3.get(classA), viewConfigA); + } + + @Test + public void testRemoveViewConfigFromStoreViewConfigMap() { + String storeName = "storeName"; + Store store = mock(Store.class); + String classA = "ClassA"; + String classB = "ClassB"; + + ViewConfig viewConfigA = mock(ViewConfig.class); + ViewConfig viewConfigB = mock(ViewConfig.class); + + Map viewConfigMap = new HashMap() { + { + put(classA, viewConfigA); + put(classB, viewConfigB); + } + }; + + doReturn(storeName).when(store).getName(); + doReturn(viewConfigMap).when(store).getViewConfigs(); + + Map newViewConfig1 = UpdateStoreUtils.removeViewConfigFromStoreViewConfigMap(store, classB); + assertEquals(newViewConfig1.size(), 1); + assertEquals(newViewConfig1.get(classA), viewConfigA); + + doReturn(null).when(store).getViewConfigs(); + Map newViewConfig2 = UpdateStoreUtils.removeViewConfigFromStoreViewConfigMap(store, classA); + assertTrue(newViewConfig2.isEmpty()); + } + + public static class PickyVenicePartitioner extends VenicePartitioner { + private static final String SCHEMA_VALID = "SCHEMA_VALID"; + + public PickyVenicePartitioner() { + this(VeniceProperties.empty(), null); + } + + public PickyVenicePartitioner(VeniceProperties props) { + this(props, null); + } + + public PickyVenicePartitioner(VeniceProperties props, Schema schema) { + super(props, schema); + } + + @Override + public int getPartitionId(byte[] keyBytes, int numPartitions) { + return 0; + } + + @Override + public int getPartitionId(ByteBuffer keyByteBuffer, int numPartitions) { + return 0; + } + + @Override + protected void checkSchema(@Nonnull Schema keySchema) throws PartitionerSchemaMismatchException { + if (!props.getBoolean(SCHEMA_VALID)) { + throw new PartitionerSchemaMismatchException("Schema is not valid"); + } + } + } +} diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/utils/ParentControllerConfigUpdateUtilsTest.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/utils/ParentControllerConfigUpdateUtilsTest.java deleted file mode 100644 index 7f6fcf4429..0000000000 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/utils/ParentControllerConfigUpdateUtilsTest.java +++ /dev/null @@ -1,308 +0,0 @@ -package com.linkedin.venice.controller.utils; - -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.linkedin.venice.controller.HelixVeniceClusterResources; -import com.linkedin.venice.controller.VeniceControllerClusterConfig; -import com.linkedin.venice.controller.VeniceHelixAdmin; -import com.linkedin.venice.controller.VeniceParentHelixAdmin; -import com.linkedin.venice.controller.kafka.protocol.admin.UpdateStore; -import com.linkedin.venice.controller.util.ParentControllerConfigUpdateUtils; -import com.linkedin.venice.meta.Store; -import com.linkedin.venice.schema.SchemaEntry; -import com.linkedin.venice.utils.TestWriteUtils; -import java.util.Collections; -import java.util.Optional; -import org.testng.Assert; -import org.testng.annotations.Test; - - -public class ParentControllerConfigUpdateUtilsTest { - @Test - public void testPartialUpdateConfigUpdate() { - VeniceParentHelixAdmin parentHelixAdmin = mock(VeniceParentHelixAdmin.class); - VeniceHelixAdmin veniceHelixAdmin = mock(VeniceHelixAdmin.class); - String cluster = "foo"; - String storeName = "bar"; - Store store = mock(Store.class); - when(parentHelixAdmin.getVeniceHelixAdmin()).thenReturn(veniceHelixAdmin); - when(veniceHelixAdmin.getStore(anyString(), anyString())).thenReturn(store); - HelixVeniceClusterResources helixVeniceClusterResources = mock(HelixVeniceClusterResources.class); - VeniceControllerClusterConfig veniceControllerClusterConfig = mock(VeniceControllerClusterConfig.class); - when(helixVeniceClusterResources.getConfig()).thenReturn(veniceControllerClusterConfig); - when(veniceHelixAdmin.getHelixVeniceClusterResources(anyString())).thenReturn(helixVeniceClusterResources); - SchemaEntry schemaEntry = new SchemaEntry(1, TestWriteUtils.USER_WITH_DEFAULT_SCHEMA); - when(veniceHelixAdmin.getValueSchemas(anyString(), anyString())).thenReturn(Collections.singletonList(schemaEntry)); - when(parentHelixAdmin.getValueSchemas(anyString(), anyString())).thenReturn(Collections.singletonList(schemaEntry)); - - /** - * Explicit request. - */ - Optional partialUpdateRequest = Optional.of(true); - // Case 1: partial update config updated. - UpdateStore setStore = new UpdateStore(); - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyPartialUpdateConfig( - parentHelixAdmin, - cluster, - storeName, - partialUpdateRequest, - setStore, - true)); - // Case 2: partial update config updated. - setStore = new UpdateStore(); - when(store.isWriteComputationEnabled()).thenReturn(true); - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyPartialUpdateConfig( - parentHelixAdmin, - cluster, - storeName, - partialUpdateRequest, - setStore, - true)); - // Case 3: partial update config updated. - partialUpdateRequest = Optional.of(false); - when(store.isWriteComputationEnabled()).thenReturn(true); - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyPartialUpdateConfig( - parentHelixAdmin, - cluster, - storeName, - partialUpdateRequest, - setStore, - true)); - // Case 4: partial update config updated. - setStore = new UpdateStore(); - when(store.isWriteComputationEnabled()).thenReturn(false); - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyPartialUpdateConfig( - parentHelixAdmin, - cluster, - storeName, - partialUpdateRequest, - setStore, - true)); - - /** - * No request. - */ - partialUpdateRequest = Optional.empty(); - when(veniceControllerClusterConfig.isEnablePartialUpdateForHybridActiveActiveUserStores()).thenReturn(false); - when(veniceControllerClusterConfig.isEnablePartialUpdateForHybridNonActiveActiveUserStores()).thenReturn(false); - // Case 1: partial update config not updated. - setStore = new UpdateStore(); - Assert.assertFalse( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyPartialUpdateConfig( - parentHelixAdmin, - cluster, - storeName, - partialUpdateRequest, - setStore, - true)); - setStore.activeActiveReplicationEnabled = true; - Assert.assertFalse( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyPartialUpdateConfig( - parentHelixAdmin, - cluster, - storeName, - partialUpdateRequest, - setStore, - true)); - // Case 2: partial update config updated. - when(veniceControllerClusterConfig.isEnablePartialUpdateForHybridActiveActiveUserStores()).thenReturn(true); - when(veniceControllerClusterConfig.isEnablePartialUpdateForHybridNonActiveActiveUserStores()).thenReturn(true); - setStore = new UpdateStore(); - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyPartialUpdateConfig( - parentHelixAdmin, - cluster, - storeName, - partialUpdateRequest, - setStore, - true)); - setStore.activeActiveReplicationEnabled = true; - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyPartialUpdateConfig( - parentHelixAdmin, - cluster, - storeName, - partialUpdateRequest, - setStore, - true)); - } - - @Test - public void testChunkingConfigUpdate() { - VeniceParentHelixAdmin parentHelixAdmin = mock(VeniceParentHelixAdmin.class); - VeniceHelixAdmin veniceHelixAdmin = mock(VeniceHelixAdmin.class); - String cluster = "foo"; - String storeName = "bar"; - Store store = mock(Store.class); - when(parentHelixAdmin.getVeniceHelixAdmin()).thenReturn(veniceHelixAdmin); - when(veniceHelixAdmin.getStore(anyString(), anyString())).thenReturn(store); - - /** - * Explicit request. - */ - Optional chunkingRequest = Optional.of(true); - when(store.isChunkingEnabled()).thenReturn(false); - // Case 1: chunking config updated. - UpdateStore setStore = new UpdateStore(); - setStore.chunkingEnabled = false; - Assert.assertTrue( - ParentControllerConfigUpdateUtils - .checkAndMaybeApplyChunkingConfigChange(parentHelixAdmin, cluster, storeName, chunkingRequest, setStore)); - // Case 2: chunking config updated. - setStore = new UpdateStore(); - setStore.chunkingEnabled = false; - when(store.isChunkingEnabled()).thenReturn(true); - Assert.assertTrue( - ParentControllerConfigUpdateUtils - .checkAndMaybeApplyChunkingConfigChange(parentHelixAdmin, cluster, storeName, chunkingRequest, setStore)); - // Case 3: chunking config updated. - chunkingRequest = Optional.of(false); - when(store.isChunkingEnabled()).thenReturn(true); - Assert.assertTrue( - ParentControllerConfigUpdateUtils - .checkAndMaybeApplyChunkingConfigChange(parentHelixAdmin, cluster, storeName, chunkingRequest, setStore)); - // Case 4: chunking config updated. - setStore = new UpdateStore(); - when(store.isChunkingEnabled()).thenReturn(false); - Assert.assertTrue( - ParentControllerConfigUpdateUtils - .checkAndMaybeApplyChunkingConfigChange(parentHelixAdmin, cluster, storeName, chunkingRequest, setStore)); - /** - * No request. - */ - chunkingRequest = Optional.empty(); - when(store.isWriteComputationEnabled()).thenReturn(false); - // Case 1: already enabled, chunking config not updated. - when(store.isChunkingEnabled()).thenReturn(true); - setStore = new UpdateStore(); - setStore.writeComputationEnabled = true; - Assert.assertFalse( - ParentControllerConfigUpdateUtils - .checkAndMaybeApplyChunkingConfigChange(parentHelixAdmin, cluster, storeName, chunkingRequest, setStore)); - // Case 2: chunking config updated. - when(store.isChunkingEnabled()).thenReturn(false); - setStore = new UpdateStore(); - setStore.writeComputationEnabled = true; - Assert.assertTrue( - ParentControllerConfigUpdateUtils - .checkAndMaybeApplyChunkingConfigChange(parentHelixAdmin, cluster, storeName, chunkingRequest, setStore)); - } - - @Test - public void testRmdChunkingConfigUpdate() { - VeniceParentHelixAdmin parentHelixAdmin = mock(VeniceParentHelixAdmin.class); - VeniceHelixAdmin veniceHelixAdmin = mock(VeniceHelixAdmin.class); - String cluster = "foo"; - String storeName = "bar"; - Store store = mock(Store.class); - when(parentHelixAdmin.getVeniceHelixAdmin()).thenReturn(veniceHelixAdmin); - when(veniceHelixAdmin.getStore(anyString(), anyString())).thenReturn(store); - - /** - * Explicit request. - */ - Optional chunkingRequest = Optional.of(true); - when(store.isChunkingEnabled()).thenReturn(false); - // Case 1: chunking config updated. - UpdateStore setStore = new UpdateStore(); - setStore.chunkingEnabled = false; - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyRmdChunkingConfigChange( - parentHelixAdmin, - cluster, - storeName, - chunkingRequest, - setStore)); - // Case 2: chunking config updated. - setStore = new UpdateStore(); - setStore.chunkingEnabled = false; - when(store.isChunkingEnabled()).thenReturn(true); - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyRmdChunkingConfigChange( - parentHelixAdmin, - cluster, - storeName, - chunkingRequest, - setStore)); - // Case 3: chunking config updated. - chunkingRequest = Optional.of(false); - when(store.isChunkingEnabled()).thenReturn(true); - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyRmdChunkingConfigChange( - parentHelixAdmin, - cluster, - storeName, - chunkingRequest, - setStore)); - // Case 4: chunking config updated. - setStore = new UpdateStore(); - when(store.isChunkingEnabled()).thenReturn(false); - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyRmdChunkingConfigChange( - parentHelixAdmin, - cluster, - storeName, - chunkingRequest, - setStore)); - - /** - * No request. - */ - chunkingRequest = Optional.empty(); - when(store.isWriteComputationEnabled()).thenReturn(false); - // Case 1: already enabled, chunking config not updated. - when(store.isChunkingEnabled()).thenReturn(true); - setStore = new UpdateStore(); - setStore.writeComputationEnabled = true; - Assert.assertFalse( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyRmdChunkingConfigChange( - parentHelixAdmin, - cluster, - storeName, - chunkingRequest, - setStore)); - // Case 2: chunking config not updated. - when(store.isChunkingEnabled()).thenReturn(false); - setStore = new UpdateStore(); - setStore.writeComputationEnabled = true; - setStore.activeActiveReplicationEnabled = false; - Assert.assertFalse( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyRmdChunkingConfigChange( - parentHelixAdmin, - cluster, - storeName, - chunkingRequest, - setStore)); - // Case 3: chunking config not updated. - when(store.isChunkingEnabled()).thenReturn(false); - setStore = new UpdateStore(); - setStore.writeComputationEnabled = false; - setStore.activeActiveReplicationEnabled = true; - Assert.assertFalse( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyRmdChunkingConfigChange( - parentHelixAdmin, - cluster, - storeName, - chunkingRequest, - setStore)); - // Case 4: chunking config updated. - when(store.isChunkingEnabled()).thenReturn(false); - setStore = new UpdateStore(); - setStore.writeComputationEnabled = true; - setStore.activeActiveReplicationEnabled = true; - Assert.assertTrue( - ParentControllerConfigUpdateUtils.checkAndMaybeApplyRmdChunkingConfigChange( - parentHelixAdmin, - cluster, - storeName, - chunkingRequest, - setStore)); - - } -} diff --git a/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java b/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java index 0150fa35cb..d4879217a5 100644 --- a/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java +++ b/tests/venice-pulsar-test/src/pulsarIntegrationTest/java/com/linkedin/venice/pulsar/sink/PulsarVeniceSinkTest.java @@ -262,7 +262,8 @@ private void updateVeniceStoreQuotas(String veniceControllerUrl, String jar, Str "bash", "-c", "java -jar " + jar + " --update-store --url " + veniceControllerUrl + " --cluster " + clusterName + " --store " - + storeName + " --storage-quota -1 --incremental-push-enabled true"); + + storeName + + " --storage-quota -1 --hybrid-rewind-seconds 86400 --hybrid-offset-lag 1000 --hybrid-data-replication-policy NONE"); execByServiceAsssertNoStdErr( "venice-client",