Skip to content

Commit

Permalink
Create a new flag on RocksDB for high spec hardware to boost performa…
Browse files Browse the repository at this point in the history 
…nce (hyperledger#4423)

* Create a new flag on RocksDB (--Xplugin-rocksdb-high-spec-enabled) for high spec hardware to boost performance.
Signed-off-by: Ameziane H <[email protected]>
  • Loading branch information
@ahamlat @eum602
ahamlat authored and eum602 committed Nov 3, 2023
1 parent a597240 commit 27ee445
Show file tree
Hide file tree
Showing 11 changed files with 152 additions and 56 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
- Resets engine QoS timer with every call to the engine API instead of only when ExchangeTransitionConfiguration is called [#4411](https://github.com/hyperledger/besu/issues/4411)
- ExchangeTransitionConfiguration mismatch will only submit a debug log not a warning anymore [#4411](https://github.com/hyperledger/besu/issues/4411)
- Upgrade besu-native to 0.6.1 and include linux arm64 build of bls12-381 [#4416](https://github.com/hyperledger/besu/pull/4416)
- Create a new flag on RocksDB (_--Xplugin-rocksdb-high-spec-enabled_) for high spec hardware to boost performance
- Transaction pool improvements to avoid filling the pool with not executable transactions, that could result in empty or semi-empty block proposals [#4425](https://github.com/hyperledger/besu/pull/4425)

### Bug Fixes
Expand Down
18 changes: 10 additions & 8 deletions ...ests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@
package org.hyperledger.besu.tests.acceptance.dsl.privacy;

import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_BACKGROUND_THREAD_COUNT;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_CACHE_CAPACITY;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_IS_HIGH_SPEC;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_BACKGROUND_COMPACTIONS;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_OPEN_FILES;

import org.hyperledger.besu.crypto.KeyPairUtil;
import org.hyperledger.besu.datatypes.Address;
Expand Down Expand Up @@ -65,10 +70,6 @@
public class PrivacyNode implements AutoCloseable {

private static final Logger LOG = LoggerFactory.getLogger(PrivacyNode.class);
private static final int MAX_OPEN_FILES = 1024;
private static final long CACHE_CAPACITY = 8388608;
private static final int MAX_BACKGROUND_COMPACTIONS = 4;
private static final int BACKGROUND_THREAD_COUNT = 4;

private final EnclaveTestHarness enclave;
private final BesuNode besu;
Expand Down Expand Up @@ -275,10 +276,11 @@ private PrivacyStorageProvider createKeyValueStorageProvider(
new RocksDBKeyValueStorageFactory(
() ->
new RocksDBFactoryConfiguration(
MAX_OPEN_FILES,
MAX_BACKGROUND_COMPACTIONS,
BACKGROUND_THREAD_COUNT,
CACHE_CAPACITY),
DEFAULT_MAX_OPEN_FILES,
DEFAULT_MAX_BACKGROUND_COMPACTIONS,
DEFAULT_BACKGROUND_THREAD_COUNT,
DEFAULT_CACHE_CAPACITY,
DEFAULT_IS_HIGH_SPEC),
Arrays.asList(KeyValueSegmentIdentifier.values()),
RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS)))
.withCommonConfiguration(new BesuConfigurationImpl(dataLocation, dbLocation))
Expand Down
18 changes: 10 additions & 8 deletions besu/src/test/java/org/hyperledger/besu/PrivacyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.ethereum.core.PrivacyParameters.DEFAULT_PRIVACY;
import static org.hyperledger.besu.ethereum.core.PrivacyParameters.FLEXIBLE_PRIVACY;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_BACKGROUND_THREAD_COUNT;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_CACHE_CAPACITY;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_IS_HIGH_SPEC;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_BACKGROUND_COMPACTIONS;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_OPEN_FILES;

import org.hyperledger.besu.config.GenesisConfigFile;
import org.hyperledger.besu.controller.BesuController;
Expand Down Expand Up @@ -61,10 +66,6 @@
@RunWith(MockitoJUnitRunner.class)
public class PrivacyTest {

private static final int MAX_OPEN_FILES = 1024;
private static final long CACHE_CAPACITY = 8388608;
private static final int MAX_BACKGROUND_COMPACTIONS = 4;
private static final int BACKGROUND_THREAD_COUNT = 4;
private final Vertx vertx = Vertx.vertx();

@Rule public final TemporaryFolder folder = new TemporaryFolder();
Expand Down Expand Up @@ -131,10 +132,11 @@ private PrivacyStorageProvider createKeyValueStorageProvider(
new RocksDBKeyValueStorageFactory(
() ->
new RocksDBFactoryConfiguration(
MAX_OPEN_FILES,
MAX_BACKGROUND_COMPACTIONS,
BACKGROUND_THREAD_COUNT,
CACHE_CAPACITY),
DEFAULT_MAX_OPEN_FILES,
DEFAULT_MAX_BACKGROUND_COMPACTIONS,
DEFAULT_BACKGROUND_THREAD_COUNT,
DEFAULT_CACHE_CAPACITY,
DEFAULT_IS_HIGH_SPEC),
Arrays.asList(KeyValueSegmentIdentifier.values()),
RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS)))
.withCommonConfiguration(new BesuConfigurationImpl(dataDir, dbDir))
Expand Down
19 changes: 10 additions & 9 deletions besu/src/test/java/org/hyperledger/besu/RunnerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@
import static java.util.Collections.emptySet;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.cli.config.NetworkName.DEV;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_BACKGROUND_THREAD_COUNT;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_CACHE_CAPACITY;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_IS_HIGH_SPEC;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_BACKGROUND_COMPACTIONS;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_OPEN_FILES;

import org.hyperledger.besu.cli.config.EthNetworkConfig;
import org.hyperledger.besu.config.GenesisConfigFile;
Expand Down Expand Up @@ -102,11 +107,6 @@
@RunWith(MockitoJUnitRunner.class)
public final class RunnerTest {

private static final int MAX_OPEN_FILES = 1024;
private static final long CACHE_CAPACITY = 8388608;
private static final int MAX_BACKGROUND_COMPACTIONS = 4;
private static final int BACKGROUND_THREAD_COUNT = 4;

private Vertx vertx;

@Before
Expand Down Expand Up @@ -415,10 +415,11 @@ private StorageProvider createKeyValueStorageProvider(final Path dataDir, final
new RocksDBKeyValueStorageFactory(
() ->
new RocksDBFactoryConfiguration(
MAX_OPEN_FILES,
MAX_BACKGROUND_COMPACTIONS,
BACKGROUND_THREAD_COUNT,
CACHE_CAPACITY),
DEFAULT_MAX_OPEN_FILES,
DEFAULT_MAX_BACKGROUND_COMPACTIONS,
DEFAULT_BACKGROUND_THREAD_COUNT,
DEFAULT_CACHE_CAPACITY,
DEFAULT_IS_HIGH_SPEC),
Arrays.asList(KeyValueSegmentIdentifier.values()),
RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS))
.withCommonConfiguration(new BesuConfigurationImpl(dataDir, dbDir))
Expand Down
4 changes: 3 additions & 1 deletion ...java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryWorldStateArchive;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_BACKGROUND_THREAD_COUNT;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_CACHE_CAPACITY;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_IS_HIGH_SPEC;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_BACKGROUND_COMPACTIONS;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_OPEN_FILES;

Expand Down Expand Up @@ -167,7 +168,8 @@ private StorageProvider createKeyValueStorageProvider(final Path dataDir, final
DEFAULT_MAX_OPEN_FILES,
DEFAULT_MAX_BACKGROUND_COMPACTIONS,
DEFAULT_BACKGROUND_THREAD_COUNT,
DEFAULT_CACHE_CAPACITY),
DEFAULT_CACHE_CAPACITY,
DEFAULT_IS_HIGH_SPEC),
Arrays.asList(KeyValueSegmentIdentifier.values()),
RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS))
.withCommonConfiguration(new BesuConfigurationImpl(dataDir, dbDir))
Expand Down
23 changes: 18 additions & 5 deletions ...org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/RocksDBCLIOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,15 @@ public class RocksDBCLIOptions {
public static final long DEFAULT_CACHE_CAPACITY = 134217728;
public static final int DEFAULT_MAX_BACKGROUND_COMPACTIONS = 4;
public static final int DEFAULT_BACKGROUND_THREAD_COUNT = 4;
public static final boolean DEFAULT_IS_HIGH_SPEC = false;

private static final String MAX_OPEN_FILES_FLAG = "--Xplugin-rocksdb-max-open-files";
private static final String CACHE_CAPACITY_FLAG = "--Xplugin-rocksdb-cache-capacity";
private static final String MAX_BACKGROUND_COMPACTIONS_FLAG =
public static final String MAX_OPEN_FILES_FLAG = "--Xplugin-rocksdb-max-open-files";
public static final String CACHE_CAPACITY_FLAG = "--Xplugin-rocksdb-cache-capacity";
public static final String MAX_BACKGROUND_COMPACTIONS_FLAG =
"--Xplugin-rocksdb-max-background-compactions";
private static final String BACKGROUND_THREAD_COUNT_FLAG =
public static final String BACKGROUND_THREAD_COUNT_FLAG =
"--Xplugin-rocksdb-background-thread-count";
public static final String IS_HIGH_SPEC = "--Xplugin-rocksdb-high-spec-enabled";

@CommandLine.Option(
names = {MAX_OPEN_FILES_FLAG},
Expand Down Expand Up @@ -63,6 +65,15 @@ public class RocksDBCLIOptions {
description = "Number of RocksDB background threads (default: ${DEFAULT-VALUE})")
int backgroundThreadCount;

@CommandLine.Option(
names = {IS_HIGH_SPEC},
hidden = true,
paramLabel = "<BOOLEAN>",
description =
"Use this flag to boost Besu performance if you have a 16 GiB RAM hardware or more (default: ${DEFAULT-VALUE})",
arity = "0")
boolean isHighSpec;

private RocksDBCLIOptions() {}

public static RocksDBCLIOptions create() {
Expand All @@ -75,12 +86,13 @@ public static RocksDBCLIOptions fromConfig(final RocksDBConfiguration config) {
options.cacheCapacity = config.getCacheCapacity();
options.maxBackgroundCompactions = config.getMaxBackgroundCompactions();
options.backgroundThreadCount = config.getBackgroundThreadCount();
options.isHighSpec = config.isHighSpec();
return options;
}

public RocksDBFactoryConfiguration toDomainObject() {
return new RocksDBFactoryConfiguration(
maxOpenFiles, maxBackgroundCompactions, backgroundThreadCount, cacheCapacity);
maxOpenFiles, maxBackgroundCompactions, backgroundThreadCount, cacheCapacity, isHighSpec);
}

@Override
Expand All @@ -90,6 +102,7 @@ public String toString() {
.add("cacheCapacity", cacheCapacity)
.add("maxBackgroundCompactions", maxBackgroundCompactions)
.add("backgroundThreadCount", backgroundThreadCount)
.add("isHighSpec", isHighSpec)
.toString();
}
}
9 changes: 8 additions & 1 deletion .../hyperledger/besu/plugin/services/storage/rocksdb/configuration/RocksDBConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,23 @@ public class RocksDBConfiguration {
private final int maxBackgroundCompactions;
private final int backgroundThreadCount;
private final long cacheCapacity;
private final boolean isHighSpec;

public RocksDBConfiguration(
final Path databaseDir,
final int maxOpenFiles,
final int maxBackgroundCompactions,
final int backgroundThreadCount,
final long cacheCapacity,
final String label) {
final String label,
final boolean isHighSpec) {
this.maxBackgroundCompactions = maxBackgroundCompactions;
this.backgroundThreadCount = backgroundThreadCount;
this.databaseDir = databaseDir;
this.maxOpenFiles = maxOpenFiles;
this.cacheCapacity = cacheCapacity;
this.label = label;
this.isHighSpec = isHighSpec;
}

public Path getDatabaseDir() {
Expand All @@ -63,4 +66,8 @@ public long getCacheCapacity() {
public String getLabel() {
return label;
}

public boolean isHighSpec() {
return isHighSpec;
}
}
13 changes: 11 additions & 2 deletions ...edger/besu/plugin/services/storage/rocksdb/configuration/RocksDBConfigurationBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_BACKGROUND_THREAD_COUNT;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_CACHE_CAPACITY;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_IS_HIGH_SPEC;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_BACKGROUND_COMPACTIONS;
import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_OPEN_FILES;

Expand All @@ -29,6 +30,7 @@ public class RocksDBConfigurationBuilder {
private long cacheCapacity = DEFAULT_CACHE_CAPACITY;
private int maxBackgroundCompactions = DEFAULT_MAX_BACKGROUND_COMPACTIONS;
private int backgroundThreadCount = DEFAULT_BACKGROUND_THREAD_COUNT;
private boolean isHighSpec = DEFAULT_IS_HIGH_SPEC;

public RocksDBConfigurationBuilder databaseDir(final Path databaseDir) {
this.databaseDir = databaseDir;
Expand Down Expand Up @@ -60,12 +62,18 @@ public RocksDBConfigurationBuilder backgroundThreadCount(final int backgroundThr
return this;
}

public RocksDBConfigurationBuilder isHighSpec(final boolean isHighSpec) {
this.isHighSpec = isHighSpec;
return this;
}

public static RocksDBConfigurationBuilder from(final RocksDBFactoryConfiguration configuration) {
return new RocksDBConfigurationBuilder()
.backgroundThreadCount(configuration.getBackgroundThreadCount())
.cacheCapacity(configuration.getCacheCapacity())
.maxBackgroundCompactions(configuration.getMaxBackgroundCompactions())
.maxOpenFiles(configuration.getMaxOpenFiles());
.maxOpenFiles(configuration.getMaxOpenFiles())
.isHighSpec(configuration.isHighSpec());
}

public RocksDBConfiguration build() {
Expand All @@ -75,6 +83,7 @@ public RocksDBConfiguration build() {
maxBackgroundCompactions,
backgroundThreadCount,
cacheCapacity,
label);
label,
isHighSpec);
}
}
9 changes: 8 additions & 1 deletion ...edger/besu/plugin/services/storage/rocksdb/configuration/RocksDBFactoryConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,19 @@ public class RocksDBFactoryConfiguration {
private final int maxBackgroundCompactions;
private final int backgroundThreadCount;
private final long cacheCapacity;
private final boolean isHighSpec;

public RocksDBFactoryConfiguration(
final int maxOpenFiles,
final int maxBackgroundCompactions,
final int backgroundThreadCount,
final long cacheCapacity) {
final long cacheCapacity,
final boolean isHighSpec) {
this.maxBackgroundCompactions = maxBackgroundCompactions;
this.backgroundThreadCount = backgroundThreadCount;
this.maxOpenFiles = maxOpenFiles;
this.cacheCapacity = cacheCapacity;
this.isHighSpec = isHighSpec;
}

public int getMaxOpenFiles() {
Expand All @@ -47,4 +50,8 @@ public int getBackgroundThreadCount() {
public long getCacheCapacity() {
return cacheCapacity;
}

public boolean isHighSpec() {
return isHighSpec;
}
}
51 changes: 42 additions & 9 deletions ...ledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ public class RocksDBColumnarKeyValueStorage
private static final String NO_SPACE_LEFT_ON_DEVICE = "No space left on device";
private static final int ROCKSDB_FORMAT_VERSION = 5;
private static final long ROCKSDB_BLOCK_SIZE = 32768;
private static final long ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC = 1_073_741_824L;
private static final long ROCKSDB_MEMTABLE_SIZE_HIGH_SPEC = 1_073_741_824L;

static {
RocksDbUtil.loadNativeLibrary();
Expand Down Expand Up @@ -111,15 +113,30 @@ public RocksDBColumnarKeyValueStorage(
.setTableFormatConfig(createBlockBasedTableConfig(configuration))));

final Statistics stats = new Statistics();
options =
new DBOptions()
.setCreateIfMissing(true)
.setMaxOpenFiles(configuration.getMaxOpenFiles())
.setMaxBackgroundCompactions(configuration.getMaxBackgroundCompactions())
.setStatistics(stats)
.setCreateMissingColumnFamilies(true)
.setEnv(
Env.getDefault().setBackgroundThreads(configuration.getBackgroundThreadCount()));
if (configuration.isHighSpec()) {
options =
new DBOptions()
.setCreateIfMissing(true)
.setMaxOpenFiles(configuration.getMaxOpenFiles())
.setDbWriteBufferSize(ROCKSDB_MEMTABLE_SIZE_HIGH_SPEC)
.setMaxBackgroundCompactions(configuration.getMaxBackgroundCompactions())
.setStatistics(stats)
.setCreateMissingColumnFamilies(true)
.setEnv(
Env.getDefault()
.setBackgroundThreads(configuration.getBackgroundThreadCount()));
} else {
options =
new DBOptions()
.setCreateIfMissing(true)
.setMaxOpenFiles(configuration.getMaxOpenFiles())
.setMaxBackgroundCompactions(configuration.getMaxBackgroundCompactions())
.setStatistics(stats)
.setCreateMissingColumnFamilies(true)
.setEnv(
Env.getDefault()
.setBackgroundThreads(configuration.getBackgroundThreadCount()));
}

txOptions = new TransactionDBOptions();
final List<ColumnFamilyHandle> columnHandles = new ArrayList<>(columnDescriptors.size());
Expand Down Expand Up @@ -153,6 +170,22 @@ public RocksDBColumnarKeyValueStorage(
}

private BlockBasedTableConfig createBlockBasedTableConfig(final RocksDBConfiguration config) {
if (config.isHighSpec()) return createBlockBasedTableConfigHighSpec();
else return createBlockBasedTableConfigDefault(config);
}

private BlockBasedTableConfig createBlockBasedTableConfigHighSpec() {
final LRUCache cache = new LRUCache(ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC);
return new BlockBasedTableConfig()
.setBlockCache(cache)
.setFormatVersion(ROCKSDB_FORMAT_VERSION)
.setOptimizeFiltersForMemory(true)
.setCacheIndexAndFilterBlocks(true)
.setBlockSize(ROCKSDB_BLOCK_SIZE);
}

private BlockBasedTableConfig createBlockBasedTableConfigDefault(
final RocksDBConfiguration config) {
final LRUCache cache = new LRUCache(config.getCacheCapacity());
return new BlockBasedTableConfig()
.setBlockCache(cache)
Expand Down
Loading

0 comments on commit 27ee445

Please sign in to comment.