From e3db18e99f0a4e446cdc19a2ae5c4eccb7606be4 Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Tue, 5 Dec 2023 10:28:00 +1000 Subject: [PATCH] Add RockDB Subcommand for printing usage per column family (#6185) * Add RockDB Subcommand for printing usage per column family Signed-off-by: Simon Dudley * changed output to follow a MD table notation. Signed-off-by: Gabriel Fukushima --------- Signed-off-by: Gabriel Fukushima Co-authored-by: Gabriel Fukushima --- CHANGELOG.md | 1 + besu/build.gradle | 1 + .../storage/RocksDbSubCommand.java | 115 ++++++++++++++++++ .../storage/RocksDbUsageHelper.java | 105 ++++++++++++++++ .../storage/StorageSubCommand.java | 4 +- .../hyperledger/besu/crypto/KeyPairUtil.java | 1 + 6 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbSubCommand.java create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f0a00eda376..7b2607eed4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - Fix the unavailability of `address` field when returning an `Account` entity on GraphQL in case of unreachable world state [#6198](https://github.com/hyperledger/besu/pull/6198) - Update OpenJ9 Docker image to latest version [#6226](https://github.com/hyperledger/besu/pull/6226) - Add error messages on authentication failures with username and password [#6212](https://github.com/hyperledger/besu/pull/6212) +- Add `rocksdb usage` to the `storage` subcommand to allow users and dev to check columns families usage [#6185](https://github.com/hyperledger/besu/pull/6185) ### Bug fixes - Fix Docker image name clash between Besu and evmtool [#6194](https://github.com/hyperledger/besu/pull/6194) diff --git a/besu/build.gradle b/besu/build.gradle index 9d72f50b021..fe5bb45c30d 100644 --- a/besu/build.gradle +++ b/besu/build.gradle @@ -77,6 +77,7 @@ dependencies { implementation 'org.springframework.security:spring-security-crypto' implementation 'org.xerial.snappy:snappy-java' implementation 'tech.pegasys:jc-kzg-4844' + implementation 'org.rocksdb:rocksdbjni' runtimeOnly 'org.apache.logging.log4j:log4j-jul' runtimeOnly 'com.splunk.logging:splunk-library-javalogging' diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbSubCommand.java new file mode 100644 index 00000000000..461175a2282 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbSubCommand.java @@ -0,0 +1,115 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.subcommands.storage; + +import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; + +import org.hyperledger.besu.cli.util.VersionProvider; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.ParentCommand; + +/** The RocksDB subcommand. */ +@Command( + name = "rocksdb", + description = "Print RocksDB information", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class, + subcommands = {RocksDbSubCommand.RocksDbUsage.class}) +public class RocksDbSubCommand implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private StorageSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public void run() { + spec.commandLine().usage(System.out); + } + + @Command( + name = "usage", + description = "Print disk usage", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class RocksDbUsage implements Runnable { + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @SuppressWarnings("unused") + @ParentCommand + private RocksDbSubCommand parentCommand; + + @Override + public void run() { + + final PrintWriter out = spec.commandLine().getOut(); + + final String dbPath = + parentCommand + .parentCommand + .parentCommand + .dataDir() + .toString() + .concat("/") + .concat(DATABASE_PATH); + + RocksDB.loadLibrary(); + Options options = new Options(); + options.setCreateIfMissing(true); + + // Open the RocksDB database with multiple column families + List cfNames; + try { + cfNames = RocksDB.listColumnFamilies(options, dbPath); + } catch (RocksDBException e) { + throw new RuntimeException(e); + } + final List cfHandles = new ArrayList<>(); + final List cfDescriptors = new ArrayList<>(); + for (byte[] cfName : cfNames) { + cfDescriptors.add(new ColumnFamilyDescriptor(cfName)); + } + RocksDbUsageHelper.printTableHeader(out); + try (final RocksDB rocksdb = RocksDB.openReadOnly(dbPath, cfDescriptors, cfHandles)) { + for (ColumnFamilyHandle cfHandle : cfHandles) { + RocksDbUsageHelper.printUsageForColumnFamily(rocksdb, cfHandle, out); + } + } catch (RocksDBException e) { + throw new RuntimeException(e); + } finally { + for (ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java new file mode 100644 index 00000000000..4bfd9f42951 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java @@ -0,0 +1,105 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.cli.subcommands.storage; + +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; + +import java.io.PrintWriter; + +import org.bouncycastle.util.Arrays; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** RocksDB Usage subcommand helper methods for formatting and printing. */ +public class RocksDbUsageHelper { + private static final Logger LOG = LoggerFactory.getLogger(RocksDbUsageHelper.class); + + static void printUsageForColumnFamily( + final RocksDB rocksdb, final ColumnFamilyHandle cfHandle, final PrintWriter out) + throws RocksDBException, NumberFormatException { + final String size = rocksdb.getProperty(cfHandle, "rocksdb.estimate-live-data-size"); + boolean emptyColumnFamily = false; + if (!size.isEmpty() && !size.isBlank()) { + try { + final long sizeLong = Long.parseLong(size); + final String totalSstFilesSize = + rocksdb.getProperty(cfHandle, "rocksdb.total-sst-files-size"); + final long totalSstFilesSizeLong = + !totalSstFilesSize.isEmpty() && !totalSstFilesSize.isBlank() + ? Long.parseLong(totalSstFilesSize) + : 0; + if (sizeLong == 0) { + emptyColumnFamily = true; + } + + if (!emptyColumnFamily) { + printLine( + out, + getNameById(cfHandle.getName()), + rocksdb.getProperty(cfHandle, "rocksdb.estimate-num-keys"), + formatOutputSize(sizeLong), + formatOutputSize(totalSstFilesSizeLong)); + } + } catch (NumberFormatException e) { + LOG.error("Failed to parse string into long: " + e.getMessage()); + } + } + } + + private static String formatOutputSize(final long size) { + if (size > (1024 * 1024 * 1024)) { + long sizeInGiB = size / (1024 * 1024 * 1024); + return sizeInGiB + " GiB"; + } else if (size > (1024 * 1024)) { + long sizeInMiB = size / (1024 * 1024); + return sizeInMiB + " MiB"; + } else if (size > 1024) { + long sizeInKiB = size / 1024; + return sizeInKiB + " KiB"; + } else { + return size + " B"; + } + } + + private static String getNameById(final byte[] id) { + for (KeyValueSegmentIdentifier segment : KeyValueSegmentIdentifier.values()) { + if (Arrays.areEqual(segment.getId(), id)) { + return segment.getName(); + } + } + return null; // id not found + } + + static void printTableHeader(final PrintWriter out) { + out.format( + "| Column Family | Keys | Column Size | SST Files Size |\n"); + out.format( + "|--------------------------------|-----------------|--------------|-----------------|\n"); + } + + static void printLine( + final PrintWriter out, + final String cfName, + final String keys, + final String columnSize, + final String sstFilesSize) { + final String format = "| %-30s | %-15s | %-12s | %-15s |\n"; + out.format(format, cfName, keys, columnSize, sstFilesSize); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java index bd40a42a431..e46ffde2739 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java @@ -45,7 +45,7 @@ description = "This command provides storage related actions.", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, - subcommands = {StorageSubCommand.RevertVariablesStorage.class}) + subcommands = {StorageSubCommand.RevertVariablesStorage.class, RocksDbSubCommand.class}) public class StorageSubCommand implements Runnable { /** The constant COMMAND_NAME. */ @@ -53,7 +53,7 @@ public class StorageSubCommand implements Runnable { @SuppressWarnings("unused") @ParentCommand - private BesuCommand parentCommand; + BesuCommand parentCommand; @SuppressWarnings("unused") @Spec diff --git a/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/KeyPairUtil.java b/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/KeyPairUtil.java index f20b1b37103..013b385c3c2 100644 --- a/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/KeyPairUtil.java +++ b/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/KeyPairUtil.java @@ -84,6 +84,7 @@ public static KeyPair loadKeyPair(final File keyFile) { final KeyPair key; if (keyFile.exists()) { + LOG.info("Attempting to load public key from {}", keyFile.getAbsolutePath()); key = load(keyFile); LOG.info( "Loaded public key {} from {}", key.getPublicKey().toString(), keyFile.getAbsolutePath());