Skip to content

Commit

Permalink
HBASE-26860 Backport "HBASE-25681 Add a switch for server/table query…
Browse files Browse the repository at this point in the history 
…Meter" to branch-2.4 (#4240)

Signed-off-by: Andrew Purtell <[email protected]>
  • Loading branch information
@bbeaudreault
bbeaudreault committed Apr 30, 2022
1 parent 485d6f8 commit 6d3b583
Show file tree
Hide file tree
Showing 8 changed files with 139 additions and 24 deletions.
3 changes: 3 additions & 0 deletions ...oop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
@InterfaceAudience.Private
public interface MetricsTableQueryMeter {

String TABLE_READ_QUERY_PER_SECOND = "tableReadQueryPerSecond";
String TABLE_WRITE_QUERY_PER_SECOND = "tableWriteQueryPerSecond";

/**
* Update table read QPS
* @param tableName The table the metric is for
Expand Down
10 changes: 10 additions & 0 deletions hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,16 @@ public interface MetricsAssertHelper {
*/
boolean checkCounterExists(String name, BaseSource source);

/**
* Check if a gauge exists.
*
* @param name name of the gauge.
* @param source The BaseSource{@link BaseSource} that will provide the tags,
* gauges, and counters.
* @return boolean true if gauge metric exists.
*/
boolean checkGaugeExists(String name, BaseSource source);

/**
* Get the value of a gauge as a double.
*
Expand Down
3 changes: 0 additions & 3 deletions ...compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeterImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,6 @@ public class MetricsTableQueryMeterImpl implements MetricsTableQueryMeter {
private final Map<TableName, TableMeters> metersByTable = new ConcurrentHashMap<>();
private final MetricRegistry metricRegistry;

private final static String TABLE_READ_QUERY_PER_SECOND = "tableReadQueryPerSecond";
private final static String TABLE_WRITE_QUERY_PER_SECOND = "tableWriteQueryPerSecond";

public MetricsTableQueryMeterImpl(MetricRegistry metricRegistry) {
this.metricRegistry = metricRegistry;
}
Expand Down
11 changes: 9 additions & 2 deletions hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,16 @@ public long getCounter(String name, BaseSource source) {
public boolean checkCounterExists(String name, BaseSource source) {
getMetrics(source);
String cName = canonicalizeMetricName(name);
return (counters.get(cName) != null) ? true : false;
return counters.get(cName) != null;
}


@Override
public boolean checkGaugeExists(String name, BaseSource source) {
getMetrics(source);
String cName = canonicalizeMetricName(name);
return gauges.get(cName) != null;
}

@Override
public double getGaugeDouble(String name, BaseSource source) {
getMetrics(source);
Expand Down
29 changes: 23 additions & 6 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ public class MetricsRegionServer {
public static final String RS_ENABLE_TABLE_METRICS_KEY =
"hbase.regionserver.enable.table.latencies";
public static final boolean RS_ENABLE_TABLE_METRICS_DEFAULT = true;
public static final String RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY =
"hbase.regionserver.enable.server.query.meter";
public static final boolean RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY_DEFAULT = true;
public static final String RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY =
"hbase.regionserver.enable.table.query.meter";
public static final boolean RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY_DEFAULT = true;

public static final String SLOW_METRIC_TIME = "hbase.ipc.slow.metric.time";
private final MetricsRegionServerSource serverSource;
Expand Down Expand Up @@ -73,9 +79,12 @@ public MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper, Confi
bulkLoadTimer = metricRegistry.timer("Bulkload");

slowMetricTime = conf.getLong(SLOW_METRIC_TIME, DEFAULT_SLOW_METRIC_TIME);
serverReadQueryMeter = metricRegistry.meter("ServerReadQueryPerSecond");
serverWriteQueryMeter = metricRegistry.meter("ServerWriteQueryPerSecond");
quotaSource = CompatibilitySingletonFactory.getInstance(MetricsRegionServerQuotaSource.class);
if (conf.getBoolean(RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY,
RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY_DEFAULT)) {
serverReadQueryMeter = metricRegistry.meter("ServerReadQueryPerSecond");
serverWriteQueryMeter = metricRegistry.meter("ServerWriteQueryPerSecond");
}
}

MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper,
Expand All @@ -93,7 +102,9 @@ public MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper, Confi
*/
static RegionServerTableMetrics createTableMetrics(Configuration conf) {
if (conf.getBoolean(RS_ENABLE_TABLE_METRICS_KEY, RS_ENABLE_TABLE_METRICS_DEFAULT)) {
return new RegionServerTableMetrics();
return new RegionServerTableMetrics(
conf.getBoolean(RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY,
RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY_DEFAULT));
}
return null;
}
Expand Down Expand Up @@ -270,21 +281,27 @@ public void updateReadQueryMeter(TableName tn, long count) {
if (tableMetrics != null && tn != null) {
tableMetrics.updateTableReadQueryMeter(tn, count);
}
this.serverReadQueryMeter.mark(count);
if (serverReadQueryMeter != null) {
serverReadQueryMeter.mark(count);
}
}

public void updateWriteQueryMeter(TableName tn, long count) {
if (tableMetrics != null && tn != null) {
tableMetrics.updateTableWriteQueryMeter(tn, count);
}
this.serverWriteQueryMeter.mark(count);
if (serverWriteQueryMeter != null) {
serverWriteQueryMeter.mark(count);
}
}

public void updateWriteQueryMeter(TableName tn) {
if (tableMetrics != null && tn != null) {
tableMetrics.updateTableWriteQueryMeter(tn);
}
this.serverWriteQueryMeter.mark();
if (serverWriteQueryMeter != null) {
serverWriteQueryMeter.mark();
}
}

/**
Expand Down
26 changes: 15 additions & 11 deletions ...e-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerTableMetrics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,14 @@
public class RegionServerTableMetrics {

private final MetricsTableLatencies latencies;
private final MetricsTableQueryMeter queryMeter;
private MetricsTableQueryMeter queryMeter;

public RegionServerTableMetrics() {
public RegionServerTableMetrics(boolean enableTableQueryMeter) {
latencies = CompatibilitySingletonFactory.getInstance(MetricsTableLatencies.class);
queryMeter = new MetricsTableQueryMeterImpl(MetricRegistries.global().
get(((MetricsTableLatenciesImpl) latencies).getMetricRegistryInfo()).get());
if (enableTableQueryMeter) {
queryMeter = new MetricsTableQueryMeterImpl(MetricRegistries.global().
get(((MetricsTableLatenciesImpl) latencies).getMetricRegistryInfo()).get());
}
}

public void updatePut(TableName table, long time) {
Expand Down Expand Up @@ -86,18 +88,20 @@ public void updateScanSize(TableName table, long size) {
}

public void updateTableReadQueryMeter(TableName table, long count) {
queryMeter.updateTableReadQueryMeter(table, count);
}

public void updateTableReadQueryMeter(TableName table) {
queryMeter.updateTableReadQueryMeter(table);
if (queryMeter != null) {
queryMeter.updateTableReadQueryMeter(table, count);
}
}

public void updateTableWriteQueryMeter(TableName table, long count) {
queryMeter.updateTableWriteQueryMeter(table, count);
if (queryMeter != null) {
queryMeter.updateTableWriteQueryMeter(table, count);
}
}

public void updateTableWriteQueryMeter(TableName table) {
queryMeter.updateTableWriteQueryMeter(table);
if (queryMeter != null) {
queryMeter.updateTableWriteQueryMeter(table);
}
}
}
28 changes: 27 additions & 1 deletion hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,14 @@
*/
package org.apache.hadoop.hbase.regionserver;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CompatibilityFactory;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
Expand Down Expand Up @@ -57,7 +60,9 @@ public static void classSetUp() {
@Before
public void setUp() {
wrapper = new MetricsRegionServerWrapperStub();
rsm = new MetricsRegionServer(wrapper, new Configuration(false), null);
Configuration conf = new Configuration(false);
conf.setBoolean(MetricsRegionServer.RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY, false);
rsm = new MetricsRegionServer(wrapper, conf, null);
serverSource = rsm.getMetricsSource();
}

Expand Down Expand Up @@ -245,5 +250,26 @@ public void testPauseMonitor() {
HELPER.assertCounter("pauseTimeWithGc_num_ops", 1, serverSource);
}

@Test
public void testServerQueryMeterSwitch() {
TableName tn1 = TableName.valueOf("table1");
// has been set disable in setUp()
rsm.updateReadQueryMeter(tn1, 500L);
assertFalse(HELPER.checkGaugeExists("ServerReadQueryPerSecond_count", serverSource));
rsm.updateWriteQueryMeter(tn1, 500L);
assertFalse(HELPER.checkGaugeExists("ServerWriteQueryPerSecond_count", serverSource));

// enable
Configuration conf = new Configuration(false);
conf.setBoolean(MetricsRegionServer.RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY, true);
rsm = new MetricsRegionServer(wrapper, conf, null);
serverSource = rsm.getMetricsSource();
rsm.updateReadQueryMeter(tn1, 500L);
assertTrue(HELPER.checkGaugeExists("ServerWriteQueryPerSecond_count", serverSource));
HELPER.assertGauge("ServerReadQueryPerSecond_count", 500L, serverSource);
assertTrue(HELPER.checkGaugeExists("ServerWriteQueryPerSecond_count", serverSource));
rsm.updateWriteQueryMeter(tn1, 500L);
HELPER.assertGauge("ServerWriteQueryPerSecond_count", 500L, serverSource);
}
}

53 changes: 52 additions & 1 deletion ...-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableLatencies.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@
*/
package org.apache.hadoop.hbase.regionserver;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CompatibilityFactory;
import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
import org.apache.hadoop.hbase.HBaseClassTestRule;
Expand Down Expand Up @@ -51,7 +53,7 @@ public void testTableWrapperAggregateMetrics() throws IOException {
assertTrue("'latencies' is actually " + latencies.getClass(),
latencies instanceof MetricsTableLatenciesImpl);
MetricsTableLatenciesImpl latenciesImpl = (MetricsTableLatenciesImpl) latencies;
RegionServerTableMetrics tableMetrics = new RegionServerTableMetrics();
RegionServerTableMetrics tableMetrics = new RegionServerTableMetrics(false);

// Metrics to each table should be disjoint
// N.B. each call to assertGauge removes all previously acquired metrics so we have to
Expand All @@ -72,4 +74,53 @@ public void testTableWrapperAggregateMetrics() throws IOException {
HELPER.assertGauge(MetricsTableLatenciesImpl.qualifyMetricsName(
tn2, MetricsTableLatencies.PUT_TIME + "_" + "99th_percentile"), 75L, latenciesImpl);
}

@Test
public void testTableQueryMeterSwitch() {
TableName tn1 = TableName.valueOf("table1");
MetricsTableLatencies latencies = CompatibilitySingletonFactory.getInstance(
MetricsTableLatencies.class);
assertTrue("'latencies' is actually " + latencies.getClass(),
latencies instanceof MetricsTableLatenciesImpl);
MetricsTableLatenciesImpl latenciesImpl = (MetricsTableLatenciesImpl) latencies;

Configuration conf = new Configuration();
conf.setBoolean(MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY, false);
boolean enableTableQueryMeter = conf.getBoolean(
MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY,
MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY_DEFAULT);
// disable
assertFalse(enableTableQueryMeter);
RegionServerTableMetrics tableMetrics = new RegionServerTableMetrics(enableTableQueryMeter);
tableMetrics.updateTableReadQueryMeter(tn1, 500L);
assertFalse(HELPER.checkGaugeExists(MetricsTableLatenciesImpl.qualifyMetricsName(
tn1, MetricsTableQueryMeterImpl.TABLE_READ_QUERY_PER_SECOND + "_" + "count"),
latenciesImpl));
tableMetrics.updateTableWriteQueryMeter(tn1, 500L);
assertFalse(HELPER.checkGaugeExists(MetricsTableLatenciesImpl.qualifyMetricsName(
tn1, MetricsTableQueryMeterImpl.TABLE_WRITE_QUERY_PER_SECOND + "_" + "count"),
latenciesImpl));

// enable
conf.setBoolean(MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY, true);
enableTableQueryMeter = conf.getBoolean(
MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY,
MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY_DEFAULT);
assertTrue(enableTableQueryMeter);
tableMetrics = new RegionServerTableMetrics(true);
tableMetrics.updateTableReadQueryMeter(tn1, 500L);
assertTrue(HELPER.checkGaugeExists(MetricsTableLatenciesImpl.qualifyMetricsName(
tn1, MetricsTableQueryMeterImpl.TABLE_READ_QUERY_PER_SECOND + "_" + "count"),
latenciesImpl));
HELPER.assertGauge(MetricsTableLatenciesImpl.qualifyMetricsName(
tn1, MetricsTableQueryMeterImpl.TABLE_READ_QUERY_PER_SECOND + "_" + "count"),
500L, latenciesImpl);
tableMetrics.updateTableWriteQueryMeter(tn1, 500L);
assertTrue(HELPER.checkGaugeExists(MetricsTableLatenciesImpl.qualifyMetricsName(
tn1, MetricsTableQueryMeterImpl.TABLE_WRITE_QUERY_PER_SECOND + "_" + "count"),
latenciesImpl));
HELPER.assertGauge(MetricsTableLatenciesImpl.qualifyMetricsName(
tn1, MetricsTableQueryMeterImpl.TABLE_WRITE_QUERY_PER_SECOND + "_" + "count"),
500L, latenciesImpl);
}
}

0 comments on commit 6d3b583

Please sign in to comment.