From 8d7cd69f1c19643307bb092d7678612328e55789 Mon Sep 17 00:00:00 2001 From: Vigya Sharma Date: Wed, 30 Sep 2020 23:24:50 -0700 Subject: [PATCH] Decision policy for consolidating actions --- .../deciders/DecisionPolicy.java | 32 +++++++++++++++ .../deciders/jvm/HeapHealthDecider.java | 34 ++++------------ .../jvm/old_gen/OldGenDecisionPolicy.java | 39 ++++++++++++++++++- 3 files changed, 76 insertions(+), 29 deletions(-) create mode 100644 src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/DecisionPolicy.java diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/DecisionPolicy.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/DecisionPolicy.java new file mode 100644 index 000000000..a7b16bb88 --- /dev/null +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/DecisionPolicy.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.Action; +import java.util.List; + +/** + * A DecisionPolicy evaluates a subset of observation summaries for + * a Decider, and returns a list of recommended Actions. They abstract out a subset + * of the decision making process for a decider. + * + *

Decision policies are invoked by deciders and never scheduled directly by the RCA framework. + */ +public interface DecisionPolicy { + + List evaluate(); + +} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/HeapHealthDecider.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/HeapHealthDecider.java index c124cccf6..f53ed6941 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/HeapHealthDecider.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/HeapHealthDecider.java @@ -20,14 +20,8 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.Decider; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.Decision; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.jvm.old_gen.OldGenDecisionPolicy; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.flow_units.ResourceFlowUnit; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotClusterSummary; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotNodeSummary; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotResourceSummary; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.ResourceUtil; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.HighHeapUsageClusterRca; -import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.NodeKey; import java.util.List; /** @@ -36,15 +30,13 @@ public class HeapHealthDecider extends Decider { public static final String NAME = "HeapHealthDecider"; - private final HighHeapUsageClusterRca highHeapUsageClusterRca; private final OldGenDecisionPolicy oldGenDecisionPolicy; private int counter = 0; public HeapHealthDecider(int decisionFrequency, final HighHeapUsageClusterRca highHeapUsageClusterRca) { //TODO : refactor parent class to remove evalIntervalSeconds completely super(5, decisionFrequency); - this.highHeapUsageClusterRca = highHeapUsageClusterRca; - oldGenDecisionPolicy = new OldGenDecisionPolicy(); + oldGenDecisionPolicy = new OldGenDecisionPolicy(highHeapUsageClusterRca); } @Override @@ -61,25 +53,13 @@ public Decision operate() { } counter = 0; - if (highHeapUsageClusterRca.getFlowUnits().isEmpty()) { - return decision; - } + // oldGenDecisionPolicy are always accepted + List oldGenPolicyActions = oldGenDecisionPolicy.evaluate(); + oldGenPolicyActions.forEach(decision::addAction); + + // TODO: Add actions from JvmScaleUpPolicy (128gb heaps) + // TODO: If no JvmScaleUpPolicy actions found, fetch and add genTuningPolicy actions - ResourceFlowUnit flowUnit = highHeapUsageClusterRca.getFlowUnits().get(0); - if (!flowUnit.hasResourceSummary()) { - return decision; - } - HotClusterSummary clusterSummary = flowUnit.getSummary(); - for (HotNodeSummary nodeSummary : clusterSummary.getHotNodeSummaryList()) { - NodeKey esNode = new NodeKey(nodeSummary.getNodeID(), nodeSummary.getHostAddress()); - for (HotResourceSummary resource : nodeSummary.getHotResourceSummaryList()) { - if (resource.getResource().equals(ResourceUtil.OLD_GEN_HEAP_USAGE)) { - List actions = oldGenDecisionPolicy.evaluate(esNode, resource.getValue()); - actions.forEach(decision::addAction); - } - //TODO : Add policy for young gen - } - } return decision; } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/old_gen/OldGenDecisionPolicy.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/old_gen/OldGenDecisionPolicy.java index 11993a6d7..836893bf3 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/old_gen/OldGenDecisionPolicy.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/decisionmaker/deciders/jvm/old_gen/OldGenDecisionPolicy.java @@ -17,8 +17,15 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.AppContext; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.actions.Action; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.DecisionPolicy; import com.amazon.opendistro.elasticsearch.performanceanalyzer.decisionmaker.deciders.configs.jvm.OldGenDecisionPolicyConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.flow_units.ResourceFlowUnit; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotClusterSummary; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotNodeSummary; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotResourceSummary; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.ResourceUtil; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.HighHeapUsageClusterRca; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.cluster.NodeKey; import java.util.ArrayList; import java.util.List; @@ -32,10 +39,15 @@ * and create dedicated action builders {@link LevelOneActionBuilder}, {@link LevelTwoActionBuilder}, * {@link LevelThreeActionBuilder} for each level of unhealthiness */ -public class OldGenDecisionPolicy { +public class OldGenDecisionPolicy implements DecisionPolicy { private static final Logger LOG = LogManager.getLogger(OldGenDecisionPolicy.class); private AppContext appContext; private RcaConf rcaConf; + private final HighHeapUsageClusterRca highHeapUsageClusterRca; + + public OldGenDecisionPolicy(final HighHeapUsageClusterRca highHeapUsageClusterRca) { + this.highHeapUsageClusterRca = highHeapUsageClusterRca; + } public void setRcaConf(final RcaConf rcaConf) { this.rcaConf = rcaConf; @@ -45,7 +57,30 @@ public void setAppContext(final AppContext appContext) { this.appContext = appContext; } - public List evaluate(final NodeKey esNode, double oldGenUsage) { + @Override + public List evaluate() { + List actions = new ArrayList<>(); + if (highHeapUsageClusterRca.getFlowUnits().isEmpty()) { + return actions; + } + + ResourceFlowUnit flowUnit = highHeapUsageClusterRca.getFlowUnits().get(0); + if (!flowUnit.hasResourceSummary()) { + return actions; + } + HotClusterSummary clusterSummary = flowUnit.getSummary(); + for (HotNodeSummary nodeSummary : clusterSummary.getHotNodeSummaryList()) { + NodeKey esNode = new NodeKey(nodeSummary.getNodeID(), nodeSummary.getHostAddress()); + for (HotResourceSummary resource : nodeSummary.getHotResourceSummaryList()) { + if (resource.getResource().equals(ResourceUtil.OLD_GEN_HEAP_USAGE)) { + actions.addAll(evaluate(esNode, resource.getValue())); + } + } + } + return actions; + } + + private List evaluate(final NodeKey esNode, double oldGenUsage) { //rca config / app context will not be null unless there is a bug in RCAScheduler. if (rcaConf == null || appContext == null) { LOG.error("rca conf/app context is null, return empty action list");