Skip to content

Commit

Permalink
Support degrade by exception count (#174)
Browse files Browse the repository at this point in the history
* Add a new DegradeRule type, degrade by exception count in the last 60 seconds
* Add demo about degrading by exception count
  • Loading branch information
CarpenterLee authored Oct 11, 2018
1 parent 570c3ee commit 9ae079c
Show file tree
Hide file tree
Showing 6 changed files with 228 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
*/
package com.alibaba.csp.sentinel.slots.block;

import com.alibaba.csp.sentinel.node.IntervalProperty;

/***
* @author youji.zj
* @author jialiang.linjl
Expand All @@ -25,7 +27,14 @@ public final class RuleConstant {
public static final int FLOW_GRADE_QPS = 1;

public static final int DEGRADE_GRADE_RT = 0;
public static final int DEGRADE_GRADE_EXCEPTION = 1;
/**
* Degrade by biz exception ratio in the current {@link IntervalProperty#INTERVAL} second(s).
*/
public static final int DEGRADE_GRADE_EXCEPTION_RATIO = 1;
/**
* Degrade by biz exception count in the last 60 seconds.
*/
public static final int DEGRADE_GRADE_EXCEPTION_COUNT = 2;

public static final int AUTHORITY_WHITE = 0;
public static final int AUTHORITY_BLACK = 1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ public boolean passCheck(Context context, DefaultNode node, int acquireCount, Ob
if (passCount.incrementAndGet() < RT_MAX_EXCEED_N) {
return true;
}
} else {
} else if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) {
double exception = clusterNode.exceptionQps();
double success = clusterNode.successQps();
long total = clusterNode.totalQps();
Expand All @@ -190,6 +190,11 @@ public boolean passCheck(Context context, DefaultNode node, int acquireCount, Ob
if (exception / success < count) {
return true;
}
} else if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT) {
double exception = clusterNode.totalException();
if (exception < count) {
return true;
}
}

synchronized (lock) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public void testExceptionRatioModeDegrade() throws Throwable {
rule.setCount(0.15);
rule.setResource(key);
rule.setTimeWindow(5);
rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION);
rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO);

when(cn.successQps()).thenReturn(8L);

Expand All @@ -97,4 +97,34 @@ public void testExceptionRatioModeDegrade() throws Throwable {
assertTrue(rule.passCheck(context, node, 1));
}

@Test
public void testExceptionCountModeDegrade() throws Throwable {
String key = "test_degrade_exception_count";
ClusterNode cn = mock(ClusterNode.class);
when(cn.totalException()).thenReturn(10L);
ClusterBuilderSlot.getClusterNodeMap().put(new StringResourceWrapper(key, EntryType.IN), cn);

Context context = mock(Context.class);
DefaultNode node = mock(DefaultNode.class);
when(node.getClusterNode()).thenReturn(cn);

DegradeRule rule = new DegradeRule();
rule.setCount(4);
rule.setResource(key);
rule.setTimeWindow(2);
rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT);

when(cn.totalException()).thenReturn(4L);

// Will fail.
assertFalse(rule.passCheck(context, node, 1));

// Restore from the degrade timeout.
TimeUnit.SECONDS.sleep(3);

when(cn.totalException()).thenReturn(0L);
// Will pass.
assertTrue(rule.passCheck(context, node, 1));
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
package com.alibaba.csp.sentinel.demo.degrade;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

import com.alibaba.csp.sentinel.Entry;
import com.alibaba.csp.sentinel.SphU;
import com.alibaba.csp.sentinel.Tracer;
import com.alibaba.csp.sentinel.slots.block.BlockException;
import com.alibaba.csp.sentinel.slots.block.RuleConstant;
import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule;
import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager;
import com.alibaba.csp.sentinel.util.TimeUtil;

/**
* <p>
* Degrade is used when the resources are in an unstable state, these resources
* will be degraded within the next defined time window. There are three ways to
* measure whether a resource is stable or not:
* <ul>
* <li>
* Exception count: When the exception count in the last 60 seconds greats than
* or equals to the threshold, access to the resource will be blocked in the
* coming time window.
* </li>
* <li>
* Exception ratio, see {@link ExceptionRatioDegradeDemo}.
* </li>
* <li>
* For average response time, see {@link RtDegradeDemo}.
* </li>
* </ul>
* </p>
* <p>
* Note: When degrading by {@link RuleConstant#DEGRADE_GRADE_EXCEPTION_COUNT}, time window
* less than 60 seconds will not work as expected. Because the exception count is
* summed by minute, when a short time window elapsed, the degradation condition
* may still be satisfied.
* </p>
*
* @author Carpenter Lee
*/
public class ExceptionCountDegradeDemo {
private static final String KEY = "abc";

private static AtomicInteger total = new AtomicInteger();
private static AtomicInteger pass = new AtomicInteger();
private static AtomicInteger block = new AtomicInteger();
private static AtomicInteger bizException = new AtomicInteger();

private static volatile boolean stop = false;
private static final int threadCount = 1;
private static int seconds = 60 + 40;

public static void main(String[] args) throws Exception {
tick();
initDegradeRule();

for (int i = 0; i < threadCount; i++) {
Thread entryThread = new Thread(new Runnable() {

@Override
public void run() {
int count = 0;
while (true) {
count++;
Entry entry = null;
try {
Thread.sleep(20);
entry = SphU.entry(KEY);
// token acquired, means pass
pass.addAndGet(1);
if (count % 2 == 0) {
// biz code raise an exception.
throw new RuntimeException("throw runtime ");
}
} catch (BlockException e) {
block.addAndGet(1);
} catch (Throwable t) {
bizException.incrementAndGet();
Tracer.trace(t);
} finally {
total.addAndGet(1);
if (entry != null) {
entry.exit();
}
}
}
}

});
entryThread.setName("working-thread");
entryThread.start();
}

}

private static void initDegradeRule() {
List<DegradeRule> rules = new ArrayList<DegradeRule>();
DegradeRule rule = new DegradeRule();
rule.setResource(KEY);
// set limit exception count to 4
rule.setCount(4);
rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT);
/**
* When degrading by {@link RuleConstant#DEGRADE_GRADE_EXCEPTION_COUNT}, time window
* less than 60 seconds will not work as expected. Because the exception count is
* summed by minute, when a short time window elapsed, the degradation condition
* may still be satisfied.
*/
rule.setTimeWindow(10);
rules.add(rule);
DegradeRuleManager.loadRules(rules);
}

private static void tick() {
Thread timer = new Thread(new TimerTask());
timer.setName("sentinel-timer-task");
timer.start();
}

static class TimerTask implements Runnable {
@Override
public void run() {
long start = System.currentTimeMillis();
System.out.println("begin to statistic!!!");
long oldTotal = 0;
long oldPass = 0;
long oldBlock = 0;
long oldBizException = 0;
while (!stop) {
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
}
long globalTotal = total.get();
long oneSecondTotal = globalTotal - oldTotal;
oldTotal = globalTotal;

long globalPass = pass.get();
long oneSecondPass = globalPass - oldPass;
oldPass = globalPass;

long globalBlock = block.get();
long oneSecondBlock = globalBlock - oldBlock;
oldBlock = globalBlock;

long globalBizException = bizException.get();
long oneSecondBizException = globalBizException - oldBizException;
oldBizException = globalBizException;

System.out.println(TimeUtil.currentTimeMillis() + ", oneSecondTotal:" + oneSecondTotal
+ ", oneSecondPass:" + oneSecondPass
+ ", oneSecondBlock:" + oneSecondBlock
+ ", oneSecondBizException:" + oneSecondBizException);
if (seconds-- <= 0) {
stop = true;
}
}
long cost = System.currentTimeMillis() - start;
System.out.println("time cost: " + cost + " ms");
System.out.println("total:" + total.get() + ", pass:" + pass.get()
+ ", block:" + block.get() + ", bizException:" + bizException.get());
System.exit(0);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,16 @@
/**
* <p>
* Degrade is used when the resources are in an unstable state, these resources
* will be degraded within the next defined time window. There are two ways to
* will be degraded within the next defined time window. There are three ways to
* measure whether a resource is stable or not:
* <ul>
* <li>
* Exception ratio: When the ratio of exception count per second and the success
* qps exceeds the threshold , access to the resource will be blocked in the
* coming window.
* qps greats than or equals to the threshold, access to the resource will be blocked
* in the coming time window.
* </li>
* <li>
* Exception Count, see {@link ExceptionCountDegradeDemo}.
* </li>
* <li>
* For average response time, see {@link RtDegradeDemo}.
Expand Down Expand Up @@ -110,7 +113,7 @@ private static void initDegradeRule() {
rule.setResource(KEY);
// set limit exception ratio to 0.1
rule.setCount(0.1);
rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION);
rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO);
rule.setTimeWindow(10);
rules.add(rule);
DegradeRuleManager.loadRules(rules);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,18 @@
* <ul>
* <li>
* Average Response Time ('DegradeRule.Grade=RuleContants.DEGRADE_GRADE_RT'): When the
* average RT exceeds the threshold ('count' in 'DegradeRule', ms), the resource
* enters a quasi-degraded state. If the RT of next coming five requests still
* average RT greats than or equals to the threshold ('count' in 'DegradeRule', ms), the
* resource enters a quasi-degraded state. If the RT of next coming five requests still
* exceed this threshold, this resource will be downgraded, which means that in
* the next time window(Defined in 'timeWindow', s units) all the access to this
* resource will be blocked.
* </li>
* <li>
* Exception ratio, see {@link ExceptionRatioDegradeDemo}.
* </li>
* <li>
* Exception Count, see {@link ExceptionCountDegradeDemo}.
* </li>
* </ul>
*
* </p>
Expand Down

0 comments on commit 9ae079c

Please sign in to comment.