From afd167a465689d390c437c49a54a2f0edb1cdbd1 Mon Sep 17 00:00:00 2001 From: quguai Date: Fri, 27 Oct 2023 11:40:49 +0800 Subject: [PATCH] feat: resource rules (flow/degrade/param/authority) support regex matching Closes #3247 --- .../sentinel/slots/block/AbstractRule.java | 22 ++- .../csp/sentinel/slots/block/RuleManager.java | 186 ++++++++++++++++++ .../block/authority/AuthorityRuleManager.java | 34 ++-- .../slots/block/authority/AuthoritySlot.java | 10 +- .../block/degrade/DegradeRuleManager.java | 62 +++--- .../slots/block/flow/FlowRuleManager.java | 25 +-- .../slots/block/flow/FlowRuleUtil.java | 14 ++ .../sentinel/slots/block/flow/FlowSlot.java | 6 +- .../csp/sentinel/util/function/Function.java | 10 + .../sentinel/slots/block/RuleManagerTest.java | 112 +++++++++++ .../AuthorityPartialIntegrationTest.java | 49 +++++ .../DegradePartialIntegrationTest.java | 73 +++++++ .../flow/FlowPartialIntegrationTest.java | 34 +++- .../flow/param/ParamFlowRuleManager.java | 27 +-- .../block/flow/param/ParamFlowRuleUtil.java | 13 +- .../ParamFlowPartialIntegrationTest.java | 68 +++++++ 16 files changed, 641 insertions(+), 104 deletions(-) create mode 100644 sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleManager.java create mode 100644 sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/RuleManagerTest.java create mode 100644 sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityPartialIntegrationTest.java create mode 100644 sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradePartialIntegrationTest.java create mode 100644 sentinel-extension/sentinel-parameter-flow-control/src/test/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowPartialIntegrationTest.java diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/AbstractRule.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/AbstractRule.java index befec3bc1a..d34cb9996e 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/AbstractRule.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/AbstractRule.java @@ -15,6 +15,8 @@ */ package com.alibaba.csp.sentinel.slots.block; +import java.util.Objects; + /** * Abstract rule entity. * @@ -44,6 +46,11 @@ public abstract class AbstractRule implements Rule { */ private String limitApp; + /** + * Whether to match resource names according to regular rules + */ + private boolean regex; + public Long getId() { return id; } @@ -72,6 +79,15 @@ public AbstractRule setLimitApp(String limitApp) { return this; } + public boolean isRegex() { + return regex; + } + + public AbstractRule setRegex(boolean regex) { + this.regex = regex; + return this; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -83,7 +99,10 @@ public boolean equals(Object o) { AbstractRule that = (AbstractRule)o; - if (resource != null ? !resource.equals(that.resource) : that.resource != null) { + if (!Objects.equals(resource, that.resource)) { + return false; + } + if (regex != that.regex) { return false; } if (!limitAppEquals(limitApp, that.limitApp)) { @@ -114,6 +133,7 @@ public int hashCode() { if (!("".equals(limitApp) || RuleConstant.LIMIT_APP_DEFAULT.equals(limitApp) || limitApp == null)) { result = 31 * result + limitApp.hashCode(); } + result = 31 * result + (regex ? 1 : 0); return result; } } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleManager.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleManager.java new file mode 100644 index 0000000000..a728afc38a --- /dev/null +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleManager.java @@ -0,0 +1,186 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.csp.sentinel.slots.block; + +import com.alibaba.csp.sentinel.util.function.Function; +import com.alibaba.csp.sentinel.util.function.Predicate; + +import java.util.*; +import java.util.regex.Pattern; + +/** + * Unified rule management tool, mainly used for matching and caching of regular rules and simple rules. + * @author quguai + * @date 2023/10/9 20:35 + */ +public class RuleManager { + + private Map> originalRules = new HashMap<>(); + private Map> regexRules = new HashMap<>(); + private Map> regexCacheRules = new HashMap<>(); + private Map> simpleRules = new HashMap<>(); + private Function, List> generator = Function.identity(); + + private final Predicate predicate; + + public RuleManager() { + predicate = r -> r instanceof AbstractRule && ((AbstractRule) r).isRegex(); + } + + public RuleManager(Function, List> generator, Predicate predicate) { + this.generator = generator; + this.predicate = predicate; + } + + /** + * Update rules from datasource, split rules map by regex, + * rebuild the regex rule cache to reduce the performance loss caused by publish rules. + * + * @param rulesMap origin rules map + */ + public void updateRules(Map> rulesMap) { + originalRules = rulesMap; + Map> regexRules = new HashMap<>(); + Map> simpleRules = new HashMap<>(); + for (Map.Entry> entry : rulesMap.entrySet()) { + String resource = entry.getKey(); + List rules = entry.getValue(); + + List rulesOfSimple = new ArrayList<>(); + List rulesOfRegex = new ArrayList<>(); + for (R rule : rules) { + if (predicate.test(rule)) { + rulesOfRegex.add(rule); + } else { + rulesOfSimple.add(rule); + } + } + if (!rulesOfRegex.isEmpty()) { + regexRules.put(Pattern.compile(resource), rulesOfRegex); + } + if (!rulesOfSimple.isEmpty()) { + simpleRules.put(resource, rulesOfSimple); + } + } + // rebuild regex cache rules + setRules(regexRules, simpleRules); + } + + /** + * Get rules by resource name, save the rule list after regular matching to improve performance + * @param resource resource name + * @return matching rule list + */ + public List getRules(String resource) { + List result = new ArrayList<>(simpleRules.getOrDefault(resource, Collections.emptyList())); + if (regexRules.isEmpty()) { + return result; + } + if (regexCacheRules.containsKey(resource)) { + result.addAll(regexCacheRules.get(resource)); + return result; + } + synchronized (this) { + if (regexCacheRules.containsKey(resource)) { + result.addAll(regexCacheRules.get(resource)); + return result; + } + List compilers = matcherFromRegexRules(resource); + regexCacheRules.put(resource, compilers); + result.addAll(compilers); + return result; + } + } + + /** + * Get rules from regex rules and simple rules + * @return rule list + */ + public List getRules() { + List rules = new ArrayList<>(); + for (Map.Entry> entry : regexRules.entrySet()) { + rules.addAll(entry.getValue()); + } + for (Map.Entry> entry : simpleRules.entrySet()) { + rules.addAll(entry.getValue()); + } + return rules; + } + + /** + * Get origin rules, includes regex and simple rules + * @return original rules + */ + public Map> getOriginalRules() { + return originalRules; + } + + /** + * Determine whether has rule based on the resource name + * @param resource resource name + * @return whether + */ + + public boolean hasConfig(String resource) { + if (resource == null) { + return false; + } + return !getRules(resource).isEmpty(); + } + + /** + * Is valid regex rules + * @param rule rule + * @return weather valid regex rule + */ + public static boolean checkRegexResourceField(AbstractRule rule) { + if (!rule.isRegex()) { + return true; + } + String resourceName = rule.getResource(); + try { + Pattern.compile(resourceName); + return true; + } catch (Exception e) { + return false; + } + } + + private List matcherFromRegexRules(String resource) { + List compilers = new ArrayList<>(); + for (Map.Entry> entry : regexRules.entrySet()) { + if (entry.getKey().matcher(resource).matches()) { + compilers.addAll(generator.apply(entry.getValue())); + } + } + return compilers; + } + + private synchronized void setRules(Map> regexRules, Map> simpleRules) { + this.regexRules = regexRules; + this.simpleRules = simpleRules; + if (regexRules.isEmpty()) { + this.regexCacheRules = Collections.emptyMap(); + return; + } + // rebuild from regex cache rules + Map> rebuildCacheRule = new HashMap<>(regexCacheRules.size()); + for (String resource : regexCacheRules.keySet()) { + rebuildCacheRule.put(resource, matcherFromRegexRules(resource)); + } + this.regexCacheRules = rebuildCacheRule; + } +} diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityRuleManager.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityRuleManager.java index aea81664b7..2ef50c5a97 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityRuleManager.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityRuleManager.java @@ -24,6 +24,7 @@ import com.alibaba.csp.sentinel.log.RecordLog; import com.alibaba.csp.sentinel.slots.block.RuleConstant; +import com.alibaba.csp.sentinel.slots.block.RuleManager; import com.alibaba.csp.sentinel.util.AssertUtil; import com.alibaba.csp.sentinel.util.StringUtil; import com.alibaba.csp.sentinel.property.DynamicSentinelProperty; @@ -39,7 +40,7 @@ */ public final class AuthorityRuleManager { - private static volatile Map> authorityRules = new ConcurrentHashMap<>(); + private static volatile RuleManager authorityRules = new RuleManager<>(); private static final RulePropertyListener LISTENER = new RulePropertyListener(); private static SentinelProperty> currentProperty = new DynamicSentinelProperty<>(); @@ -70,7 +71,7 @@ public static void loadRules(List rules) { } public static boolean hasConfig(String resource) { - return authorityRules.containsKey(resource); + return authorityRules.hasConfig(resource); } /** @@ -79,34 +80,27 @@ public static boolean hasConfig(String resource) { * @return a new copy of the rules. */ public static List getRules() { - List rules = new ArrayList<>(); - if (authorityRules == null) { - return rules; - } - for (Map.Entry> entry : authorityRules.entrySet()) { - rules.addAll(entry.getValue()); - } - return rules; + return authorityRules.getRules(); } private static class RulePropertyListener implements PropertyListener> { @Override public synchronized void configLoad(List value) { - authorityRules = loadAuthorityConf(value); + authorityRules.updateRules(loadAuthorityConf(value)); RecordLog.info("[AuthorityRuleManager] Authority rules loaded: {}", authorityRules); } @Override public synchronized void configUpdate(List conf) { - authorityRules = loadAuthorityConf(conf); - + authorityRules.updateRules(loadAuthorityConf(conf)); + RecordLog.info("[AuthorityRuleManager] Authority rules received: {}", authorityRules); } - private Map> loadAuthorityConf(List list) { - Map> newRuleMap = new ConcurrentHashMap<>(); + private Map> loadAuthorityConf(List list) { + Map> newRuleMap = new ConcurrentHashMap<>(); if (list == null || list.isEmpty()) { return newRuleMap; @@ -123,10 +117,10 @@ private Map> loadAuthorityConf(List li } String identity = rule.getResource(); - Set ruleSet = newRuleMap.get(identity); + List ruleSet = newRuleMap.get(identity); // putIfAbsent if (ruleSet == null) { - ruleSet = new HashSet<>(); + ruleSet = new ArrayList<>(); ruleSet.add(rule); newRuleMap.put(identity, ruleSet); } else { @@ -140,12 +134,12 @@ private Map> loadAuthorityConf(List li } - static Map> getAuthorityRules() { - return authorityRules; + static List getRules(String resource) { + return authorityRules.getRules(resource); } public static boolean isValidRule(AuthorityRule rule) { return rule != null && !StringUtil.isBlank(rule.getResource()) - && rule.getStrategy() >= 0 && StringUtil.isNotBlank(rule.getLimitApp()); + && rule.getStrategy() >= 0 && StringUtil.isNotBlank(rule.getLimitApp()) && RuleManager.checkRegexResourceField(rule); } } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthoritySlot.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthoritySlot.java index 5217b7168f..89c71fde4c 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthoritySlot.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthoritySlot.java @@ -15,8 +15,7 @@ */ package com.alibaba.csp.sentinel.slots.block.authority; -import java.util.Map; -import java.util.Set; +import java.util.List; import com.alibaba.csp.sentinel.Constants; import com.alibaba.csp.sentinel.context.Context; @@ -48,13 +47,8 @@ public void exit(Context context, ResourceWrapper resourceWrapper, int count, Ob } void checkBlackWhiteAuthority(ResourceWrapper resource, Context context) throws AuthorityException { - Map> authorityRules = AuthorityRuleManager.getAuthorityRules(); - if (authorityRules == null) { - return; - } - - Set rules = authorityRules.get(resource.getName()); + List rules = AuthorityRuleManager.getRules(resource.getName()); if (rules == null) { return; } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRuleManager.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRuleManager.java index a1214a7b14..1f9f7b0edc 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRuleManager.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRuleManager.java @@ -21,12 +21,14 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import com.alibaba.csp.sentinel.log.RecordLog; import com.alibaba.csp.sentinel.property.DynamicSentinelProperty; import com.alibaba.csp.sentinel.property.PropertyListener; import com.alibaba.csp.sentinel.property.SentinelProperty; import com.alibaba.csp.sentinel.slots.block.RuleConstant; +import com.alibaba.csp.sentinel.slots.block.RuleManager; import com.alibaba.csp.sentinel.slots.block.degrade.circuitbreaker.CircuitBreaker; import com.alibaba.csp.sentinel.slots.block.degrade.circuitbreaker.ExceptionCircuitBreaker; import com.alibaba.csp.sentinel.slots.block.degrade.circuitbreaker.ResponseTimeCircuitBreaker; @@ -42,8 +44,8 @@ */ public final class DegradeRuleManager { - private static volatile Map> circuitBreakers = new HashMap<>(); - private static volatile Map> ruleMap = new HashMap<>(); + private static volatile RuleManager circuitBreakers = new RuleManager<>(DegradeRuleManager::generateCbs, cb -> cb.getRule().isRegex()); + private static volatile RuleManager ruleMap = new RuleManager<>(); private static final RulePropertyListener LISTENER = new RulePropertyListener(); private static SentinelProperty> currentProperty @@ -70,14 +72,11 @@ public static void register2Property(SentinelProperty> propert } static List getCircuitBreakers(String resourceName) { - return circuitBreakers.get(resourceName); + return circuitBreakers.getRules(resourceName); } public static boolean hasConfig(String resource) { - if (resource == null) { - return false; - } - return circuitBreakers.containsKey(resource); + return circuitBreakers.hasConfig(resource); } /** @@ -88,16 +87,11 @@ public static boolean hasConfig(String resource) { * @return list of existing circuit breaking rules, or empty list if no rules were loaded */ public static List getRules() { - List rules = new ArrayList<>(); - for (Map.Entry> entry : ruleMap.entrySet()) { - rules.addAll(entry.getValue()); - } - return rules; + return ruleMap.getRules(); } - public static Set getRulesOfResource(String resource) { AssertUtil.assertNotBlank(resource, "resource name cannot be blank"); - return ruleMap.get(resource); + return new HashSet<>(ruleMap.getRules(resource)); } /** @@ -124,7 +118,7 @@ public static void loadRules(List rules) { public static boolean setRulesForResource(String resourceName, Set rules) { AssertUtil.notEmpty(resourceName, "resourceName cannot be empty"); try { - Map> newRuleMap = new HashMap<>(ruleMap); + Map> newRuleMap = ruleMap.getOriginalRules(); if (rules == null) { newRuleMap.remove(resourceName); } else { @@ -134,10 +128,10 @@ public static boolean setRulesForResource(String resourceName, Set newSet.add(rule); } } - newRuleMap.put(resourceName, newSet); + newRuleMap.put(resourceName, new ArrayList<>(newSet)); } List allRules = new ArrayList<>(); - for (Set set : newRuleMap.values()) { + for (List set : newRuleMap.values()) { allRules.addAll(set); } return currentProperty.updateValue(allRules); @@ -189,6 +183,9 @@ public static boolean isValidRule(DegradeRule rule) { if (rule.getMinRequestAmount() <= 0 || rule.getStatIntervalMs() <= 0) { return false; } + if (!RuleManager.checkRegexResourceField(rule)) { + return false; + } switch (rule.getGrade()) { case RuleConstant.DEGRADE_GRADE_RT: return rule.getSlowRatioThreshold() >= 0 && rule.getSlowRatioThreshold() <= 1; @@ -201,24 +198,17 @@ public static boolean isValidRule(DegradeRule rule) { } } + private static List generateCbs(List cbs) { + return cbs.stream().map(cb -> newCircuitBreakerFrom(cb.getRule())).collect(Collectors.toList()); + } + private static class RulePropertyListener implements PropertyListener> { private synchronized void reloadFrom(List list) { Map> cbs = buildCircuitBreakers(list); - Map> rm = new HashMap<>(cbs.size()); - - for (Map.Entry> e : cbs.entrySet()) { - assert e.getValue() != null && !e.getValue().isEmpty(); - - Set rules = new HashSet<>(e.getValue().size()); - for (CircuitBreaker cb : e.getValue()) { - rules.add(cb.getRule()); - } - rm.put(e.getKey(), rules); - } - - DegradeRuleManager.circuitBreakers = cbs; - DegradeRuleManager.ruleMap = rm; + Map> rules = buildCircuitBreakerRules(cbs); + circuitBreakers.updateRules(cbs); + ruleMap.updateRules(rules); } @Override @@ -233,6 +223,16 @@ public void configLoad(List conf) { RecordLog.info("[DegradeRuleManager] Degrade rules loaded: {}", ruleMap); } + private Map> buildCircuitBreakerRules(Map> cbs) { + Map> result = new HashMap<>(cbs.size()); + for (Map.Entry> entry : cbs.entrySet()) { + String resource = entry.getKey(); + Set rules = entry.getValue().stream().map(CircuitBreaker::getRule).collect(Collectors.toSet()); + result.put(resource, new ArrayList<>(rules)); + } + return result; + } + private Map> buildCircuitBreakers(List list) { Map> cbMap = new HashMap<>(8); if (list == null || list.isEmpty()) { diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleManager.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleManager.java index 0fb7f4fe2e..967d4346d2 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleManager.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleManager.java @@ -22,6 +22,7 @@ import com.alibaba.csp.sentinel.property.DynamicSentinelProperty; import com.alibaba.csp.sentinel.property.PropertyListener; import com.alibaba.csp.sentinel.property.SentinelProperty; +import com.alibaba.csp.sentinel.slots.block.RuleManager; import com.alibaba.csp.sentinel.util.AssertUtil; import com.alibaba.csp.sentinel.util.StringUtil; @@ -48,7 +49,7 @@ */ public class FlowRuleManager { - private static volatile Map> flowRules = new HashMap<>(); + private static volatile RuleManager flowRules = new RuleManager<>(); private static final FlowPropertyListener LISTENER = new FlowPropertyListener(); private static SentinelProperty> currentProperty = new DynamicSentinelProperty>(); @@ -105,11 +106,7 @@ public static void register2Property(SentinelProperty> property) * @return a new copy of the rules. */ public static List getRules() { - List rules = new ArrayList(); - for (Map.Entry> entry : flowRules.entrySet()) { - rules.addAll(entry.getValue()); - } - return rules; + return flowRules.getRules(); } /** @@ -121,12 +118,12 @@ public static void loadRules(List rules) { currentProperty.updateValue(rules); } - static Map> getFlowRuleMap() { - return flowRules; + static List getFlowRules(String resource) { + return flowRules.getRules(resource); } public static boolean hasConfig(String resource) { - return flowRules.containsKey(resource); + return flowRules.hasConfig(resource); } public static boolean isOtherOrigin(String origin, String resourceName) { @@ -134,7 +131,7 @@ public static boolean isOtherOrigin(String origin, String resourceName) { return false; } - List rules = flowRules.get(resourceName); + List rules = flowRules.getRules(resourceName); if (rules != null) { for (FlowRule rule : rules) { @@ -152,18 +149,14 @@ private static final class FlowPropertyListener implements PropertyListener value) { Map> rules = FlowRuleUtil.buildFlowRuleMap(value); - if (rules != null) { - flowRules = rules; - } + flowRules.updateRules(rules); RecordLog.info("[FlowRuleManager] Flow rules received: {}", rules); } @Override public synchronized void configLoad(List conf) { Map> rules = FlowRuleUtil.buildFlowRuleMap(conf); - if (rules != null) { - flowRules = rules; - } + flowRules.updateRules(rules); RecordLog.info("[FlowRuleManager] Flow rules loaded: {}", rules); } } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleUtil.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleUtil.java index b607ef6bb4..bc28a9dccb 100644 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleUtil.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleUtil.java @@ -18,6 +18,7 @@ import com.alibaba.csp.sentinel.log.RecordLog; import com.alibaba.csp.sentinel.slots.block.ClusterRuleConstant; import com.alibaba.csp.sentinel.slots.block.RuleConstant; +import com.alibaba.csp.sentinel.slots.block.RuleManager; import com.alibaba.csp.sentinel.slots.block.flow.controller.DefaultController; import com.alibaba.csp.sentinel.slots.block.flow.controller.ThrottlingController; import com.alibaba.csp.sentinel.slots.block.flow.controller.WarmUpController; @@ -170,6 +171,9 @@ public static boolean isValidRule(FlowRule rule) { if (!baseValid) { return false; } + if (!checkRegexField(rule)) { + return false; + } if (rule.getGrade() == RuleConstant.FLOW_GRADE_QPS) { // Check strategy and control (shaping) behavior. return checkClusterField(rule) && checkStrategyField(rule) && checkControlBehaviorField(rule); @@ -237,6 +241,16 @@ private static boolean checkStrategyField(/*@NonNull*/ FlowRule rule) { return true; } + private static boolean checkRegexField(FlowRule rule) { + if (!RuleManager.checkRegexResourceField(rule)) { + return false; + } + if (rule.isRegex()) { + return !rule.isClusterMode() && rule.getControlBehavior() == RuleConstant.CONTROL_BEHAVIOR_DEFAULT; + } + return true; + } + private static boolean checkControlBehaviorField(/*@NonNull*/ FlowRule rule) { switch (rule.getControlBehavior()) { case RuleConstant.CONTROL_BEHAVIOR_WARM_UP: diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowSlot.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowSlot.java index 96b051a735..745f826121 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowSlot.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowSlot.java @@ -26,8 +26,6 @@ import com.alibaba.csp.sentinel.util.function.Function; import java.util.Collection; -import java.util.List; -import java.util.Map; /** *

@@ -179,9 +177,7 @@ public void exit(Context context, ResourceWrapper resourceWrapper, int count, Ob private final Function> ruleProvider = new Function>() { @Override public Collection apply(String resource) { - // Flow rule map should not be null. - Map> flowRules = FlowRuleManager.getFlowRuleMap(); - return flowRules.get(resource); + return FlowRuleManager.getFlowRules(resource); } }; } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/util/function/Function.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/util/function/Function.java index edcaa76c72..127bc13604 100644 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/util/function/Function.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/util/function/Function.java @@ -27,4 +27,14 @@ public interface Function { * @return the function result */ R apply(T t); + + /** + * Returns a function that always returns its input argument. + * + * @param the type of the input and output objects to the function + * @return a function that always returns its input argument + */ + static Function identity() { + return t -> t; + } } diff --git a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/RuleManagerTest.java b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/RuleManagerTest.java new file mode 100644 index 0000000000..12135e6d01 --- /dev/null +++ b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/RuleManagerTest.java @@ -0,0 +1,112 @@ +package com.alibaba.csp.sentinel.slots.block; + +import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.*; +import java.util.regex.Pattern; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.when; + +public class RuleManagerTest { + + private RuleManager ruleManager; + + @Before + public void setUp() throws Exception { + ruleManager = new RuleManager<>(); + } + + @Test + public void testUpdateRules() throws Exception { + // Setup + final Map> rulesMap = generateFlowRules(true); + + // Run the test + ruleManager.updateRules(rulesMap); + + // Verify the results + assertEquals(ruleManager.getRules().size(), 2); + Field regexRules = RuleManager.class.getDeclaredField("regexRules"); + regexRules.setAccessible(true); + assertEquals(((Map)regexRules.get(ruleManager)).size(), 1); + Field simpleRules = RuleManager.class.getDeclaredField("simpleRules"); + simpleRules.setAccessible(true); + assertEquals(((Map)simpleRules.get(ruleManager)).size(), 1); + } + + @Test + public void testGetRulesWithCache() throws Exception { + // Setup + final Map> rulesMap = generateFlowRules(true); + + // Run the test + ruleManager.updateRules(rulesMap); + + // Verify the results + Field regexCacheRules = RuleManager.class.getDeclaredField("regexCacheRules"); + regexCacheRules.setAccessible(true); + assertEquals(((Map)regexCacheRules.get(ruleManager)).size(), 0); + ruleManager.getRules("rule2"); + assertEquals(((Map)regexCacheRules.get(ruleManager)).size(), 1); + } + + @Test + public void testRebuildRulesWhenUpdateRules() throws Exception { + // Setup + final Map> rulesMap = generateFlowRules(true); + + // Run the test + ruleManager.updateRules(rulesMap); + ruleManager.getRules("rule2"); + ruleManager.updateRules(generateFlowRules(true)); + + // Verify the results + Field regexCacheRules = RuleManager.class.getDeclaredField("regexCacheRules"); + regexCacheRules.setAccessible(true); + assertEquals(((Map)regexCacheRules.get(ruleManager)).size(), 1); + + // Clean up regular rules + ruleManager.updateRules(generateFlowRules(false)); + // Verify the results + assertEquals(((Map)regexCacheRules.get(ruleManager)).size(), 0); + } + @Test + public void testValidRegexRule() { + // Setup + FlowRule flowRule = new FlowRule(); + flowRule.setRegex(true); + flowRule.setResource("{}"); + // Run the test and verify + Assert.assertFalse(RuleManager.checkRegexResourceField(flowRule)); + + flowRule.setResource(".*"); + // Run the test and verify + Assert.assertTrue(RuleManager.checkRegexResourceField(flowRule)); + } + + @Test + public void testHasConfig() { + // Setup + final Map> rulesMap = generateFlowRules(true); + + // Run the test and verify the results + ruleManager.updateRules(rulesMap); + assertTrue(ruleManager.hasConfig("rule1")); + assertFalse(ruleManager.hasConfig("rule3")); + } + + private Map> generateFlowRules(boolean withRegex) { + Map> result = new HashMap<>(2); + FlowRule flowRule1 = new FlowRule("rule1"); + flowRule1.setRegex(withRegex); + result.put(flowRule1.getResource(), Collections.singletonList(flowRule1)); + FlowRule flowRule2 = new FlowRule("rule2"); + result.put(flowRule2.getResource(), Collections.singletonList(flowRule2)); + return result; + } +} diff --git a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityPartialIntegrationTest.java b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityPartialIntegrationTest.java new file mode 100644 index 0000000000..ce369caee5 --- /dev/null +++ b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityPartialIntegrationTest.java @@ -0,0 +1,49 @@ +package com.alibaba.csp.sentinel.slots.block.authority; + +import com.alibaba.csp.sentinel.Entry; +import com.alibaba.csp.sentinel.SphU; +import com.alibaba.csp.sentinel.context.ContextUtil; +import com.alibaba.csp.sentinel.slots.block.BlockException; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author quguai + * @date 2023/10/27 11:48 + */ +public class AuthorityPartialIntegrationTest { + + @Test + public void testRegex() { + AuthorityRule authorityRule = new AuthorityRule(); + authorityRule.setRegex(true); + authorityRule.setStrategy(1); + authorityRule.setLimitApp("appA"); + authorityRule.setResource(".*"); + AuthorityRuleManager.loadRules(Collections.singletonList(authorityRule)); + verifyFlow("testRegex_1", "appA", false); + verifyFlow("testRegex_2", "appA", false); + verifyFlow("testRegex_1", "appB", true); + verifyFlow("testRegex_2", "appB", true); + } + + private void verifyFlow(String resource, String origin, boolean shouldPass) { + ContextUtil.enter("a", origin); + Entry e = null; + try { + e = SphU.entry(resource); + assertTrue(shouldPass); + } catch (BlockException e1) { + assertFalse(shouldPass); + } finally { + if (e != null) { + e.exit(); + } + ContextUtil.exit(); + } + } +} diff --git a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradePartialIntegrationTest.java b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradePartialIntegrationTest.java new file mode 100644 index 0000000000..00047995a3 --- /dev/null +++ b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradePartialIntegrationTest.java @@ -0,0 +1,73 @@ +package com.alibaba.csp.sentinel.slots.block.degrade; + +import com.alibaba.csp.sentinel.Entry; +import com.alibaba.csp.sentinel.SphU; +import com.alibaba.csp.sentinel.Tracer; +import com.alibaba.csp.sentinel.context.ContextUtil; +import com.alibaba.csp.sentinel.slots.block.BlockException; +import com.alibaba.csp.sentinel.slots.block.RuleConstant; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author quguai + * @date 2023/10/27 13:56 + */ +public class DegradePartialIntegrationTest { + + @Before + public void setUp() throws Exception { + DegradeRuleManager.loadRules(new ArrayList<>()); + } + + @After + public void tearDown() throws Exception { + DegradeRuleManager.loadRules(new ArrayList<>()); + } + + @Test + public void testDegradeRegex() { + DegradeRule rule = new DegradeRule(".*") + .setCount(0.5d) + .setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) + .setStatIntervalMs(20 * 1000) + .setTimeWindow(10) + .setMinRequestAmount(1); + rule.setRegex(true); + DegradeRuleManager.loadRules(Collections.singletonList(rule)); + + verifyDegradeFlow("testDegradeRegex_1", true, true); + verifyDegradeFlow("testDegradeRegex_1", true, false); + + verifyDegradeFlow("testDegradeRegex_2", true, true); + verifyDegradeFlow("testDegradeRegex_2", true, false); + + } + + private void verifyDegradeFlow(String resource, boolean error, boolean shouldPass) { + Entry entry = null; + try { + entry = SphU.entry(resource); + assertTrue(shouldPass); + if (error) { + int i = 10 / 0; + } + } catch (BlockException e1) { + assertFalse(shouldPass); + } catch (Exception ex) { + Tracer.traceEntry(ex, entry); + } finally { + if (entry != null) { + entry.exit(); + } + } + } +} diff --git a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/flow/FlowPartialIntegrationTest.java b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/flow/FlowPartialIntegrationTest.java index faf8cca70f..a48be564ea 100755 --- a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/flow/FlowPartialIntegrationTest.java +++ b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/flow/FlowPartialIntegrationTest.java @@ -15,9 +15,6 @@ */ package com.alibaba.csp.sentinel.slots.block.flow; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -32,6 +29,8 @@ import com.alibaba.csp.sentinel.slots.block.BlockException; import com.alibaba.csp.sentinel.slots.block.RuleConstant; +import static org.junit.Assert.*; + /** * @author jialiang.linjl */ @@ -114,6 +113,21 @@ public void run() { System.out.println("done"); } + @Test + public void testQpsRegex() { + FlowRule flowRule = new FlowRule(); + String resource = ".*"; + flowRule.setResource(resource); + flowRule.setGrade(RuleConstant.FLOW_GRADE_QPS); + flowRule.setRegex(true); + flowRule.setCount(1); + FlowRuleManager.loadRules(Collections.singletonList(flowRule)); + verifyFlow("testQpsRegex_1", true); + verifyFlow("testQpsRegex_2", true); + verifyFlow("testQpsRegex_1", false); + verifyFlow("testQpsRegex_2", false); + } + @Test public void testOriginFlowRule() { String RESOURCE_NAME = "testOriginFlowRule"; @@ -255,4 +269,18 @@ public void testStrategy_chain() { ContextUtil.exit(); } + + private void verifyFlow(String resource, boolean shouldPass) { + Entry e = null; + try { + e = SphU.entry(resource); + assertTrue(shouldPass); + } catch (BlockException e1) { + assertFalse(shouldPass); + } finally { + if (e != null) { + e.exit(); + } + } + } } diff --git a/sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleManager.java b/sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleManager.java index 0802226ae9..a8e72e1379 100644 --- a/sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleManager.java +++ b/sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleManager.java @@ -24,6 +24,7 @@ import com.alibaba.csp.sentinel.property.DynamicSentinelProperty; import com.alibaba.csp.sentinel.property.PropertyListener; import com.alibaba.csp.sentinel.property.SentinelProperty; +import com.alibaba.csp.sentinel.slots.block.RuleManager; import com.alibaba.csp.sentinel.util.AssertUtil; /** @@ -35,8 +36,7 @@ */ public final class ParamFlowRuleManager { - private static final Map> PARAM_FLOW_RULES = new ConcurrentHashMap<>(); - + private static final RuleManager PARAM_FLOW_RULES = new RuleManager<>(); private final static RulePropertyListener PROPERTY_LISTENER = new RulePropertyListener(); private static SentinelProperty> currentProperty = new DynamicSentinelProperty<>(); @@ -75,12 +75,11 @@ public static void register2Property(SentinelProperty> prope } public static List getRulesOfResource(String resourceName) { - return new ArrayList<>(PARAM_FLOW_RULES.get(resourceName)); + return new ArrayList<>(PARAM_FLOW_RULES.getRules(resourceName)); } public static boolean hasRules(String resourceName) { - List rules = PARAM_FLOW_RULES.get(resourceName); - return rules != null && !rules.isEmpty(); + return PARAM_FLOW_RULES.hasConfig(resourceName); } /** @@ -89,11 +88,7 @@ public static boolean hasRules(String resourceName) { * @return a new copy of the rules. */ public static List getRules() { - List rules = new ArrayList<>(); - for (Map.Entry> entry : PARAM_FLOW_RULES.entrySet()) { - rules.addAll(entry.getValue()); - } - return rules; + return PARAM_FLOW_RULES.getRules(); } static class RulePropertyListener implements PropertyListener> { @@ -101,20 +96,14 @@ static class RulePropertyListener implements PropertyListener list) { Map> rules = aggregateAndPrepareParamRules(list); - if (rules != null) { - PARAM_FLOW_RULES.clear(); - PARAM_FLOW_RULES.putAll(rules); - } + PARAM_FLOW_RULES.updateRules(rules); RecordLog.info("[ParamFlowRuleManager] Parameter flow rules received: {}", PARAM_FLOW_RULES); } @Override public void configLoad(List list) { Map> rules = aggregateAndPrepareParamRules(list); - if (rules != null) { - PARAM_FLOW_RULES.clear(); - PARAM_FLOW_RULES.putAll(rules); - } + PARAM_FLOW_RULES.updateRules(rules); RecordLog.info("[ParamFlowRuleManager] Parameter flow rules received: {}", PARAM_FLOW_RULES); } @@ -128,7 +117,7 @@ private Map> aggregateAndPrepareParamRules(List> entry : PARAM_FLOW_RULES.entrySet()) { + for (Map.Entry> entry : PARAM_FLOW_RULES.getOriginalRules().entrySet()) { String resource = entry.getKey(); if (!newRuleMap.containsKey(resource)) { ParameterMetricStorage.clearParamMetricForResource(resource); diff --git a/sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleUtil.java b/sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleUtil.java index dd4c1207b6..80c62130c8 100644 --- a/sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleUtil.java +++ b/sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleUtil.java @@ -26,6 +26,7 @@ import com.alibaba.csp.sentinel.log.RecordLog; import com.alibaba.csp.sentinel.slots.block.RuleConstant; +import com.alibaba.csp.sentinel.slots.block.RuleManager; import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleUtil; import com.alibaba.csp.sentinel.util.AssertUtil; import com.alibaba.csp.sentinel.util.StringUtil; @@ -48,7 +49,7 @@ public static boolean isValidRule(ParamFlowRule rule) { && rule.getGrade() >= 0 && rule.getParamIdx() != null && rule.getBurstCount() >= 0 && rule.getControlBehavior() >= 0 && rule.getDurationInSec() > 0 && rule.getMaxQueueingTimeMs() >= 0 - && checkCluster(rule); + && checkCluster(rule) & checkRegexField(rule); } private static boolean checkCluster(/*@PreChecked*/ ParamFlowRule rule) { @@ -65,6 +66,16 @@ private static boolean checkCluster(/*@PreChecked*/ ParamFlowRule rule) { return validClusterRuleId(clusterConfig.getFlowId()); } + private static boolean checkRegexField(ParamFlowRule rule) { + if (!RuleManager.checkRegexResourceField(rule)) { + return false; + } + if (rule.isRegex()) { + return !rule.isClusterMode() && rule.getControlBehavior() == RuleConstant.CONTROL_BEHAVIOR_DEFAULT; + } + return true; + } + public static boolean validClusterRuleId(Long id) { return id != null && id > 0; } diff --git a/sentinel-extension/sentinel-parameter-flow-control/src/test/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowPartialIntegrationTest.java b/sentinel-extension/sentinel-parameter-flow-control/src/test/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowPartialIntegrationTest.java new file mode 100644 index 0000000000..7c11d27d7c --- /dev/null +++ b/sentinel-extension/sentinel-parameter-flow-control/src/test/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowPartialIntegrationTest.java @@ -0,0 +1,68 @@ +package com.alibaba.csp.sentinel.slots.block.flow.param; + +import com.alibaba.csp.sentinel.Entry; +import com.alibaba.csp.sentinel.EntryType; +import com.alibaba.csp.sentinel.SphU; +import com.alibaba.csp.sentinel.slots.block.BlockException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author quguai + * @date 2023/10/27 13:44 + */ +public class ParamFlowPartialIntegrationTest { + + @Before + public void setUp() throws Exception { + ParamFlowRuleManager.loadRules(new ArrayList<>()); + } + + @After + public void tearDown() throws Exception { + ParamFlowRuleManager.loadRules(new ArrayList<>()); + } + + @Test + public void testParamFlowRegex() { + ParamFlowRule rule = new ParamFlowRule(".*") + .setParamIdx(0) + .setCount(1); + rule.setRegex(true); + ParamFlowRuleManager.loadRules(Collections.singletonList(rule)); + verifyFlow("testParamFlowRegex_1", true, "args"); + verifyFlow("testParamFlowRegex_1", true, "args_1"); + + verifyFlow("testParamFlowRegex_1", false, "args"); + verifyFlow("testParamFlowRegex_1", false, "args_1"); + + verifyFlow("testParamFlowRegex_2", true, "args"); + verifyFlow("testParamFlowRegex_2", true, "args_1"); + + verifyFlow("testParamFlowRegex_2", false, "args"); + verifyFlow("testParamFlowRegex_2", false, "args_1"); + } + + + private void verifyFlow(String resource, boolean shouldPass, String... args) { + Entry e = null; + try { + e = SphU.entry(resource, 1, EntryType.IN, args); + assertTrue(shouldPass); + } catch (BlockException e1) { + assertFalse(shouldPass); + } finally { + if (e != null) { + e.exit(1, args); + } + } + } + +}