crabapples
/
Sentinel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Automatically de-duplicate rules when loading rules (#571) 

* De-duplicate rules automatically when loading rules
* Update rule managers

Signed-off-by: Eric Zhao <sczyh16@gmail.com>
This commit is contained in:
Eric Zhao 2019-03-14 14:05:13 +08:00 committed by GitHub
parent ae774dba61
commit 1741da0bab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 138 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityRuleManager.java
View File

@ -16,12 +16,15 @@
package com.alibaba.csp.sentinel.slots.block.authority;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.alibaba.csp.sentinel.log.RecordLog;
import com.alibaba.csp.sentinel.slots.block.RuleConstant;
import com.alibaba.csp.sentinel.util.AssertUtil;
import com.alibaba.csp.sentinel.util.StringUtil;
import com.alibaba.csp.sentinel.property.DynamicSentinelProperty;
import com.alibaba.csp.sentinel.property.PropertyListener;
@ -36,24 +39,22 @@ import com.alibaba.csp.sentinel.property.SentinelProperty;
*/
public final class AuthorityRuleManager {
private static Map<String, List<AuthorityRule>> authorityRules
= new ConcurrentHashMap<String, List<AuthorityRule>>();
private static Map<String, Set<AuthorityRule>> authorityRules = new ConcurrentHashMap<>();
final static RulePropertyListener listener = new RulePropertyListener();
private static SentinelProperty<List<AuthorityRule>> currentProperty
= new DynamicSentinelProperty<List<AuthorityRule>>();
private static final RulePropertyListener LISTENER = new RulePropertyListener();
private static SentinelProperty<List<AuthorityRule>> currentProperty = new DynamicSentinelProperty<>();
static {
currentProperty.addListener(listener);
currentProperty.addListener(LISTENER);
}
public static void register2Property(SentinelProperty<List<AuthorityRule>> property) {
synchronized (listener) {
AssertUtil.notNull(property, "property cannot be null");
synchronized (LISTENER) {
if (currentProperty != null) {
currentProperty.removeListener(listener);
currentProperty.removeListener(LISTENER);
}
property.addListener(listener);
property.addListener(LISTENER);
currentProperty = property;
RecordLog.info("[AuthorityRuleManager] Registering new property to authority rule manager");
}
@ -78,11 +79,11 @@ public final class AuthorityRuleManager {
* @return a new copy of the rules.
*/
public static List<AuthorityRule> getRules() {
List<AuthorityRule> rules = new ArrayList<AuthorityRule>();
List<AuthorityRule> rules = new ArrayList<>();
if (authorityRules == null) {
return rules;
}
for (Map.Entry<String, List<AuthorityRule>> entry : authorityRules.entrySet()) {
for (Map.Entry<String, Set<AuthorityRule>> entry : authorityRules.entrySet()) {
rules.addAll(entry.getValue());
}
return rules;
@ -92,7 +93,7 @@ public final class AuthorityRuleManager {
@Override
public void configUpdate(List<AuthorityRule> conf) {
Map<String, List<AuthorityRule>> rules = loadAuthorityConf(conf);
Map<String, Set<AuthorityRule>> rules = loadAuthorityConf(conf);
authorityRules.clear();
if (rules != null) {
@ -101,8 +102,8 @@ public final class AuthorityRuleManager {
RecordLog.info("[AuthorityRuleManager] Authority rules received: " + authorityRules);
}
private Map<String, List<AuthorityRule>> loadAuthorityConf(List<AuthorityRule> list) {
Map<String, List<AuthorityRule>> newRuleMap = new ConcurrentHashMap<String, List<AuthorityRule>>();
private Map<String, Set<AuthorityRule>> loadAuthorityConf(List<AuthorityRule> list) {
Map<String, Set<AuthorityRule>> newRuleMap = new ConcurrentHashMap<>();
if (list == null || list.isEmpty()) {
return newRuleMap;
@ -119,12 +120,12 @@ public final class AuthorityRuleManager {
}
String identity = rule.getResource();
List<AuthorityRule> ruleM = newRuleMap.get(identity);
Set<AuthorityRule> ruleSet = newRuleMap.get(identity);
// putIfAbsent
if (ruleM == null) {
ruleM = new ArrayList<AuthorityRule>();
ruleM.add(rule);
newRuleMap.put(identity, ruleM);
if (ruleSet == null) {
ruleSet = new HashSet<>();
ruleSet.add(rule);
newRuleMap.put(identity, ruleSet);
} else {
// One resource should only have at most one authority rule, so just ignore redundant rules.
RecordLog.warn("[AuthorityRuleManager] Ignoring redundant rule: " + rule.toString());
@ -136,7 +137,7 @@ public final class AuthorityRuleManager {
@Override
public void configLoad(List<AuthorityRule> value) {
Map<String, List<AuthorityRule>> rules = loadAuthorityConf(value);
Map<String, Set<AuthorityRule>> rules = loadAuthorityConf(value);
authorityRules.clear();
if (rules != null) {
@ -146,11 +147,11 @@ public final class AuthorityRuleManager {
}
}
static Map<String, List<AuthorityRule>> getAuthorityRules() {
static Map<String, Set<AuthorityRule>> getAuthorityRules() {
return authorityRules;
}
static boolean isValidRule(AuthorityRule rule) {
public static boolean isValidRule(AuthorityRule rule) {
return rule != null && !StringUtil.isBlank(rule.getResource())
&& rule.getStrategy() >= 0 && StringUtil.isNotBlank(rule.getLimitApp());
}

6
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthoritySlot.java
View File

@ -15,8 +15,8 @@
*/
package com.alibaba.csp.sentinel.slots.block.authority;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.alibaba.csp.sentinel.context.Context;
import com.alibaba.csp.sentinel.node.DefaultNode;
@ -45,13 +45,13 @@ public class AuthoritySlot extends AbstractLinkedProcessorSlot<DefaultNode> {
}
void checkBlackWhiteAuthority(ResourceWrapper resource, Context context) throws AuthorityException {
Map<String, List<AuthorityRule>> authorityRules = AuthorityRuleManager.getAuthorityRules();
Map<String, Set<AuthorityRule>> authorityRules = AuthorityRuleManager.getAuthorityRules();
if (authorityRules == null) {
return;
}
List<AuthorityRule> rules = authorityRules.get(resource.getName());
Set<AuthorityRule> rules = authorityRules.get(resource.getName());
if (rules == null) {
return;
}

107
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRuleManager.java
View File

@ -16,10 +16,14 @@
package com.alibaba.csp.sentinel.slots.block.degrade;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.alibaba.csp.sentinel.Constants;
import com.alibaba.csp.sentinel.context.Context;
import com.alibaba.csp.sentinel.log.RecordLog;
import com.alibaba.csp.sentinel.node.DefaultNode;
@ -29,23 +33,24 @@ import com.alibaba.csp.sentinel.property.SentinelProperty;
import com.alibaba.csp.sentinel.slotchain.ResourceWrapper;
import com.alibaba.csp.sentinel.slots.block.BlockException;
import com.alibaba.csp.sentinel.slots.block.RuleConstant;
import com.alibaba.csp.sentinel.util.AssertUtil;
import com.alibaba.csp.sentinel.util.StringUtil;
/***
/**
* @author youji.zj
* @author jialiang.linjl
* @author Eric Zhao
*/
public class DegradeRuleManager {
public final class DegradeRuleManager {
private static volatile Map<String, List<DegradeRule>> degradeRules
= new ConcurrentHashMap<String, List<DegradeRule>>();
private static final Map<String, Set<DegradeRule>> degradeRules = new ConcurrentHashMap<>();
final static RulePropertyListener listener = new RulePropertyListener();
private static final RulePropertyListener LISTENER = new RulePropertyListener();
private static SentinelProperty<List<DegradeRule>> currentProperty
= new DynamicSentinelProperty<List<DegradeRule>>();
= new DynamicSentinelProperty<>();
static {
currentProperty.addListener(listener);
currentProperty.addListener(LISTENER);
}
/**
@ -55,21 +60,19 @@ public class DegradeRuleManager {
* @param property the property to listen.
*/
public static void register2Property(SentinelProperty<List<DegradeRule>> property) {
synchronized (listener) {
AssertUtil.notNull(property, "property cannot be null");
synchronized (LISTENER) {
RecordLog.info("[DegradeRuleManager] Registering new property to degrade rule manager");
currentProperty.removeListener(listener);
property.addListener(listener);
currentProperty.removeListener(LISTENER);
property.addListener(LISTENER);
currentProperty = property;
}
}
public static void checkDegrade(ResourceWrapper resource, Context context, DefaultNode node, int count)
throws BlockException {
if (degradeRules == null) {
return;
}
List<DegradeRule> rules = degradeRules.get(resource.getName());
Set<DegradeRule> rules = degradeRules.get(resource.getName());
if (rules == null) {
return;
}
@ -82,6 +85,9 @@ public class DegradeRuleManager {
}
public static boolean hasConfig(String resource) {
if (resource == null) {
return false;
}
return degradeRules.containsKey(resource);
}
@ -91,11 +97,8 @@ public class DegradeRuleManager {
* @return a new copy of the rules.
*/
public static List<DegradeRule> getRules() {
List<DegradeRule> rules = new ArrayList<DegradeRule>();
if (degradeRules == null) {
return rules;
}
for (Map.Entry<String, List<DegradeRule>> entry : degradeRules.entrySet()) {
List<DegradeRule> rules = new ArrayList<>();
for (Map.Entry<String, Set<DegradeRule>> entry : degradeRules.entrySet()) {
rules.addAll(entry.getValue());
}
return rules;
@ -110,7 +113,42 @@ public class DegradeRuleManager {
try {
currentProperty.updateValue(rules);
} catch (Throwable e) {
RecordLog.info(e.getMessage(), e);
RecordLog.warn("[DegradeRuleManager] Unexpected error when loading degrade rules", e);
}
}
/**
* Set degrade rules for provided resource. Former rules of the resource will be replaced.
*
* @param resourceName valid resource name
* @param rules new rule set to load
* @return whether the rules has actually been updated
* @since 1.5.0
*/
public static boolean setRulesForResource(String resourceName, Set<DegradeRule> rules) {
AssertUtil.notEmpty(resourceName, "resourceName cannot be empty");
try {
Map<String, Set<DegradeRule>> newRuleMap = new HashMap<>(degradeRules);
if (rules == null) {
newRuleMap.remove(resourceName);
} else {
Set<DegradeRule> newSet = new HashSet<>();
for (DegradeRule rule : rules) {
if (isValidRule(rule) && resourceName.equals(rule.getResource())) {
newSet.add(rule);
}
}
newRuleMap.put(resourceName, newSet);
}
List<DegradeRule> allRules = new ArrayList<>();
for (Set<DegradeRule> set : newRuleMap.values()) {
allRules.addAll(set);
}
return currentProperty.updateValue(allRules);
} catch (Throwable e) {
RecordLog.warn(
"[DegradeRuleManager] Unexpected error when setting degrade rules for resource: " + resourceName, e);
return false;
}
}
@ -118,7 +156,7 @@ public class DegradeRuleManager {
@Override
public void configUpdate(List<DegradeRule> conf) {
Map<String, List<DegradeRule>> rules = loadDegradeConf(conf);
Map<String, Set<DegradeRule>> rules = loadDegradeConf(conf);
if (rules != null) {
degradeRules.clear();
degradeRules.putAll(rules);
@ -128,7 +166,7 @@ public class DegradeRuleManager {
@Override
public void configLoad(List<DegradeRule> conf) {
Map<String, List<DegradeRule>> rules = loadDegradeConf(conf);
Map<String, Set<DegradeRule>> rules = loadDegradeConf(conf);
if (rules != null) {
degradeRules.clear();
degradeRules.putAll(rules);
@ -136,8 +174,8 @@ public class DegradeRuleManager {
RecordLog.info("[DegradeRuleManager] Degrade rules loaded: " + degradeRules);
}
private Map<String, List<DegradeRule>> loadDegradeConf(List<DegradeRule> list) {
Map<String, List<DegradeRule>> newRuleMap = new ConcurrentHashMap<String, List<DegradeRule>>();
private Map<String, Set<DegradeRule>> loadDegradeConf(List<DegradeRule> list) {
Map<String, Set<DegradeRule>> newRuleMap = new ConcurrentHashMap<>();
if (list == null || list.isEmpty()) {
return newRuleMap;
@ -145,7 +183,8 @@ public class DegradeRuleManager {
for (DegradeRule rule : list) {
if (!isValidRule(rule)) {
RecordLog.warn("[DegradeRuleManager] Ignoring invalid degrade rule when loading new rules: " + rule);
RecordLog.warn(
"[DegradeRuleManager] Ignoring invalid degrade rule when loading new rules: " + rule);
continue;
}
@ -154,17 +193,16 @@ public class DegradeRuleManager {
}
String identity = rule.getResource();
List<DegradeRule> ruleM = newRuleMap.get(identity);
if (ruleM == null) {
ruleM = new ArrayList<DegradeRule>();
newRuleMap.put(identity, ruleM);
Set<DegradeRule> ruleSet = newRuleMap.get(identity);
if (ruleSet == null) {
ruleSet = new HashSet<>();
newRuleMap.put(identity, ruleSet);
}
ruleM.add(rule);
ruleSet.add(rule);
}
return newRuleMap;
}
}
public static boolean isValidRule(DegradeRule rule) {
@ -173,6 +211,13 @@ public class DegradeRuleManager {
if (!baseValid) {
return false;
}
// Warn for RT mode that exceeds the {@code TIME_DROP_VALVE}.
int maxAllowedRt = Constants.TIME_DROP_VALVE;
if (rule.getGrade() == RuleConstant.DEGRADE_GRADE_RT && rule.getCount() > maxAllowedRt) {
RecordLog.warn(String.format("[DegradeRuleManager] WARN: setting large RT threshold (%.1f ms) in RT mode"
+ " will not take effect since it exceeds the max allowed value (%d ms)", rule.getCount(),
maxAllowedRt));
}
// Check exception ratio mode.
if (rule.getGrade() == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO && rule.getCount() > 1) {
return false;

2
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleManager.java
View File

@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit;
import com.alibaba.csp.sentinel.concurrent.NamedThreadFactory;
import com.alibaba.csp.sentinel.log.RecordLog;
import com.alibaba.csp.sentinel.util.AssertUtil;
import com.alibaba.csp.sentinel.util.StringUtil;
import com.alibaba.csp.sentinel.node.metric.MetricTimerListener;
import com.alibaba.csp.sentinel.property.DynamicSentinelProperty;
@ -65,6 +66,7 @@ public class FlowRuleManager {
* @param property the property to listen.
*/
public static void register2Property(SentinelProperty<List<FlowRule>> property) {
AssertUtil.notNull(property, "property cannot be null");
synchronized (LISTENER) {
RecordLog.info("[FlowRuleManager] Registering new property to flow rule manager");
currentProperty.removeListener(LISTENER);

24
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleUtil.java
View File

@ -18,8 +18,11 @@ package com.alibaba.csp.sentinel.slots.block.flow;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.alibaba.csp.sentinel.log.RecordLog;
@ -85,10 +88,11 @@ public final class FlowRuleUtil {
*/
public static <K> Map<K, List<FlowRule>> buildFlowRuleMap(List<FlowRule> list, Function<FlowRule, K> groupFunction,
Predicate<FlowRule> filter, boolean shouldSort) {
Map<K, List<FlowRule>> newRuleMap = new ConcurrentHashMap<K, List<FlowRule>>();
Map<K, List<FlowRule>> newRuleMap = new ConcurrentHashMap<>();
if (list == null || list.isEmpty()) {
return newRuleMap;
}
Map<K, Set<FlowRule>> tmpMap = new ConcurrentHashMap<>();
for (FlowRule rule : list) {
if (!isValidRule(rule)) {
@ -109,22 +113,24 @@ public final class FlowRuleUtil {
if (key == null) {
continue;
}
List<FlowRule> flowRules = newRuleMap.get(key);
Set<FlowRule> flowRules = tmpMap.get(key);
if (flowRules == null) {
flowRules = new ArrayList<FlowRule>();
newRuleMap.put(key, flowRules);
// Use hash set here to remove duplicate rules.
flowRules = new HashSet<>();
tmpMap.put(key, flowRules);
}
flowRules.add(rule);
}
if (shouldSort && !newRuleMap.isEmpty()) {
Comparator<FlowRule> comparator = new FlowRuleComparator();
// Sort the rules.
for (List<FlowRule> rules : newRuleMap.values()) {
Comparator<FlowRule> comparator = new FlowRuleComparator();
for (Entry<K, Set<FlowRule>> entries : tmpMap.entrySet()) {
List<FlowRule> rules = new ArrayList<>(entries.getValue());
if (shouldSort) {
// Sort the rules.
Collections.sort(rules, comparator);
}
newRuleMap.put(entries.getKey(), rules);
}
return newRuleMap;

35
sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleManager.java
View File

@ -16,6 +16,7 @@
package com.alibaba.csp.sentinel.slots.block.flow.param;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -26,6 +27,7 @@ import com.alibaba.csp.sentinel.property.DynamicSentinelProperty;
import com.alibaba.csp.sentinel.property.PropertyListener;
import com.alibaba.csp.sentinel.property.SentinelProperty;
import com.alibaba.csp.sentinel.slots.block.RuleConstant;
import com.alibaba.csp.sentinel.util.AssertUtil;
import com.alibaba.csp.sentinel.util.StringUtil;
/**
@ -37,12 +39,10 @@ import com.alibaba.csp.sentinel.util.StringUtil;
*/
public final class ParamFlowRuleManager {
private static final Map<String, List<ParamFlowRule>> paramFlowRules
= new ConcurrentHashMap<String, List<ParamFlowRule>>();
private static final Map<String, Set<ParamFlowRule>> paramFlowRules = new ConcurrentHashMap<>();
private final static RulePropertyListener PROPERTY_LISTENER = new RulePropertyListener();
private static SentinelProperty<List<ParamFlowRule>> currentProperty
= new DynamicSentinelProperty<List<ParamFlowRule>>();
private static SentinelProperty<List<ParamFlowRule>> currentProperty = new DynamicSentinelProperty<>();
static {
currentProperty.addListener(PROPERTY_LISTENER);
@ -68,6 +68,7 @@ public final class ParamFlowRuleManager {
* @param property the property to listen
*/
public static void register2Property(SentinelProperty<List<ParamFlowRule>> property) {
AssertUtil.notNull(property, "property cannot be null");
synchronized (PROPERTY_LISTENER) {
currentProperty.removeListener(PROPERTY_LISTENER);
property.addListener(PROPERTY_LISTENER);
@ -77,11 +78,11 @@ public final class ParamFlowRuleManager {
}
public static List<ParamFlowRule> getRulesOfResource(String resourceName) {
return paramFlowRules.get(resourceName);
return new ArrayList<>(paramFlowRules.get(resourceName));
}
public static boolean hasRules(String resourceName) {
List<ParamFlowRule> rules = paramFlowRules.get(resourceName);
Set<ParamFlowRule> rules = paramFlowRules.get(resourceName);
return rules != null && !rules.isEmpty();
}
@ -91,8 +92,8 @@ public final class ParamFlowRuleManager {
* @return a new copy of the rules.
*/
public static List<ParamFlowRule> getRules() {
List<ParamFlowRule> rules = new ArrayList<ParamFlowRule>();
for (Map.Entry<String, List<ParamFlowRule>> entry : paramFlowRules.entrySet()) {
List<ParamFlowRule> rules = new ArrayList<>();
for (Map.Entry<String, Set<ParamFlowRule>> entry : paramFlowRules.entrySet()) {
rules.addAll(entry.getValue());
}
return rules;
@ -102,7 +103,7 @@ public final class ParamFlowRuleManager {
@Override
public void configUpdate(List<ParamFlowRule> list) {
Map<String, List<ParamFlowRule>> rules = aggregateHotParamRules(list);
Map<String, Set<ParamFlowRule>> rules = aggregateHotParamRules(list);
if (rules != null) {
paramFlowRules.clear();
paramFlowRules.putAll(rules);
@ -112,7 +113,7 @@ public final class ParamFlowRuleManager {
@Override
public void configLoad(List<ParamFlowRule> list) {
Map<String, List<ParamFlowRule>> rules = aggregateHotParamRules(list);
Map<String, Set<ParamFlowRule>> rules = aggregateHotParamRules(list);
if (rules != null) {
paramFlowRules.clear();
paramFlowRules.putAll(rules);
@ -120,8 +121,8 @@ public final class ParamFlowRuleManager {
RecordLog.info("[ParamFlowRuleManager] Hot spot parameter flow rules received: " + paramFlowRules);
}
private Map<String, List<ParamFlowRule>> aggregateHotParamRules(List<ParamFlowRule> list) {
Map<String, List<ParamFlowRule>> newRuleMap = new ConcurrentHashMap<String, List<ParamFlowRule>>();
private Map<String, Set<ParamFlowRule>> aggregateHotParamRules(List<ParamFlowRule> list) {
Map<String, Set<ParamFlowRule>> newRuleMap = new ConcurrentHashMap<>();
if (list == null || list.isEmpty()) {
// No parameter flow rules, so clear all the metrics.
@ -143,12 +144,12 @@ public final class ParamFlowRuleManager {
ParamFlowRuleUtil.fillExceptionFlowItems(rule);
String resourceName = rule.getResource();
List<ParamFlowRule> ruleList = newRuleMap.get(resourceName);
if (ruleList == null) {
ruleList = new ArrayList<ParamFlowRule>();
newRuleMap.put(resourceName, ruleList);
Set<ParamFlowRule> ruleSet = newRuleMap.get(resourceName);
if (ruleSet == null) {
ruleSet = new HashSet<>();
newRuleMap.put(resourceName, ruleSet);
}
ruleList.add(rule);
ruleSet.add(rule);
}
// Clear unused hot param metrics.