crabapples
/
Sentinel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve SystemRuleManager and refactor checking logic of maxCpuUsage strategy (#1050) 

- The BBR period for maxCpuUsage strategy is deprecated. Now Sentinel will just check the CPU usage value, which could avert surge of CPU usage quickly
- Improve SystemRuleManager and SystemStatusListener
- Add some test cases

Signed-off-by: Eric Zhao <sczyh16@gmail.com>
This commit is contained in:
Eric Zhao 2019-09-19 16:36:12 +08:00 committed by GitHub
parent 9ff6e47bb0
commit 8b40981fca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 150 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/system/SystemRuleManager.java
View File

@ -63,7 +63,7 @@ import com.alibaba.csp.sentinel.slots.block.BlockException;
* @author jialiang.linjl
* @author leyou
*/
public class SystemRuleManager {
public final class SystemRuleManager {
private static volatile double highestSystemLoad = Double.MAX_VALUE;
/**
@ -95,7 +95,7 @@ public class SystemRuleManager {
static {
checkSystemStatus.set(false);
statusListener = new SystemStatusListener();
scheduler.scheduleAtFixedRate(statusListener, 5, 1, TimeUnit.SECONDS);
scheduler.scheduleAtFixedRate(statusListener, 0, 1, TimeUnit.SECONDS);
currentProperty.addListener(listener);
}
@ -107,6 +107,7 @@ public class SystemRuleManager {
*/
public static void register2Property(SentinelProperty<List<SystemRule>> property) {
synchronized (listener) {
RecordLog.info("[SystemRuleManager] Registering new property to system rule manager");
currentProperty.removeListener(listener);
property.addListener(listener);
currentProperty = property;
@ -167,26 +168,22 @@ public class SystemRuleManager {
return result;
}
public static double getQps() {
public static double getInboundQpsThreshold() {
return qps;
}
public static void setQps(double qps) {
SystemRuleManager.qps = qps;
}
public static long getMaxRt() {
public static long getRtThreshold() {
return maxRt;
}
public static long getMaxThread() {
public static long getMaxThreadThreshold() {
return maxThread;
}
static class SystemPropertyListener extends SimplePropertyListener<List<SystemRule>> {
@Override
public void configUpdate(List<SystemRule> rules) {
public synchronized void configUpdate(List<SystemRule> rules) {
restoreSetting();
// systemRules = rules;
if (rules != null && rules.size() >= 1) {
@ -234,14 +231,10 @@ public class SystemRuleManager {
return checkSystemStatus.get();
}
public static double getHighestSystemLoad() {
public static double getSystemLoadThreshold() {
return highestSystemLoad;
}
public static void setHighestSystemLoad(double highestSystemLoad) {
SystemRuleManager.highestSystemLoad = highestSystemLoad;
}
public static double getCpuUsageThreshold() {
return highestCpuUsage;
}
@ -257,9 +250,14 @@ public class SystemRuleManager {
}
if (rule.getHighestCpuUsage() >= 0) {
highestCpuUsage = Math.min(highestCpuUsage, rule.getHighestCpuUsage());
highestCpuUsageIsSet = true;
checkStatus = true;
if (rule.getHighestCpuUsage() > 1) {
RecordLog.warn(String.format("[SystemRuleManager] Ignoring invalid SystemRule: "
+ "highestCpuUsage %.3f > 1", rule.getHighestCpuUsage()));
} else {
highestCpuUsage = Math.min(highestCpuUsage, rule.getHighestCpuUsage());
highestCpuUsageIsSet = true;
checkStatus = true;
}
}
if (rule.getAvgRt() >= 0) {
@ -290,6 +288,9 @@ public class SystemRuleManager {
* @throws BlockException when any system rule's threshold is exceeded.
*/
public static void checkSystem(ResourceWrapper resourceWrapper) throws BlockException {
if (resourceWrapper == null) {
return;
}
// Ensure the checking switch is on.
if (!checkSystemStatus.get()) {
return;
@ -326,9 +327,7 @@ public class SystemRuleManager {
// cpu usage
if (highestCpuUsageIsSet && getCurrentCpuUsage() > highestCpuUsage) {
if (!checkBbr(currentThread)) {
throw new SystemBlockException(resourceWrapper.getName(), "cpu");
}
throw new SystemBlockException(resourceWrapper.getName(), "cpu");
}
}

33
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/system/SystemStatusListener.java
View File

@ -46,32 +46,35 @@ public class SystemStatusListener implements Runnable {
try {
OperatingSystemMXBean osBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class);
currentLoad = osBean.getSystemLoadAverage();
/**
/*
* Java Doc copied from {@link OperatingSystemMXBean#getSystemCpuLoad()}:</br>
* Returns the "recent cpu usage" for the whole system. This value is a double in the [0.0,1.0] interval.
* A value of 0.0 means that all CPUs were idle during the recent period of time observed, while a value
* of 1.0 means that all CPUs were actively running 100% of the time during the recent period being
* observed. All values betweens 0.0 and 1.0 are possible depending of the activities going on in the
* observed. All values between 0.0 and 1.0 are possible depending of the activities going on in the
* system. If the system recent cpu usage is not available, the method returns a negative value.
*/
currentCpuUsage = osBean.getSystemCpuLoad();
StringBuilder sb = new StringBuilder();
if (currentLoad > SystemRuleManager.getHighestSystemLoad()) {
sb.append("load:").append(currentLoad).append(";");
sb.append("cpu:").append(currentCpuUsage).append(";");
sb.append("qps:").append(Constants.ENTRY_NODE.passQps()).append(";");
sb.append("rt:").append(Constants.ENTRY_NODE.avgRt()).append(";");
sb.append("thread:").append(Constants.ENTRY_NODE.curThreadNum()).append(";");
sb.append("success:").append(Constants.ENTRY_NODE.successQps()).append(";");
sb.append("minRt:").append(Constants.ENTRY_NODE.minRt()).append(";");
sb.append("maxSuccess:").append(Constants.ENTRY_NODE.maxSuccessQps()).append(";");
RecordLog.info(sb.toString());
if (currentLoad > SystemRuleManager.getSystemLoadThreshold()) {
writeSystemStatusLog();
}
} catch (Throwable e) {
RecordLog.info("could not get system error ", e);
RecordLog.warn("[SystemStatusListener] Failed to get system metrics from JMX", e);
}
}
private void writeSystemStatusLog() {
StringBuilder sb = new StringBuilder();
sb.append("Load exceeds the threshold: ");
sb.append("load:").append(String.format("%.4f", currentLoad)).append("; ");
sb.append("cpuUsage:").append(String.format("%.4f", currentCpuUsage)).append("; ");
sb.append("qps:").append(String.format("%.4f", Constants.ENTRY_NODE.passQps())).append("; ");
sb.append("rt:").append(String.format("%.4f", Constants.ENTRY_NODE.avgRt())).append("; ");
sb.append("thread:").append(Constants.ENTRY_NODE.curThreadNum()).append("; ");
sb.append("success:").append(String.format("%.4f", Constants.ENTRY_NODE.successQps())).append("; ");
sb.append("minRt:").append(String.format("%.2f", Constants.ENTRY_NODE.minRt())).append("; ");
sb.append("maxSuccess:").append(String.format("%.2f", Constants.ENTRY_NODE.maxSuccessQps())).append("; ");
RecordLog.info(sb.toString());
}
}

111
sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/system/SystemRuleManagerTest.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Copyright 1999-2018 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.csp.sentinel.slots.system;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import com.alibaba.csp.sentinel.EntryType;
import com.alibaba.csp.sentinel.slotchain.StringResourceWrapper;
import com.alibaba.csp.sentinel.slots.block.BlockException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
/**
* @author Eric Zhao
*/
public class SystemRuleManagerTest {
@Test
public void testLoadInvalidRules() {
SystemRule rule1 = new SystemRule();
rule1.setHighestSystemLoad(-0.9d);
SystemRule rule2 = new SystemRule();
rule2.setHighestCpuUsage(2.7d);
SystemRuleManager.loadRules(Arrays.asList(rule1, rule2));
assertEquals(0, SystemRuleManager.getRules().size());
}
@Test
public void testLoadAndGetRules() {
SystemRule rule1 = new SystemRule();
rule1.setHighestSystemLoad(1.2d);
SystemRule rule2 = new SystemRule();
rule2.setMaxThread(17);
SystemRule rule3 = new SystemRule();
rule3.setHighestCpuUsage(0.7d);
SystemRule rule4 = new SystemRule();
rule4.setQps(1500);
SystemRule rule5 = new SystemRule();
rule5.setAvgRt(50);
SystemRuleManager.loadRules(Arrays.asList(rule1, rule2, rule3, rule4, rule5));
assertEquals(1.2d, SystemRuleManager.getSystemLoadThreshold(), 0.01);
assertEquals(17, SystemRuleManager.getMaxThreadThreshold());
assertEquals(0.7d, SystemRuleManager.getCpuUsageThreshold(), 0.01);
assertEquals(1500, SystemRuleManager.getInboundQpsThreshold(), 0.01);
assertEquals(50, SystemRuleManager.getRtThreshold());
}
@Test
public void testLoadDuplicateTypeOfRules() {
SystemRule rule1 = new SystemRule();
rule1.setHighestSystemLoad(1.2d);
SystemRule rule2 = new SystemRule();
rule2.setHighestSystemLoad(2.3d);
SystemRule rule3 = new SystemRule();
rule3.setHighestSystemLoad(3.4d);
SystemRuleManager.loadRules(Arrays.asList(rule1, rule2, rule3));
List<SystemRule> rules = SystemRuleManager.getRules();
assertEquals(1, rules.size());
assertEquals(1.2d, rules.get(0).getHighestSystemLoad(), 0.01);
assertEquals(1.2d, SystemRuleManager.getSystemLoadThreshold(), 0.01);
}
@Test
public void testCheckMaxCpuUsageNotBBR() throws Exception {
SystemRule rule1 = new SystemRule();
rule1.setHighestCpuUsage(0d);
SystemRuleManager.loadRules(Collections.singletonList(rule1));
// Wait until SystemStatusListener triggered the first CPU usage collecting.
Thread.sleep(1500);
boolean blocked = false;
try {
SystemRuleManager.checkSystem(new StringResourceWrapper("testCheckMaxCpuUsageNotBBR", EntryType.IN));
} catch (BlockException ex) {
blocked = true;
}
assertTrue("The entry should be blocked under SystemRule maxCpuUsage=0", blocked);
}
@Before
public void setUp() throws Exception {
SystemRuleManager.loadRules(new ArrayList<SystemRule>());
}
@After
public void tearDown() throws Exception {
SystemRuleManager.loadRules(new ArrayList<SystemRule>());
}
}