crabapples
/
Sentinel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for request item pattern matching in API gateway flow control (#842) 

- Update the `GatewayParamParser` to support matching the request item.
- Update the internal logic of converting gateway rules to parameter flow rules. The unified mismatched parameters (`SentinelGatewayConstants.GATEWAY_NOT_MATCH_PARAM`) will be configured as an exception item with a large threshold (indicating always pass).
- Add a `GatewayRegexCache` to cache the compiled regex for performance.
- Constant rename: separate URL matching strategy from parameter matching strategy

Signed-off-by: Eric Zhao <sczyh16@gmail.com>
This commit is contained in:
Eric Zhao 2019-06-18 14:08:54 +08:00 committed by GitHub
parent d59beaec66
commit 56e463e1fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 273 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
sentinel-adapter/sentinel-api-gateway-adapter-common/src/main/java/com/alibaba/csp/sentinel/adapter/gateway/common/SentinelGatewayConstants.java
View File

@ -32,15 +32,20 @@ public final class SentinelGatewayConstants {
public static final int PARAM_PARSE_STRATEGY_URL_PARAM = 3;
public static final int PARAM_PARSE_STRATEGY_COOKIE = 4;
public static final int URL_MATCH_STRATEGY_EXACT = 0;
public static final int URL_MATCH_STRATEGY_PREFIX = 1;
public static final int URL_MATCH_STRATEGY_REGEX = 2;
public static final int PARAM_MATCH_STRATEGY_EXACT = 0;
public static final int PARAM_MATCH_STRATEGY_PREFIX = 1;
public static final int PARAM_MATCH_STRATEGY_REGEX = 2;
public static final int PARAM_MATCH_STRATEGY_CONTAINS = 3;
public static final String GATEWAY_CONTEXT_DEFAULT = "sentinel_gateway_context_default";
public static final String GATEWAY_CONTEXT_PREFIX = "sentinel_gateway_context$$";
public static final String GATEWAY_CONTEXT_ROUTE_PREFIX = "sentinel_gateway_context$$route$$";
public static final String GATEWAY_NOT_MATCH_PARAM = "$$not_match";
public static final String GATEWAY_NOT_MATCH_PARAM = "$NM";
public static final String GATEWAY_DEFAULT_PARAM = "$D";
private SentinelGatewayConstants() {}

4
sentinel-adapter/sentinel-api-gateway-adapter-common/src/main/java/com/alibaba/csp/sentinel/adapter/gateway/common/api/ApiPathPredicateItem.java
View File

@ -17,6 +17,8 @@ package com.alibaba.csp.sentinel.adapter.gateway.common.api;
import java.util.Objects;
import com.alibaba.csp.sentinel.adapter.gateway.common.SentinelGatewayConstants;
/**
* @author Eric Zhao
* @since 1.6.0
@ -24,7 +26,7 @@ import java.util.Objects;
public class ApiPathPredicateItem implements ApiPredicateItem {
private String pattern;
private int matchStrategy;
private int matchStrategy = SentinelGatewayConstants.URL_MATCH_STRATEGY_EXACT;
public ApiPathPredicateItem setPattern(String pattern) {
this.pattern = pattern;

18
sentinel-adapter/sentinel-api-gateway-adapter-common/src/main/java/com/alibaba/csp/sentinel/adapter/gateway/common/param/GatewayParamParser.java
View File

@ -17,6 +17,7 @@ package com.alibaba.csp.sentinel.adapter.gateway.common.param;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Pattern;
import com.alibaba.csp.sentinel.adapter.gateway.common.SentinelGatewayConstants;
import com.alibaba.csp.sentinel.adapter.gateway.common.rule.GatewayFlowRule;
@ -153,13 +154,22 @@ public class GatewayParamParser<T> {
}
private String parseWithMatchStrategyInternal(int matchStrategy, String value, String pattern) {
// TODO: implement here.
if (value == null) {
return null;
}
if (matchStrategy == SentinelGatewayConstants.PARAM_MATCH_STRATEGY_REGEX) {
return value;
switch (matchStrategy) {
case SentinelGatewayConstants.PARAM_MATCH_STRATEGY_EXACT:
return value.equals(pattern) ? value : SentinelGatewayConstants.GATEWAY_NOT_MATCH_PARAM;
case SentinelGatewayConstants.PARAM_MATCH_STRATEGY_CONTAINS:
return value.contains(pattern) ? value : SentinelGatewayConstants.GATEWAY_NOT_MATCH_PARAM;
case SentinelGatewayConstants.PARAM_MATCH_STRATEGY_REGEX:
Pattern regex = GatewayRegexCache.getRegexPattern(pattern);
if (regex == null) {
return value;
}
return regex.matcher(value).matches() ? value : SentinelGatewayConstants.GATEWAY_NOT_MATCH_PARAM;
default:
return value;
}
return value;
}
}

58
sentinel-adapter/sentinel-api-gateway-adapter-common/src/main/java/com/alibaba/csp/sentinel/adapter/gateway/common/param/GatewayRegexCache.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright 1999-2019 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.csp.sentinel.adapter.gateway.common.param;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import com.alibaba.csp.sentinel.log.RecordLog;
/**
* @author Eric Zhao
* @since 1.6.2
*/
public final class GatewayRegexCache {
private static final Map<String, Pattern> REGEX_CACHE = new ConcurrentHashMap<>();
public static Pattern getRegexPattern(String pattern) {
if (pattern == null) {
return null;
}
return REGEX_CACHE.get(pattern);
}
public static boolean addRegexPattern(String pattern) {
if (pattern == null) {
return false;
}
try {
Pattern regex = Pattern.compile(pattern);
REGEX_CACHE.put(pattern, regex);
return true;
} catch (Exception ex) {
RecordLog.warn("[GatewayRegexCache] Failed to compile the regex: " + pattern, ex);
return false;
}
}
public static void clear() {
REGEX_CACHE.clear();
}
private GatewayRegexCache() {}
}

20
sentinel-adapter/sentinel-api-gateway-adapter-common/src/main/java/com/alibaba/csp/sentinel/adapter/gateway/common/rule/GatewayRuleConverter.java
View File

@ -15,7 +15,9 @@
*/
package com.alibaba.csp.sentinel.adapter.gateway.common.rule;
import com.alibaba.csp.sentinel.adapter.gateway.common.SentinelGatewayConstants;
import com.alibaba.csp.sentinel.slots.block.flow.FlowRule;
import com.alibaba.csp.sentinel.slots.block.flow.param.ParamFlowItem;
import com.alibaba.csp.sentinel.slots.block.flow.param.ParamFlowRule;
/**
@ -32,6 +34,18 @@ final class GatewayRuleConverter {
.setMaxQueueingTimeMs(rule.getMaxQueueingTimeoutMs());
}
static ParamFlowItem generateNonMatchPassParamItem() {
return new ParamFlowItem().setClassType(String.class.getName())
.setCount(1000_0000)
.setObject(SentinelGatewayConstants.GATEWAY_NOT_MATCH_PARAM);
}
static ParamFlowItem generateNonMatchBlockParamItem() {
return new ParamFlowItem().setClassType(String.class.getName())
.setCount(0)
.setObject(SentinelGatewayConstants.GATEWAY_NOT_MATCH_PARAM);
}
static ParamFlowRule applyNonParamToParamRule(/*@Valid*/ GatewayFlowRule gatewayRule, int idx) {
return new ParamFlowRule(gatewayRule.getResource())
.setCount(gatewayRule.getCount())
@ -63,7 +77,11 @@ final class GatewayRuleConverter {
GatewayParamFlowItem gatewayItem = gatewayRule.getParamItem();
// Apply the current idx to gateway rule item.
gatewayItem.setIndex(idx);
// TODO: implement for pattern-based parameters.
// Apply for pattern-based parameters.
String valuePattern = gatewayItem.getPattern();
if (valuePattern != null) {
paramRule.getParamFlowItemList().add(generateNonMatchPassParamItem());
}
return paramRule;
}

12
sentinel-adapter/sentinel-api-gateway-adapter-common/src/main/java/com/alibaba/csp/sentinel/adapter/gateway/common/rule/GatewayRuleManager.java
View File

@ -25,6 +25,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.alibaba.csp.sentinel.adapter.gateway.common.SentinelGatewayConstants;
import com.alibaba.csp.sentinel.adapter.gateway.common.param.GatewayRegexCache;
import com.alibaba.csp.sentinel.log.RecordLog;
import com.alibaba.csp.sentinel.property.DynamicSentinelProperty;
import com.alibaba.csp.sentinel.property.PropertyListener;
@ -132,6 +133,16 @@ public final class GatewayRuleManager {
return idxMap.get(resourceName);
}
private void cacheRegexPattern(/*@NonNull*/ GatewayParamFlowItem item) {
String pattern = item.getPattern();
if (StringUtil.isNotEmpty(pattern) &&
item.getMatchStrategy() == SentinelGatewayConstants.PARAM_MATCH_STRATEGY_REGEX) {
if (GatewayRegexCache.getRegexPattern(pattern) == null) {
GatewayRegexCache.addRegexPattern(pattern);
}
}
}
private synchronized void applyGatewayRuleInternal(Set<GatewayFlowRule> conf) {
if (conf == null || conf.isEmpty()) {
applyToConvertedParamMap(new HashSet<ParamFlowRule>());
@ -163,6 +174,7 @@ public final class GatewayRuleManager {
if (paramFlowRules.add(GatewayRuleConverter.applyToParamRule(rule, idx))) {
idxMap.put(rule.getResource(), idx + 1);
}
cacheRegexPattern(rule.getParamItem());
}
// Apply to the gateway rule map.
Set<GatewayFlowRule> ruleSet = gatewayRuleMap.get(resourceName);

92
sentinel-adapter/sentinel-api-gateway-adapter-common/src/test/java/com/alibaba/csp/sentinel/adapter/gateway/common/param/GatewayParamParserTest.java
View File

@ -186,6 +186,96 @@ public class GatewayParamParserTest {
assertThat(params[apiRule1.getParamItem().getIndex()]).isEqualTo(expectedUrlParamValue2);
}
@Test
public void testParseParametersWithItemPatternMatching() {
RequestItemParser<Object> itemParser = mock(RequestItemParser.class);
GatewayParamParser<Object> paramParser = new GatewayParamParser<>(itemParser);
// Create a fake request.
Object request = new Object();
// Prepare gateway rules.
Set<GatewayFlowRule> rules = new HashSet<>();
final String routeId1 = "my_test_route_F&@";
final String api1 = "my_test_route_E5K";
final String headerName = "X-Sentinel-Flag";
final String paramName = "p";
String nameEquals = "Wow";
String nameContains = "warn";
String valueRegex = "\\d+";
GatewayFlowRule routeRule1 = new GatewayFlowRule(routeId1)
.setCount(10)
.setIntervalSec(1)
.setControlBehavior(RuleConstant.CONTROL_BEHAVIOR_RATE_LIMITER)
.setMaxQueueingTimeoutMs(600)
.setParamItem(new GatewayParamFlowItem()
.setParseStrategy(SentinelGatewayConstants.PARAM_PARSE_STRATEGY_HEADER)
.setFieldName(headerName)
.setPattern(nameEquals)
.setMatchStrategy(SentinelGatewayConstants.PARAM_MATCH_STRATEGY_EXACT)
);
GatewayFlowRule routeRule2 = new GatewayFlowRule(routeId1)
.setCount(20)
.setIntervalSec(1)
.setBurst(5)
.setParamItem(new GatewayParamFlowItem()
.setParseStrategy(SentinelGatewayConstants.PARAM_PARSE_STRATEGY_URL_PARAM)
.setFieldName(paramName)
.setPattern(nameContains)
.setMatchStrategy(SentinelGatewayConstants.PARAM_MATCH_STRATEGY_CONTAINS)
);
GatewayFlowRule apiRule1 = new GatewayFlowRule(api1)
.setResourceMode(SentinelGatewayConstants.RESOURCE_MODE_CUSTOM_API_NAME)
.setCount(5)
.setIntervalSec(1)
.setParamItem(new GatewayParamFlowItem()
.setParseStrategy(SentinelGatewayConstants.PARAM_PARSE_STRATEGY_URL_PARAM)
.setFieldName(paramName)
.setPattern(valueRegex)
.setMatchStrategy(SentinelGatewayConstants.PARAM_MATCH_STRATEGY_REGEX)
);
rules.add(routeRule1);
rules.add(routeRule2);
rules.add(apiRule1);
GatewayRuleManager.loadRules(rules);
mockSingleHeader(itemParser, headerName, nameEquals);
mockSingleUrlParam(itemParser, paramName, nameContains);
Object[] params = paramParser.parseParameterFor(routeId1, request, routeIdPredicate);
assertThat(params.length).isEqualTo(2);
assertThat(params[routeRule1.getParamItem().getIndex()]).isEqualTo(nameEquals);
assertThat(params[routeRule2.getParamItem().getIndex()]).isEqualTo(nameContains);
mockSingleHeader(itemParser, headerName, nameEquals + "_foo");
mockSingleUrlParam(itemParser, paramName, nameContains + "_foo");
params = paramParser.parseParameterFor(routeId1, request, routeIdPredicate);
assertThat(params.length).isEqualTo(2);
assertThat(params[routeRule1.getParamItem().getIndex()])
.isEqualTo(SentinelGatewayConstants.GATEWAY_NOT_MATCH_PARAM);
assertThat(params[routeRule2.getParamItem().getIndex()])
.isEqualTo(nameContains + "_foo");
mockSingleHeader(itemParser, headerName, "foo");
mockSingleUrlParam(itemParser, paramName, "foo");
params = paramParser.parseParameterFor(routeId1, request, routeIdPredicate);
assertThat(params.length).isEqualTo(2);
assertThat(params[routeRule1.getParamItem().getIndex()])
.isEqualTo(SentinelGatewayConstants.GATEWAY_NOT_MATCH_PARAM);
assertThat(params[routeRule2.getParamItem().getIndex()])
.isEqualTo(SentinelGatewayConstants.GATEWAY_NOT_MATCH_PARAM);
mockSingleUrlParam(itemParser, paramName, "23");
params = paramParser.parseParameterFor(api1, request, apiNamePredicate);
assertThat(params.length).isEqualTo(1);
assertThat(params[apiRule1.getParamItem().getIndex()]).isEqualTo("23");
mockSingleUrlParam(itemParser, paramName, "some233");
params = paramParser.parseParameterFor(api1, request, apiNamePredicate);
assertThat(params.length).isEqualTo(1);
assertThat(params[apiRule1.getParamItem().getIndex()])
.isEqualTo(SentinelGatewayConstants.GATEWAY_NOT_MATCH_PARAM);
}
private void mockClientHostAddress(/*@Mock*/ RequestItemParser parser, String address) {
when(parser.getRemoteAddress(any())).thenReturn(address);
}
@ -218,11 +308,13 @@ public class GatewayParamParserTest {
public void setUp() {
GatewayApiDefinitionManager.loadApiDefinitions(new HashSet<ApiDefinition>());
GatewayRuleManager.loadRules(new HashSet<GatewayFlowRule>());
GatewayRegexCache.clear();
}
@After
public void tearDown() {
GatewayApiDefinitionManager.loadApiDefinitions(new HashSet<ApiDefinition>());
GatewayRuleManager.loadRules(new HashSet<GatewayFlowRule>());
GatewayRegexCache.clear();
}
}

49
sentinel-adapter/sentinel-api-gateway-adapter-common/src/test/java/com/alibaba/csp/sentinel/adapter/gateway/common/param/GatewayRegexCacheTest.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Copyright 1999-2019 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.csp.sentinel.adapter.gateway.common.param;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Eric Zhao
*/
public class GatewayRegexCacheTest {
@Before
public void setUp() {
GatewayRegexCache.clear();
}
@After
public void tearDown() {
GatewayRegexCache.clear();
}
@Test
public void testAddAndGetRegexPattern() {
// Test for invalid pattern.
assertThat(GatewayRegexCache.addRegexPattern("\\")).isFalse();
assertThat(GatewayRegexCache.addRegexPattern(null)).isFalse();
// Test for good pattern.
String goodPattern = "\\d+";
assertThat(GatewayRegexCache.addRegexPattern(goodPattern)).isTrue();
assertThat(GatewayRegexCache.getRegexPattern(goodPattern)).isNotNull();
}
}

4
sentinel-adapter/sentinel-spring-cloud-gateway-adapter/src/main/java/com/alibaba/csp/sentinel/adapter/gateway/sc/api/matcher/WebExchangeApiMatcher.java
View File

@ -59,9 +59,9 @@ public class WebExchangeApiMatcher extends AbstractApiMatcher<ServerWebExchange>
return Optional.empty();
}
switch (item.getMatchStrategy()) {
case SentinelGatewayConstants.PARAM_MATCH_STRATEGY_REGEX:
case SentinelGatewayConstants.URL_MATCH_STRATEGY_REGEX:
return Optional.of(RouteMatchers.regexPath(pattern));
case SentinelGatewayConstants.PARAM_MATCH_STRATEGY_PREFIX:
case SentinelGatewayConstants.URL_MATCH_STRATEGY_PREFIX:
return Optional.of(RouteMatchers.antPath(pattern));
default:
return Optional.of(RouteMatchers.exactPath(pattern));

4
sentinel-adapter/sentinel-spring-cloud-gateway-adapter/src/test/java/com/alibaba/csp/sentinel/adapter/gateway/sc/SentinelGatewayFilterTest.java
View File

@ -44,14 +44,14 @@ public class SentinelGatewayFilterTest {
ApiDefinition api1 = new ApiDefinition(apiName1)
.setPredicateItems(Collections.singleton(
new ApiPathPredicateItem().setPattern("/product/**")
.setMatchStrategy(SentinelGatewayConstants.PARAM_MATCH_STRATEGY_PREFIX)
.setMatchStrategy(SentinelGatewayConstants.URL_MATCH_STRATEGY_PREFIX)
));
String apiName2 = "another_customized_api";
ApiDefinition api2 = new ApiDefinition(apiName2)
.setPredicateItems(new HashSet<ApiPredicateItem>() {{
add(new ApiPathPredicateItem().setPattern("/something"));
add(new ApiPathPredicateItem().setPattern("/other/**")
.setMatchStrategy(SentinelGatewayConstants.PARAM_MATCH_STRATEGY_PREFIX));
.setMatchStrategy(SentinelGatewayConstants.URL_MATCH_STRATEGY_PREFIX));
}});
apiDefinitions.add(api1);
apiDefinitions.add(api2);

4
sentinel-adapter/sentinel-zuul-adapter/src/main/java/com/alibaba/csp/sentinel/adapter/gateway/zuul/api/matcher/RequestContextApiMatcher.java
View File

@ -61,9 +61,9 @@ public class RequestContextApiMatcher extends AbstractApiMatcher<RequestContext>
return null;
}
switch (item.getMatchStrategy()) {
case SentinelGatewayConstants.PARAM_MATCH_STRATEGY_REGEX:
case SentinelGatewayConstants.URL_MATCH_STRATEGY_REGEX:
return ZuulRouteMatchers.regexPath(pattern);
case SentinelGatewayConstants.PARAM_MATCH_STRATEGY_PREFIX:
case SentinelGatewayConstants.URL_MATCH_STRATEGY_PREFIX:
return ZuulRouteMatchers.antPath(pattern);
default:
return ZuulRouteMatchers.exactPath(pattern);

14
sentinel-demo/sentinel-demo-spring-cloud-gateway/src/main/java/com/alibaba/csp/sentinel/demo/spring/sc/gateway/GatewayConfiguration.java
View File

@ -83,12 +83,12 @@ public class GatewayConfiguration {
.setPredicateItems(new HashSet<ApiPredicateItem>() {{
add(new ApiPathPredicateItem().setPattern("/ahas"));
add(new ApiPathPredicateItem().setPattern("/product/**")
.setMatchStrategy(SentinelGatewayConstants.PARAM_MATCH_STRATEGY_PREFIX));
.setMatchStrategy(SentinelGatewayConstants.URL_MATCH_STRATEGY_PREFIX));
}});
ApiDefinition api2 = new ApiDefinition("another_customized_api")
.setPredicateItems(new HashSet<ApiPredicateItem>() {{
add(new ApiPathPredicateItem().setPattern("/**")
.setMatchStrategy(SentinelGatewayConstants.PARAM_MATCH_STRATEGY_PREFIX));
.setMatchStrategy(SentinelGatewayConstants.URL_MATCH_STRATEGY_PREFIX));
}});
definitions.add(api1);
definitions.add(api2);
@ -127,6 +127,16 @@ public class GatewayConfiguration {
.setFieldName("pa")
)
);
rules.add(new GatewayFlowRule("httpbin_route")
.setCount(2)
.setIntervalSec(30)
.setParamItem(new GatewayParamFlowItem()
.setParseStrategy(SentinelGatewayConstants.PARAM_PARSE_STRATEGY_URL_PARAM)
.setFieldName("type")
.setPattern("warn")
.setMatchStrategy(SentinelGatewayConstants.PARAM_MATCH_STRATEGY_CONTAINS)
)
);
rules.add(new GatewayFlowRule("some_customized_api")
.setResourceMode(SentinelGatewayConstants.RESOURCE_MODE_CUSTOM_API_NAME)

4
sentinel-demo/sentinel-demo-zuul-gateway/src/main/java/com/alibaba/csp/sentinel/demo/zuul/gateway/GatewayRuleConfig.java
View File

@ -52,12 +52,12 @@ public class GatewayRuleConfig {
.setPredicateItems(new HashSet<ApiPredicateItem>() {{
add(new ApiPathPredicateItem().setPattern("/ahas"));
add(new ApiPathPredicateItem().setPattern("/aliyun_product/**")
.setMatchStrategy(SentinelGatewayConstants.PARAM_MATCH_STRATEGY_PREFIX));
.setMatchStrategy(SentinelGatewayConstants.URL_MATCH_STRATEGY_PREFIX));
}});
ApiDefinition api2 = new ApiDefinition("another_customized_api")
.setPredicateItems(new HashSet<ApiPredicateItem>() {{
add(new ApiPathPredicateItem().setPattern("/**")
.setMatchStrategy(SentinelGatewayConstants.PARAM_MATCH_STRATEGY_PREFIX));
.setMatchStrategy(SentinelGatewayConstants.URL_MATCH_STRATEGY_PREFIX));
}});
definitions.add(api1);
definitions.add(api2);