crabapples
/
Sentinel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add MetricsRepository interface and reuse original in-memory implementation (#126) 

- Abstract a universal `MetricsRepository` interface so that users can implement their own metrics persistence.
- Reuse original in-memory implementation (`InMemoryMetricsRepository`) as the default repository.

Signed-off-by: Eric Zhao <sczyh16@gmail.com>
This commit is contained in:
Eric Zhao 2018-09-06 18:59:31 +08:00 committed by GitHub
parent ed30f85f9c
commit 02cf5e5639
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 125 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/metric/MetricFetcher.java
View File

@ -40,7 +40,7 @@ import com.alibaba.csp.sentinel.util.StringUtil;
import com.taobao.csp.sentinel.dashboard.datasource.entity.MetricEntity;
import com.taobao.csp.sentinel.dashboard.discovery.AppManagement;
import com.taobao.csp.sentinel.dashboard.discovery.MachineInfo;
import com.taobao.csp.sentinel.dashboard.inmem.InMemMetricStore;
import com.taobao.csp.sentinel.dashboard.repository.metric.MetricsRepository;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.concurrent.FutureCallback;
@ -78,7 +78,7 @@ public class MetricFetcher {
private Map<String, AtomicLong> appLastFetchTime = new ConcurrentHashMap<>();
@Autowired
private InMemMetricStore metricStore;
private MetricsRepository<MetricEntity> metricStore;
@Autowired
private AppManagement appManagement;

104
sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/inmem/InMemMetricStore.java → sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/repository/metric/InMemoryMetricsRepository.java Executable file → Normal file
View File

@ -13,66 +13,77 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.taobao.csp.sentinel.dashboard.inmem;
package com.taobao.csp.sentinel.dashboard.repository.metric;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import com.alibaba.csp.sentinel.util.StringUtil;
import com.taobao.csp.sentinel.dashboard.datasource.entity.MetricEntity;
import org.springframework.stereotype.Component;
/**
* Store metrics in memory.
* Caches metrics data in a period of time in memory.
*
* @author leyou
* @author Carpenter Lee
* @author Eric Zhao
*/
@Component
public class InMemMetricStore {
public static final long MAX_METRIC_LIVE_TIME_MS = 1000 * 60 * 5;
public class InMemoryMetricsRepository implements MetricsRepository<MetricEntity> {
private static final long MAX_METRIC_LIVE_TIME_MS = 1000 * 60 * 5;
/**
* {@code app -> resource -> timestamp -> metric}
*/
private Map<String, Map<String, LinkedHashMap<Long, MetricEntity>>> allMetrics = new ConcurrentHashMap<>();
/**
* Save all metrics in memory. Metric older than {@link #MAX_METRIC_LIVE_TIME_MS} will be removed.
*
* @param metrics metrics to be saved.
*/
@Override
public synchronized void save(MetricEntity entity) {
if (entity == null || StringUtil.isBlank(entity.getApp())) {
return;
}
allMetrics.computeIfAbsent(entity.getApp(), e -> new HashMap<>(16))
.computeIfAbsent(entity.getResource(), e -> new LinkedHashMap<Long, MetricEntity>() {
@Override
protected boolean removeEldestEntry(Entry<Long, MetricEntity> eldest) {
// Metric older than {@link #MAX_METRIC_LIVE_TIME_MS} will be removed.
return eldest.getKey() < System.currentTimeMillis() - MAX_METRIC_LIVE_TIME_MS;
}
}).put(entity.getTimestamp().getTime(), entity);
}
@Override
public synchronized void saveAll(Iterable<MetricEntity> metrics) {
if (metrics == null) {
return;
}
for (MetricEntity entity : metrics) {
allMetrics.computeIfAbsent(entity.getApp(), e -> new HashMap<>(16))
.computeIfAbsent(entity.getResource(), e -> new LinkedHashMap<Long, MetricEntity>() {
@Override
protected boolean removeEldestEntry(Map.Entry<Long, MetricEntity> eldest) {
return eldest.getKey() < System.currentTimeMillis() - MAX_METRIC_LIVE_TIME_MS;
}
}).put(entity.getTimestamp().getTime(), entity);
}
metrics.forEach(this::save);
}
public synchronized List<MetricEntity> queryByAppAndResouce(String app,
String resource,
long startTime,
long endTime) {
@Override
public synchronized List<MetricEntity> queryByAppAndResourceBetween(String app, String resource,
long startTime, long endTime) {
List<MetricEntity> results = new ArrayList<>();
Map<String, LinkedHashMap<Long, MetricEntity>> resouceMap = allMetrics.get(app);
if (resouceMap == null) {
if (StringUtil.isBlank(app)) {
return results;
}
LinkedHashMap<Long, MetricEntity> metricsMap = resouceMap.get(resource);
Map<String, LinkedHashMap<Long, MetricEntity>> resourceMap = allMetrics.get(app);
if (resourceMap == null) {
return results;
}
LinkedHashMap<Long, MetricEntity> metricsMap = resourceMap.get(resource);
if (metricsMap == null) {
return results;
}
for (Map.Entry<Long, MetricEntity> entry : metricsMap.entrySet()) {
for (Entry<Long, MetricEntity> entry : metricsMap.entrySet()) {
if (entry.getKey() >= startTime && entry.getKey() <= endTime) {
results.add(entry.getValue());
}
@ -80,14 +91,12 @@ public class InMemMetricStore {
return results;
}
/**
* Find resources of App order by last minute b_qps desc
*
* @param app app name
* @return Resources list, order by last minute b_qps desc.
*/
public synchronized List<String> findResourcesOfApp(String app) {
@Override
public List<String> listResourcesOfApp(String app) {
List<String> results = new ArrayList<>();
if (StringUtil.isBlank(app)) {
return results;
}
// resource -> timestamp -> metric
Map<String, LinkedHashMap<Long, MetricEntity>> resourceMap = allMetrics.get(app);
if (resourceMap == null) {
@ -96,8 +105,8 @@ public class InMemMetricStore {
final long minTimeMs = System.currentTimeMillis() - 1000 * 60;
Map<String, MetricEntity> resourceCount = new HashMap<>(32);
for (Map.Entry<String, LinkedHashMap<Long, MetricEntity>> resourceMetrics : resourceMap.entrySet()) {
for (Map.Entry<Long, MetricEntity> metrics : resourceMetrics.getValue().entrySet()) {
for (Entry<String, LinkedHashMap<Long, MetricEntity>> resourceMetrics : resourceMap.entrySet()) {
for (Entry<Long, MetricEntity> metrics : resourceMetrics.getValue().entrySet()) {
if (metrics.getKey() < minTimeMs) {
continue;
}
@ -114,16 +123,19 @@ public class InMemMetricStore {
}
}
}
return resourceCount.entrySet().stream().sorted((o1, o2) -> {
MetricEntity e1 = o1.getValue();
MetricEntity e2 = o2.getValue();
int t = e2.getBlockedQps().compareTo(e1.getBlockedQps());
if (t != 0) {
return t;
}
return e2.getPassedQps().compareTo(e1.getPassedQps());
}).map(e -> e.getKey())
// Order by last minute b_qps DESC.
return resourceCount.entrySet()
.stream()
.sorted((o1, o2) -> {
MetricEntity e1 = o1.getValue();
MetricEntity e2 = o2.getValue();
int t = e2.getBlockedQps().compareTo(e1.getBlockedQps());
if (t != 0) {
return t;
}
return e2.getPassedQps().compareTo(e1.getPassedQps());
})
.map(Entry::getKey)
.collect(Collectors.toList());
}
}

60
sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/repository/metric/MetricsRepository.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright 1999-2018 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.taobao.csp.sentinel.dashboard.repository.metric;
import java.util.List;
/**
* Repository interface for aggregated metrics data.
*
* @param <T> type of metrics
* @author Eric Zhao
*/
public interface MetricsRepository<T> {
/**
* Save the metric to the storage repository.
*
* @param metric metric data to save
*/
void save(T metric);
/**
* Save all metrics to the storage repository.
*
* @param metrics metrics to save
*/
void saveAll(Iterable<T> metrics);
/**
* Get all metrics by {@code appName} and {@code resourceName} between a period of time.
*
* @param app application name for Sentinel
* @param resource resource name
* @param startTime start timestamp
* @param endTime end timestamp
* @return all metrics in query conditions
*/
List<T> queryByAppAndResourceBetween(String app, String resource, long startTime, long endTime);
/**
* List resource name of provided application name.
*
* @param app application name
* @return list of resources
*/
List<String> listResourcesOfApp(String app);
}

10
sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/view/MetricController.java
View File

@ -24,6 +24,7 @@ import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import com.taobao.csp.sentinel.dashboard.repository.metric.MetricsRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -35,7 +36,6 @@ import org.springframework.web.bind.annotation.ResponseBody;
import com.alibaba.csp.sentinel.util.StringUtil;
import com.taobao.csp.sentinel.dashboard.datasource.entity.MetricEntity;
import com.taobao.csp.sentinel.dashboard.inmem.InMemMetricStore;
import com.taobao.csp.sentinel.dashboard.view.vo.MetricVo;
/**
@ -50,7 +50,7 @@ public class MetricController {
private static final long maxQueryIntervalMs = 1000 * 60 * 60;
@Autowired
private InMemMetricStore metricStore;
private MetricsRepository<MetricEntity> metricStore;
@ResponseBody
@RequestMapping("/queryTopResourceMetric.json")
@ -83,7 +83,7 @@ public class MetricController {
if (endTime - startTime > maxQueryIntervalMs) {
return Result.ofFail(-1, "time intervalMs is too big, must <= 1h");
}
List<String> resources = metricStore.findResourcesOfApp(app);
List<String> resources = metricStore.listResourcesOfApp(app);
logger.info("queryTopResourceMetric(), resources.size()={}", resources.size());
if (resources == null || resources.isEmpty()) {
return Result.ofSuccess(null);
@ -110,7 +110,7 @@ public class MetricController {
logger.info("topResource={}", topResource);
long time = System.currentTimeMillis();
for (final String resource : topResource) {
List<MetricEntity> entities = metricStore.queryByAppAndResouce(
List<MetricEntity> entities = metricStore.queryByAppAndResourceBetween(
app, resource, startTime, endTime);
logger.info("resource={}, entities.size()={}", resource, entities == null ? "null" : entities.size());
List<MetricVo> vos = MetricVo.fromMetricEntities(entities, resource);
@ -151,7 +151,7 @@ public class MetricController {
if (endTime - startTime > maxQueryIntervalMs) {
return Result.ofFail(-1, "time intervalMs is too big, must <= 1h");
}
List<MetricEntity> entities = metricStore.queryByAppAndResouce(
List<MetricEntity> entities = metricStore.queryByAppAndResourceBetween(
app, identity, startTime, endTime);
List<MetricVo> vos = MetricVo.fromMetricEntities(entities, identity);
return Result.ofSuccess(sortMetricVoAndDistinct(vos));