metric: Add Prometheus exporter module for Sentinel metrics (#3173)

sentinel-extension/pom.xml

@@ -25,6 +25,7 @@
         <module>sentinel-datasource-eureka</module>
         <module>sentinel-annotation-cdi-interceptor</module>
         <module>sentinel-metric-exporter</module>
+        <module>sentinel-prometheus-metric-exporter</module>
     </modules>
 
 </project>

sentinel-extension/sentinel-prometheus-metric-exporter/README.md

@@ -0,0 +1,102 @@
+# Sentinel Prometheus Exporter
+
+Sentinel Prometheus Exporter is a module which provides the Sentinel metrics data for prometheus.
+
+You can integrate it into your Sentinel application, and then get the sentinel metrics in your prometheus.
+
+## How it works
+
+when the prometheus server collect the sentinel metrics,it get metrics from sentinel logs
+![image](https://github.com/alibaba/Sentinel/assets/71377602/2982209b-a3c7-403b-ae50-1dc7a17f90b7)
+
+## How to use
+
+To use Sentinel Prometheus Exporter, you should add the following dependency:
+
+### 1. add sentinel-prometheus-exporter
+
+```xml
+<dependency>
+    <groupId>com.alibaba.csp</groupId>
+    <artifactId>sentinel-prometheus-metric-exporter</artifactId>
+    <version>x.y.z</version>
+</dependency>
+```
+
+### 2. add prometheus dependency
+
+```xml
+<dependency>
+    <groupId>io.prometheus</groupId>
+    <artifactId>simpleclient</artifactId>
+    <version>0.3.0</version>
+</dependency>
+```
+
+```xml
+<dependency>
+    <groupId>io.prometheus</groupId>
+    <artifactId>simpleclient_httpserver</artifactId>
+    <version>0.3.0</version>
+</dependency>
+```
+
+### 3. set prometheus.yml with fetch config
+
+```yaml
+scrape_configs:
+- job_name: 'sentinelMetrics'
+  static_configs:
+    - targets: ['localhost:9092']
+```
+
+```yaml
+Note: the port needs to be the same as the value in the configuration (csp.sentinel.prometheus.fetch.port)
+```
+
+## Params for exporter
+
+you can set system params to control the exporter behavior
+
+### 1.csp.sentinel.prometheus.fetch.port
+
+the port for prometheus exporter,default 9092
+
+### 2.csp.sentinel.prometheus.fetch.size
+
+the max fetch nums for prometheus exporter,in case the memory is not enough,default 1024
+
+### 3.csp.sentinel.prometheus.fetch.delay
+
+the delay time for fetching , may be it is still do some statistics work according to the sliding window size when fetching,
+
+so need to set the delay time to insure the accuracy.
+
+unit: second
+
+default: 0
+
+### 4.csp.sentinel.prometheus.fetch.identify
+
+set the resource which need to fetch,default null,fetch all resources
+
+### 5.csp.sentinel.prometheus.fetch.types
+
+the types need to fetch,such as passQps,concurrency
+
+format: "xx|xx|xx"
+
+default: "passQps|blockQps|exceptionQps|rt|concurrency"
+
+you can reset the types as you need to,exm: "passQps|rt|concurrency|occupiedPassQps"
+
+the type is same as the MetricNode class variables, with range:
+{"passQps","blockQps","successQps","exceptionQps","rt","occupiedPassQps","concurrency"}
+
+### 6.csp.sentinel.prometheus.app
+
+set the appName when do PromSQL
+
+## how it looks
+
+![image](https://github.com/alibaba/Sentinel/assets/71377602/dedde134-53ed-4b4e-b184-98e55184aacf)

sentinel-extension/sentinel-prometheus-metric-exporter/pom.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.alibaba.csp</groupId>
+        <artifactId>sentinel-parent</artifactId>
+        <version>2.0.0-alpha</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>sentinel-prometheus-metric-exporter</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.csp</groupId>
+            <artifactId>sentinel-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient</artifactId>
+            <version>0.3.0</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_httpserver</artifactId>
+            <version>0.3.0</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>

sentinel-extension/sentinel-prometheus-metric-exporter/src/main/java/com/alibaba/csp/sentinel/metric/prom/MetricConstants.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright 1999-2019 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.metric.prom;
+
+/**
+ * The{@link PromExporterInit} the Collector for prometheus exporter.
+ *
+ * @author karl-sy
+ * @date 2023-08-08 09:30
+ * @since 2.0.0
+ */
+public final class MetricConstants {
+
+    public static final String METRIC_HELP = "sentinel_metrics";
+
+    public static final String RESOURCE = "resource";
+
+    public static final String CLASSIFICATION = "classification";
+
+    public static final String METRIC_TYPE = "type";
+
+    public static final String PASS_QPS = "passQps";
+
+    public static final String BLOCK_QPS = "blockQps";
+
+    public static final String SUCCESS_QPS = "successQps";
+
+    public static final String EXCEPTION_QPS = "exceptionQps";
+
+    public static final String RT = "rt";
+
+    public static final String OCC_PASS_QPS = "occupiedPassQps";
+
+    public static final String CONCURRENCY = "concurrency";
+
+    private MetricConstants() {
+    }
+}

sentinel-extension/sentinel-prometheus-metric-exporter/src/main/java/com/alibaba/csp/sentinel/metric/prom/PromExporterInit.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 1999-2021 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.metric.prom;
+
+import com.alibaba.csp.sentinel.init.InitFunc;
+import com.alibaba.csp.sentinel.log.RecordLog;
+import com.alibaba.csp.sentinel.metric.prom.collector.SentinelCollector;
+import com.alibaba.csp.sentinel.metric.prom.config.PrometheusGlobalConfig;
+import io.prometheus.client.exporter.HTTPServer;
+
+/**
+ * The{@link PromExporterInit} the InitFunc for prometheus exporter.
+ *
+ * @author karl-sy
+ * @date 2023-07-13 21:15
+ * @since 2.0.0
+ */
+public class PromExporterInit implements InitFunc {
+
+    @Override
+    public void init() throws Exception {
+        HTTPServer server = null;
+        try {
+            new SentinelCollector().register();
+            // 开启http服务供prometheus调用
+            // 默认只提供一个接口 http://ip:port/metrics，返回所有指标
+            int promPort = PrometheusGlobalConfig.getPromFetchPort();
+            server = new HTTPServer(promPort);
+        } catch (Throwable e) {
+            RecordLog.warn("[PromExporterInit] failed to init prometheus exporter with exception:", e);
+        }
+
+        HTTPServer finalServer = server;
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            if (finalServer != null) {
+                finalServer.stop();
+            }
+        }));
+    }
+
+}

sentinel-extension/sentinel-prometheus-metric-exporter/src/main/java/com/alibaba/csp/sentinel/metric/prom/collector/SentinelCollector.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright 1999-2021 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.metric.prom.collector;
+
+import com.alibaba.csp.sentinel.config.SentinelConfig;
+import com.alibaba.csp.sentinel.log.RecordLog;
+import com.alibaba.csp.sentinel.metric.prom.MetricConstants;
+import com.alibaba.csp.sentinel.metric.prom.config.PrometheusGlobalConfig;
+import com.alibaba.csp.sentinel.metric.prom.types.GaugeMetricFamily;
+import com.alibaba.csp.sentinel.node.metric.MetricNode;
+import com.alibaba.csp.sentinel.node.metric.MetricSearcher;
+import com.alibaba.csp.sentinel.node.metric.MetricWriter;
+import com.alibaba.csp.sentinel.util.PidUtil;
+import io.prometheus.client.Collector;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * The{@link PromExporterInit} Collector for prometheus exporter.
+ *
+ * @author karl-sy
+ * @date 2023-07-13 21:15
+ * @since 2.0.0
+ */
+public class SentinelCollector extends Collector {
+
+    private final Object lock = new Object();
+
+    private static final int ONE_SECOND = 1000;
+    private static final String appName = PrometheusGlobalConfig.getPromFetchApp();
+
+    private static final String[] types =  PrometheusGlobalConfig.getPromFetchTypes();
+
+    private static final String identify =  PrometheusGlobalConfig.getPromFetchIdentify();
+
+    private static final int fetchSize = PrometheusGlobalConfig.getPromFetchSize();
+
+    private static final int delayTime = PrometheusGlobalConfig.getPromFetchDelayTime();
+
+    private volatile MetricSearcher searcher;
+
+    private volatile Long lastFetchTime;
+
+    @Override
+    public List<MetricFamilySamples> collect() {
+        if (searcher == null) {
+            synchronized (lock) {
+                if (searcher == null) {
+                    searcher = new MetricSearcher(MetricWriter.METRIC_BASE_DIR,
+                            MetricWriter.formMetricFileName(SentinelConfig.getAppName(), PidUtil.getPid()));
+                }
+                RecordLog.warn("[SentinelCollector] init sentinel metrics searcher with appName:{}", appName);
+                lastFetchTime = System.currentTimeMillis() / ONE_SECOND * ONE_SECOND;
+            }
+        }
+
+        List<MetricFamilySamples> list = new ArrayList<>();
+
+        long endTime = System.currentTimeMillis() / ONE_SECOND * ONE_SECOND - (long) delayTime * ONE_SECOND;
+        try {
+            List<MetricNode> nodes = searcher.findByTimeAndResource(lastFetchTime, endTime, identify);
+            if(nodes == null){
+                return list;
+            }
+            if(nodes.size() > fetchSize){
+                nodes = nodes.subList(0,fetchSize);
+            }
+            GaugeMetricFamily metricFamily = new GaugeMetricFamily(appName, MetricConstants.METRIC_HELP,
+                    Arrays.asList(MetricConstants.RESOURCE, MetricConstants.CLASSIFICATION,
+                            MetricConstants.METRIC_TYPE));
+            for (MetricNode node : nodes) {
+                long recordTime = node.getTimestamp();
+                for (String type : types) {
+                    double val = getTypeVal(node,type);
+                    metricFamily.addMetric(Arrays.asList(node.getResource(), String.valueOf(node.getClassification()),type), val,recordTime);
+                }
+            }
+            list.add(metricFamily);
+        } catch (Exception e) {
+            RecordLog.warn("[SentinelCollector] failed to fetch sentinel metrics with exception:", e);
+        }finally {
+            lastFetchTime = endTime + ONE_SECOND;
+        }
+
+        return list;
+    }
+
+    public double getTypeVal(MetricNode node,String type){
+        if(MetricConstants.PASS_QPS.equals(type)){
+            return node.getPassQps();
+        }
+        if(MetricConstants.BLOCK_QPS.equals(type)){
+            return node.getBlockQps();
+        }
+        if(MetricConstants.SUCCESS_QPS.equals(type)){
+            return node.getSuccessQps();
+        }
+        if(MetricConstants.EXCEPTION_QPS.equals(type)){
+            return node.getExceptionQps();
+        }
+        if(MetricConstants.RT.equals(type)){
+            return node.getRt();
+        }
+        if(MetricConstants.OCC_PASS_QPS.equals(type)){
+            return node.getOccupiedPassQps();
+        }
+        if(MetricConstants.CONCURRENCY.equals(type)){
+            return node.getConcurrency();
+        }
+        return -1.0;
+    }
+}

sentinel-extension/sentinel-prometheus-metric-exporter/src/main/java/com/alibaba/csp/sentinel/metric/prom/config/PrometheusGlobalConfig.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright 1999-2021 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.metric.prom.config;
+
+import com.alibaba.csp.sentinel.config.SentinelConfig;
+import com.alibaba.csp.sentinel.util.StringUtil;
+
+/**
+ * The config for prometheus exporter.
+ *
+ * @author karl-sy
+ * @date 2023-07-13 21:15
+ * @since 2.0.0
+ */
+public class PrometheusGlobalConfig {
+
+    public static final String PROM_FETCH_PORT = "csp.sentinel.prometheus.fetch.port";
+    public static final String DEFAULT_PROM_FETCH_PORT = "9092";
+
+    public static final String PROM_FETCH_SIZE = "csp.sentinel.prometheus.fetch.size";
+    public static final String DEFAULT_PROM_FETCH_SIZE = "1024";
+
+    public static final String PROM_FETCH_DELAY = "csp.sentinel.prometheus.fetch.delay";
+    public static final String DEFAULT_PROM_FETCH_DELAY = "0";
+
+    public static final String PROM_FETCH_IDENTIFY = "csp.sentinel.prometheus.fetch.identify";
+
+    public static final String PROM_FETCH_TYPES = "csp.sentinel.prometheus.fetch.types";
+    public static final String DEFAULT_PROM_FETCH_TYPES = "passQps|blockQps|exceptionQps|rt|concurrency";
+
+    public static final String PROM_APP = "csp.sentinel.prometheus.app";
+    public static final String DEFAULT_PROM_APP = "SENTINEL_APP";
+
+    public static int getPromFetchPort() {
+        String config = SentinelConfig.getConfig(PROM_FETCH_PORT);
+        config = StringUtil.isNotBlank(config) ? config : DEFAULT_PROM_FETCH_PORT;
+        return Integer.parseInt(config);
+    }
+
+    public static int getPromFetchSize() {
+        String config = SentinelConfig.getConfig(PROM_FETCH_SIZE);
+        config = StringUtil.isNotBlank(config) ? config : DEFAULT_PROM_FETCH_SIZE;
+        return Integer.parseInt(config);
+    }
+
+    public static int getPromFetchDelayTime() {
+        String config = SentinelConfig.getConfig(PROM_FETCH_DELAY);
+        config = StringUtil.isNotBlank(config) ? config : DEFAULT_PROM_FETCH_DELAY;
+        return Integer.parseInt(config);
+    }
+
+    public static String getPromFetchIdentify() {
+        return SentinelConfig.getConfig(PROM_FETCH_IDENTIFY);
+    }
+
+    public static String[] getPromFetchTypes() {
+        String config = SentinelConfig.getConfig(PROM_FETCH_TYPES);
+        config = StringUtil.isNotBlank(config) ? config : DEFAULT_PROM_FETCH_TYPES;
+        try {
+            return config.split("\\|");
+        }catch (Throwable e){
+            return DEFAULT_PROM_FETCH_TYPES.split("\\|");
+        }
+    }
+
+    public static String getPromFetchApp() {
+        String appName = SentinelConfig.getConfig(PROM_APP);
+        if (appName == null) {
+            appName = SentinelConfig.getAppName();
+        }
+
+        if (appName == null) {
+            appName = DEFAULT_PROM_APP;
+        }
+        appName = appName.replaceAll("\\.","_");
+        appName = appName.replaceAll("-","_");
+        return appName;
+    }
+
+}

sentinel-extension/sentinel-prometheus-metric-exporter/src/main/java/com/alibaba/csp/sentinel/metric/prom/types/GaugeMetricFamily.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 1999-2021 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.metric.prom.types;
+
+
+import io.prometheus.client.Collector;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The{@link SentinelCollector} the MetricFamilySamples for prometheus exporter.
+ *
+ * @author karl-sy
+ * @date 2023-07-13 21:15
+ * @since 2.0.0
+ */
+public class GaugeMetricFamily extends Collector.MetricFamilySamples {
+
+    private final List<String> labelNames;
+
+    public GaugeMetricFamily(String name, String help, double value) {
+        super(name, Collector.Type.GAUGE, help, new ArrayList<Sample>());
+        labelNames = Collections.emptyList();
+        samples.add(new Sample(
+                name,
+                labelNames,
+                Collections.<String>emptyList(),
+                value));
+    }
+
+    public GaugeMetricFamily(String name, String help, List<String> labelNames) {
+        super(name, Collector.Type.GAUGE, help, new ArrayList<Sample>());
+        this.labelNames = labelNames;
+    }
+
+    public GaugeMetricFamily addMetric(List<String> labelValues, double value, long timestampMs) {
+        if (labelValues.size() != labelNames.size()) {
+            throw new IllegalArgumentException("Incorrect number of labels.");
+        }
+        samples.add(new Sample(name, labelNames, labelValues, value, timestampMs));
+        return this;
+    }
+}

sentinel-extension/sentinel-prometheus-metric-exporter/src/main/resources/META-INF/services/com.alibaba.csp.sentinel.init.InitFunc

@@ -0,0 +1 @@
+com.alibaba.csp.sentinel.metric.prom.PromExporterInit

sentinel-extension/sentinel-prometheus-metric-exporter/src/test/java/com/alibaba/csp/sentinel/metric/prom/collector/SentinelCollectorTest.java

@@ -0,0 +1,17 @@
+package com.alibaba.csp.sentinel.metric.prom.collector;
+
+import com.alibaba.csp.sentinel.node.metric.MetricNode;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SentinelCollectorTest {
+    @Test
+    public void testCollector(){
+        SentinelCollector collector = new SentinelCollector();
+
+        MetricNode node = new MetricNode();
+        node.setPassQps(10);
+        double val = collector.getTypeVal(node,"passQps");
+        Assert.assertEquals(val, 10,1e-4);
+    }
+}

sentinel-extension/sentinel-prometheus-metric-exporter/src/test/java/com/alibaba/csp/sentinel/metric/prom/types/GaugeMetricFamilyTest.java

@@ -0,0 +1,18 @@
+package com.alibaba.csp.sentinel.metric.prom.types;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+
+public class GaugeMetricFamilyTest {
+
+    @Test
+    public void testGaugeMetricFamily(){
+
+        GaugeMetricFamily metricFamily = new GaugeMetricFamily("appName",
+                "sentinel_metrics", Collections.singletonList("type"));
+        metricFamily.addMetric(Collections.singletonList("rt"), 1.0,System.currentTimeMillis());
+        Assert.assertEquals(metricFamily.samples.get(0).value, 1.0,1e-4);
+    }
+}