diff --git a/assets/grafana/rss-dashboard.json b/assets/grafana/rss-dashboard.json
index 73e560172cd..f5d14b0e748 100644
--- a/assets/grafana/rss-dashboard.json
+++ b/assets/grafana/rss-dashboard.json
@@ -1,12 +1,39 @@
{
+ "__inputs": [
+ {
+ "name": "DS_PROMETHEUS",
+ "label": "Prometheus",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__elements": [],
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "8.5.0"
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
"annotations": {
"list": [
{
"builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
@@ -17,14 +44,15 @@
"tags": [],
"type": "dashboard"
},
- "type": "dashboard"
+ "type": "dashboard",
+ "datasource": "${DS_PROMETHEUS}"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
- "id": 3,
+ "id": null,
"links": [],
"liveNow": false,
"panels": [
@@ -39,10 +67,6 @@
"id": 38,
"panels": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -81,8 +105,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -113,22 +136,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_WorkerCount_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_WorkerCount_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -167,8 +184,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -199,22 +215,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_OverloadWorkerCount_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_OverloadWorkerCount_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -253,8 +263,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -285,22 +294,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_BlacklistedWorkerCount_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_BlacklistedWorkerCount_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -339,8 +342,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -371,16 +373,14 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_RegisteredShuffleCount_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_RegisteredShuffleCount_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "Overall",
@@ -397,10 +397,6 @@
"id": 28,
"panels": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -439,8 +435,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -471,22 +466,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_TotalSlots_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_TotalSlots_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -525,8 +514,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -557,22 +545,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_SlotsUsed_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_SlotsUsed_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -611,8 +593,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -643,22 +624,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_SlotsAvailable_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_SlotsAvailable_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -697,8 +672,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -730,22 +704,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_MasterPushDataTime_Mean",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_MasterPushDataTime_Mean",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -784,8 +752,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -817,22 +784,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_MasterPushDataTime_Max",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_MasterPushDataTime_Max",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -871,8 +832,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -904,22 +864,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_SlavePushDataTime_Mean",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_SlavePushDataTime_Mean",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -958,8 +912,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -991,22 +944,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_SlavePushDataTime_Max",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_SlavePushDataTime_Max",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1045,8 +992,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1078,22 +1024,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_ReserveSlotsTime_Mean",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_ReserveSlotsTime_Mean",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1132,8 +1072,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1165,22 +1104,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_ReserveSlotsTime_Max",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_ReserveSlotsTime_Max",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1219,8 +1152,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1252,22 +1184,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_CommitFilesTime_Mean",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_CommitFilesTime_Mean",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1306,8 +1232,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1339,22 +1264,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_CommitFilesTime_Max",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_CommitFilesTime_Max",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1393,8 +1312,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1425,16 +1343,14 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_PushDataFailCount_Count",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_PushDataFailCount_Count",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "worker",
@@ -1451,10 +1367,6 @@
"id": 12,
"panels": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1493,8 +1405,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1526,22 +1437,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_FetchChunkTime_Mean",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_FetchChunkTime_Mean",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1580,8 +1485,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1613,22 +1517,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_FetchChunkTime_Max",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_FetchChunkTime_Max",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1667,8 +1565,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1700,22 +1597,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_OpenStreamTime_Mean",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_OpenStreamTime_Mean",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1754,8 +1645,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1787,16 +1677,14 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_OpenStreamTime_Max",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_OpenStreamTime_Max",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "FetchRelatives",
@@ -1813,10 +1701,6 @@
"id": 10,
"panels": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1855,8 +1739,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1888,22 +1771,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_ReserveSlotsTime_Mean",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_ReserveSlotsTime_Mean",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -1942,8 +1819,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1975,22 +1851,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_ReserveSlotsTime_Max",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_ReserveSlotsTime_Max",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2029,8 +1899,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2062,22 +1931,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_TakeBufferTime_Mean",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_TakeBufferTime_Mean",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2116,8 +1979,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2149,22 +2011,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_TakeBufferTime_Max",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_TakeBufferTime_Max",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2203,8 +2059,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2236,22 +2091,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_FlushDataTime_Mean",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_FlushDataTime_Mean",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2290,8 +2139,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2323,16 +2171,14 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_FlushDataTime_Max",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_FlushDataTime_Max",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "FlushDataRelatives",
@@ -2349,10 +2195,6 @@
"id": 8,
"panels": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2391,8 +2233,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2424,22 +2265,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_NettyMemory_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_NettyMemory_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2478,8 +2313,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2511,22 +2345,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_DiskBuffer_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_DiskBuffer_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2565,8 +2393,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2598,22 +2425,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_push_server_usedDirectMemory_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_push_server_usedDirectMemory_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2652,8 +2473,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2685,22 +2505,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_fetch_server_usedDirectMemory_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_fetch_server_usedDirectMemory_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2739,8 +2553,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2771,22 +2584,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_PausePushData_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_PausePushData_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2825,8 +2632,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2857,16 +2663,14 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_PausePushDataAndReplicate_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_PausePushDataAndReplicate_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "MemoryRelatives",
@@ -2883,10 +2687,6 @@
"id": 50,
"panels": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -2925,8 +2725,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2958,22 +2757,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_SortTime_Mean",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_SortTime_Mean",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -3012,8 +2805,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -3045,22 +2837,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_SortTime_Max",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_SortTime_Max",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -3099,8 +2885,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -3131,22 +2916,16 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_SortingFiles_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_SortingFiles_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
},
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"fieldConfig": {
"defaults": {
"color": {
@@ -3185,8 +2964,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -3218,16 +2996,14 @@
},
"targets": [
{
- "datasource": {
- "type": "prometheus",
- "uid": "z1lSI_Qnk"
- },
"expr": "metrics_SortMemory_Value",
- "refId": "A"
+ "refId": "A",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "metrics_SortMemory_Value",
- "type": "timeseries"
+ "type": "timeseries",
+ "datasource": "${DS_PROMETHEUS}"
}
],
"title": "SortRelatives",
@@ -3247,8 +3023,8 @@
},
"timepicker": {},
"timezone": "",
- "title": "RSS",
- "uid": "U_qgru_7z",
- "version": 20,
+ "title": "343RSS",
+ "uid": "U_qgru_7z1345342",
+ "version": 1,
"weekStart": ""
}
\ No newline at end of file
diff --git a/client-spark/shuffle-manager-2/pom.xml b/client-spark/shuffle-manager-2/pom.xml
index 62e106e192d..a7d33c12852 100644
--- a/client-spark/shuffle-manager-2/pom.xml
+++ b/client-spark/shuffle-manager-2/pom.xml
@@ -21,7 +21,7 @@
com.aliyun.emr
remote-shuffle-service
- 0.1.3
+ ${project.version}
../../pom.xml
@@ -132,6 +132,10 @@
io.netty
com.aliyun.emr.io.netty
+
+ org.apache.commons
+ org.apache.celeborn.org.apache.commons
+
@@ -144,6 +148,7 @@
com.google.protobuf:protobuf-java
com.google.guava:guava
io.netty:*
+ org.apache.commons:commons-lang3
diff --git a/client-spark/shuffle-manager-3/pom.xml b/client-spark/shuffle-manager-3/pom.xml
index 0fabc2a42ef..30bebe65376 100644
--- a/client-spark/shuffle-manager-3/pom.xml
+++ b/client-spark/shuffle-manager-3/pom.xml
@@ -21,7 +21,7 @@
com.aliyun.emr
remote-shuffle-service
- 0.1.3
+ ${project.version}
../../pom.xml
@@ -120,6 +120,10 @@
io.netty
com.aliyun.emr.io.netty
+
+ org.apache.commons
+ org.apache.celeborn.org.apache.commons
+
@@ -132,6 +136,7 @@
com.google.protobuf:protobuf-java
com.google.guava:guava
io.netty:*
+ org.apache.commons:commons-lang3
diff --git a/client-spark/shuffle-manager-common/pom.xml b/client-spark/shuffle-manager-common/pom.xml
index f1abb916b2e..991ce66455c 100644
--- a/client-spark/shuffle-manager-common/pom.xml
+++ b/client-spark/shuffle-manager-common/pom.xml
@@ -21,7 +21,7 @@
com.aliyun.emr
remote-shuffle-service
- 0.1.3
+ ${project.version}
../../pom.xml
diff --git a/client/pom.xml b/client/pom.xml
index 6de6a1cc8bd..0ed9a8141b4 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -23,7 +23,7 @@
com.aliyun.emr
remote-shuffle-service
- 0.1.3
+ ${project.version}
../pom.xml
diff --git a/client/src/main/java/com/aliyun/emr/rss/client/read/RssInputStream.java b/client/src/main/java/com/aliyun/emr/rss/client/read/RssInputStream.java
index 947d6ca7544..822fb63f2fd 100644
--- a/client/src/main/java/com/aliyun/emr/rss/client/read/RssInputStream.java
+++ b/client/src/main/java/com/aliyun/emr/rss/client/read/RssInputStream.java
@@ -33,6 +33,7 @@
import java.util.concurrent.atomic.LongAdder;
import io.netty.buffer.ByteBuf;
+import org.roaringbitmap.RoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -164,8 +165,12 @@ private boolean skipLocation(int startMapIndex, int endMapIndex, PartitionLocati
if (endMapIndex == Integer.MAX_VALUE) {
return false;
}
+ RoaringBitmap bitmap = location.getMapIdBitMap();
+ if (bitmap == null && location.getPeer() != null) {
+ bitmap = location.getPeer().getMapIdBitMap();
+ }
for (int i = startMapIndex; i < endMapIndex; i++) {
- if (location.getMapIdBitMap().contains(i)) {
+ if (bitmap.contains(i)) {
return false;
}
}
diff --git a/client/src/main/scala/com/aliyun/emr/rss/client/write/LifecycleManager.scala b/client/src/main/scala/com/aliyun/emr/rss/client/write/LifecycleManager.scala
index 45cb82295cc..42de53014ce 100644
--- a/client/src/main/scala/com/aliyun/emr/rss/client/write/LifecycleManager.scala
+++ b/client/src/main/scala/com/aliyun/emr/rss/client/write/LifecycleManager.scala
@@ -17,15 +17,16 @@
package com.aliyun.emr.rss.client.write
+import java.nio.ByteBuffer
import java.util
-import java.util.concurrent.{ConcurrentHashMap, ScheduledFuture, TimeUnit}
-import java.util.concurrent.atomic.LongAdder
+import java.util.concurrent.{Callable, ConcurrentHashMap, ScheduledFuture, TimeUnit}
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ListBuffer
import scala.util.Random
+import com.google.common.cache.{Cache, CacheBuilder}
import io.netty.util.internal.ConcurrentSet
import org.roaringbitmap.RoaringBitmap
@@ -38,7 +39,7 @@ import com.aliyun.emr.rss.common.protocol.RpcNameConstants.WORKER_EP
import com.aliyun.emr.rss.common.protocol.message.ControlMessages._
import com.aliyun.emr.rss.common.protocol.message.StatusCode
import com.aliyun.emr.rss.common.rpc._
-import com.aliyun.emr.rss.common.rpc.netty.{NettyRpcEndpointRef, NettyRpcEnv}
+import com.aliyun.emr.rss.common.rpc.netty.{LocalNettyRpcCallContext, NettyRpcEndpointRef, NettyRpcEnv, RemoteNettyRpcCallContext}
import com.aliyun.emr.rss.common.util.{ThreadUtils, Utils}
class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint with Logging {
@@ -52,6 +53,9 @@ class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint wit
private val splitMode = RssConf.partitionSplitMode(conf)
private val storageHint = RssConf.storageHint(conf)
private val rangeReadFilter = RssConf.rangeReadFilterEnabled(conf)
+ private val rpcCacheSize = RssConf.rpcCacheSize(conf)
+ private val rpcCacheConcurrentLevel = RssConf.rpcCacheConcurrentLevel(conf)
+ private val rpcCacheExpireMs = RssConf.rpcCacheExpireTimeMs(conf)
private val unregisterShuffleTime = new ConcurrentHashMap[Int, Long]()
@@ -67,6 +71,11 @@ class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint wit
// shuffle id -> (partitionId -> newest PartitionLocation)
private val latestPartitionLocation =
new ConcurrentHashMap[Int, ConcurrentHashMap[Int, PartitionLocation]]()
+ private val getReducerFileGroupRpcCache: Cache[Int, ByteBuffer] = CacheBuilder.newBuilder()
+ .concurrencyLevel(rpcCacheConcurrentLevel)
+ .expireAfterWrite(rpcCacheExpireMs, TimeUnit.MILLISECONDS)
+ .maximumSize(rpcCacheSize)
+ .build().asInstanceOf[Cache[Int, ByteBuffer]]
val newMapFunc =
new util.function.Function[Int, ConcurrentHashMap[Int, PartitionLocation]]() {
@@ -582,12 +591,26 @@ class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint wit
if (dataLostShuffleSet.contains(shuffleId)) {
context.reply(GetReducerFileGroupResponse(StatusCode.Failed, null, null))
} else {
- val shuffleFileGroup = reducerFileGroupsMap.get(shuffleId)
- context.reply(GetReducerFileGroupResponse(
- StatusCode.Success,
- shuffleFileGroup,
- shuffleMapperAttempts.get(shuffleId)
- ))
+ if (context.isInstanceOf[LocalNettyRpcCallContext]) {
+ // This branch is for the UTs
+ context.reply(GetReducerFileGroupResponse(
+ StatusCode.Success,
+ reducerFileGroupsMap.getOrDefault(shuffleId, Array.empty),
+ shuffleMapperAttempts.getOrDefault(shuffleId, Array.empty)))
+ } else {
+ val cachedMsg = getReducerFileGroupRpcCache.get(
+ shuffleId,
+ new Callable[ByteBuffer]() {
+ override def call(): ByteBuffer = {
+ val returnedMsg = GetReducerFileGroupResponse(
+ StatusCode.Success,
+ reducerFileGroupsMap.getOrDefault(shuffleId, Array.empty),
+ shuffleMapperAttempts.getOrDefault(shuffleId, Array.empty))
+ context.asInstanceOf[RemoteNettyRpcCallContext].nettyEnv.serialize(returnedMsg)
+ }
+ })
+ context.asInstanceOf[RemoteNettyRpcCallContext].callback.onSuccess(cachedMsg)
+ }
}
}
@@ -708,7 +731,6 @@ class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint wit
}
committedSlaveIds.asScala.foreach { id =>
val slavePartition = slavePartMap.get(id)
- slavePartition.setMapIdBitMap(committedMapIdBitmap.get(id))
val masterPartition = committedPartitions.get(id)
if (masterPartition ne null) {
masterPartition.setPeer(slavePartition)
@@ -716,6 +738,7 @@ class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint wit
} else {
logWarning(s"Shuffle $shuffleId partition $id: master lost, " +
s"use slave $slavePartition.")
+ slavePartition.setMapIdBitMap(committedMapIdBitmap.get(id))
committedPartitions.put(id, slavePartition)
}
}
diff --git a/common/pom.xml b/common/pom.xml
index cc78e6ae37a..01f7fe18d22 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
com.aliyun.emr
remote-shuffle-service
- 0.1.3
+ ${project.version}
../pom.xml
diff --git a/common/src/main/scala/com/aliyun/emr/rss/common/RssConf.scala b/common/src/main/scala/com/aliyun/emr/rss/common/RssConf.scala
index d494f8761f9..91209338b84 100644
--- a/common/src/main/scala/com/aliyun/emr/rss/common/RssConf.scala
+++ b/common/src/main/scala/com/aliyun/emr/rss/common/RssConf.scala
@@ -823,6 +823,18 @@ object RssConf extends Logging {
conf.getBoolean("rss.range.read.filter.enabled", false)
}
+ def rpcCacheSize(conf: RssConf): Int = {
+ conf.getInt("rss.rpc.cache.size", 256)
+ }
+
+ def rpcCacheConcurrentLevel(conf: RssConf): Int = {
+ conf.getInt("rss.rpc.cache.concurrent.level", 32)
+ }
+
+ def rpcCacheExpireTimeMs(conf: RssConf): Long = {
+ conf.getTimeAsMs("rss.rpc.cache.expire", "15s")
+ }
+
val WorkingDirName = "hadoop/rss-worker/shuffle_data"
// If we want to use multi-raft group we can
diff --git a/common/src/main/scala/com/aliyun/emr/rss/common/rpc/netty/NettyRpcCallContext.scala b/common/src/main/scala/com/aliyun/emr/rss/common/rpc/netty/NettyRpcCallContext.scala
index e84c57c1d15..eb4f1c651dd 100644
--- a/common/src/main/scala/com/aliyun/emr/rss/common/rpc/netty/NettyRpcCallContext.scala
+++ b/common/src/main/scala/com/aliyun/emr/rss/common/rpc/netty/NettyRpcCallContext.scala
@@ -55,8 +55,8 @@ private[rss] class LocalNettyRpcCallContext(
* A [[RpcCallContext]] that will call [[RpcResponseCallback]] to send the reply back.
*/
private[rss] class RemoteNettyRpcCallContext(
- nettyEnv: NettyRpcEnv,
- callback: RpcResponseCallback,
+ val nettyEnv: NettyRpcEnv,
+ val callback: RpcResponseCallback,
senderAddress: RpcAddress)
extends NettyRpcCallContext(senderAddress) {
diff --git a/common/src/main/scala/com/aliyun/emr/rss/common/util/Utils.scala b/common/src/main/scala/com/aliyun/emr/rss/common/util/Utils.scala
index 89a108d6c47..f1a6d173072 100644
--- a/common/src/main/scala/com/aliyun/emr/rss/common/util/Utils.scala
+++ b/common/src/main/scala/com/aliyun/emr/rss/common/util/Utils.scala
@@ -21,8 +21,8 @@ import java.io.{File, FileInputStream, InputStreamReader, IOException}
import java.lang.management.ManagementFactory
import java.math.{MathContext, RoundingMode}
import java.net._
-import java.nio.charset.StandardCharsets
import java.nio.ByteBuffer
+import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.{Locale, Properties, UUID}
import java.util
diff --git a/pom.xml b/pom.xml
index 88cb8eebe69..f43630c0dc2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,12 +23,13 @@
com.aliyun.emr
remote-shuffle-service
- 0.1.3
+ ${project.version}
pom
Aliyun E-MapReduce Shuffle Service Project Parent POM
+ 0.1.4
UTF-8
UTF-8
1.8
diff --git a/server-common/pom.xml b/server-common/pom.xml
index d57af6df476..1ef87a9cede 100644
--- a/server-common/pom.xml
+++ b/server-common/pom.xml
@@ -22,7 +22,7 @@
com.aliyun.emr
remote-shuffle-service
- 0.1.3
+ ${project.version}
../pom.xml
diff --git a/server-master/pom.xml b/server-master/pom.xml
index 96ec6ff222b..2b61d0c6781 100644
--- a/server-master/pom.xml
+++ b/server-master/pom.xml
@@ -5,7 +5,7 @@
remote-shuffle-service
com.aliyun.emr
- 0.1.3
+ ${project.version}
4.0.0
diff --git a/server-worker/pom.xml b/server-worker/pom.xml
index 1ffa9a51e3f..74347b1d9db 100644
--- a/server-worker/pom.xml
+++ b/server-worker/pom.xml
@@ -5,7 +5,7 @@
remote-shuffle-service
com.aliyun.emr
- 0.1.3
+ ${project.version}
4.0.0
diff --git a/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/Controller.scala b/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/Controller.scala
index a085f19a33f..dd519fee613 100644
--- a/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/Controller.scala
+++ b/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/Controller.scala
@@ -18,7 +18,7 @@
package com.aliyun.emr.rss.service.deploy.worker
import java.io.IOException
-import java.util.{ArrayList => jArrayList, List => jList, HashMap => jHashMap}
+import java.util.{ArrayList => jArrayList, HashMap => jHashMap, List => jList}
import java.util.concurrent._
import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference}
import java.util.function.BiFunction
diff --git a/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/LocalStorageManager.scala b/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/LocalStorageManager.scala
index 94604647971..038bdb81196 100644
--- a/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/LocalStorageManager.scala
+++ b/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/LocalStorageManager.scala
@@ -346,7 +346,7 @@ private[worker] final class LocalStorageManager(
@throws[IOException]
def createWriter(appId: String, shuffleId: Int, location: PartitionLocation,
- splitThreshold: Long, splitMode: PartitionSplitMode,rangeReadFilter: Boolean): FileWriter = {
+ splitThreshold: Long, splitMode: PartitionSplitMode, rangeReadFilter: Boolean): FileWriter = {
if (!hasAvailableWorkingDirs()) {
throw new IOException("No available working dirs!")
}