diff --git a/assets/grafana/rss-dashboard.json b/assets/grafana/rss-dashboard.json index 73e560172cd..f5d14b0e748 100644 --- a/assets/grafana/rss-dashboard.json +++ b/assets/grafana/rss-dashboard.json @@ -1,12 +1,39 @@ { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "8.5.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -17,14 +44,15 @@ "tags": [], "type": "dashboard" }, - "type": "dashboard" + "type": "dashboard", + "datasource": "${DS_PROMETHEUS}" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 3, + "id": null, "links": [], "liveNow": false, "panels": [ @@ -39,10 +67,6 @@ "id": 38, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -81,8 +105,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -113,22 +136,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_WorkerCount_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_WorkerCount_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -167,8 +184,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -199,22 +215,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_OverloadWorkerCount_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_OverloadWorkerCount_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -253,8 +263,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -285,22 +294,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_BlacklistedWorkerCount_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_BlacklistedWorkerCount_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -339,8 +342,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -371,16 +373,14 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_RegisteredShuffleCount_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_RegisteredShuffleCount_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" } ], "title": "Overall", @@ -397,10 +397,6 @@ "id": 28, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -439,8 +435,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -471,22 +466,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_TotalSlots_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_TotalSlots_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -525,8 +514,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -557,22 +545,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_SlotsUsed_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_SlotsUsed_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -611,8 +593,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -643,22 +624,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_SlotsAvailable_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_SlotsAvailable_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -697,8 +672,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -730,22 +704,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_MasterPushDataTime_Mean", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_MasterPushDataTime_Mean", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -784,8 +752,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -817,22 +784,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_MasterPushDataTime_Max", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_MasterPushDataTime_Max", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -871,8 +832,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -904,22 +864,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_SlavePushDataTime_Mean", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_SlavePushDataTime_Mean", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -958,8 +912,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -991,22 +944,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_SlavePushDataTime_Max", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_SlavePushDataTime_Max", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1045,8 +992,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1078,22 +1024,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_ReserveSlotsTime_Mean", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_ReserveSlotsTime_Mean", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1132,8 +1072,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1165,22 +1104,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_ReserveSlotsTime_Max", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_ReserveSlotsTime_Max", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1219,8 +1152,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1252,22 +1184,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_CommitFilesTime_Mean", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_CommitFilesTime_Mean", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1306,8 +1232,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1339,22 +1264,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_CommitFilesTime_Max", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_CommitFilesTime_Max", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1393,8 +1312,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1425,16 +1343,14 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_PushDataFailCount_Count", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_PushDataFailCount_Count", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" } ], "title": "worker", @@ -1451,10 +1367,6 @@ "id": 12, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1493,8 +1405,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1526,22 +1437,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_FetchChunkTime_Mean", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_FetchChunkTime_Mean", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1580,8 +1485,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1613,22 +1517,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_FetchChunkTime_Max", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_FetchChunkTime_Max", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1667,8 +1565,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1700,22 +1597,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_OpenStreamTime_Mean", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_OpenStreamTime_Mean", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1754,8 +1645,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1787,16 +1677,14 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_OpenStreamTime_Max", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_OpenStreamTime_Max", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" } ], "title": "FetchRelatives", @@ -1813,10 +1701,6 @@ "id": 10, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1855,8 +1739,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1888,22 +1771,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_ReserveSlotsTime_Mean", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_ReserveSlotsTime_Mean", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -1942,8 +1819,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1975,22 +1851,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_ReserveSlotsTime_Max", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_ReserveSlotsTime_Max", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2029,8 +1899,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2062,22 +1931,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_TakeBufferTime_Mean", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_TakeBufferTime_Mean", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2116,8 +1979,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2149,22 +2011,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_TakeBufferTime_Max", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_TakeBufferTime_Max", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2203,8 +2059,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2236,22 +2091,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_FlushDataTime_Mean", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_FlushDataTime_Mean", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2290,8 +2139,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2323,16 +2171,14 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_FlushDataTime_Max", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_FlushDataTime_Max", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" } ], "title": "FlushDataRelatives", @@ -2349,10 +2195,6 @@ "id": 8, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2391,8 +2233,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2424,22 +2265,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_NettyMemory_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_NettyMemory_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2478,8 +2313,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2511,22 +2345,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_DiskBuffer_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_DiskBuffer_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2565,8 +2393,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2598,22 +2425,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_push_server_usedDirectMemory_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_push_server_usedDirectMemory_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2652,8 +2473,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2685,22 +2505,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_fetch_server_usedDirectMemory_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_fetch_server_usedDirectMemory_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2739,8 +2553,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2771,22 +2584,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_PausePushData_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_PausePushData_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2825,8 +2632,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2857,16 +2663,14 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_PausePushDataAndReplicate_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_PausePushDataAndReplicate_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" } ], "title": "MemoryRelatives", @@ -2883,10 +2687,6 @@ "id": 50, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -2925,8 +2725,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2958,22 +2757,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_SortTime_Mean", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_SortTime_Mean", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -3012,8 +2805,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3045,22 +2837,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_SortTime_Max", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_SortTime_Max", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -3099,8 +2885,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3131,22 +2916,16 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_SortingFiles_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_SortingFiles_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" }, { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "fieldConfig": { "defaults": { "color": { @@ -3185,8 +2964,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3218,16 +2996,14 @@ }, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "z1lSI_Qnk" - }, "expr": "metrics_SortMemory_Value", - "refId": "A" + "refId": "A", + "datasource": "${DS_PROMETHEUS}" } ], "title": "metrics_SortMemory_Value", - "type": "timeseries" + "type": "timeseries", + "datasource": "${DS_PROMETHEUS}" } ], "title": "SortRelatives", @@ -3247,8 +3023,8 @@ }, "timepicker": {}, "timezone": "", - "title": "RSS", - "uid": "U_qgru_7z", - "version": 20, + "title": "343RSS", + "uid": "U_qgru_7z1345342", + "version": 1, "weekStart": "" } \ No newline at end of file diff --git a/client-spark/shuffle-manager-2/pom.xml b/client-spark/shuffle-manager-2/pom.xml index 62e106e192d..a7d33c12852 100644 --- a/client-spark/shuffle-manager-2/pom.xml +++ b/client-spark/shuffle-manager-2/pom.xml @@ -21,7 +21,7 @@ com.aliyun.emr remote-shuffle-service - 0.1.3 + ${project.version} ../../pom.xml @@ -132,6 +132,10 @@ io.netty com.aliyun.emr.io.netty + + org.apache.commons + org.apache.celeborn.org.apache.commons + @@ -144,6 +148,7 @@ com.google.protobuf:protobuf-java com.google.guava:guava io.netty:* + org.apache.commons:commons-lang3 diff --git a/client-spark/shuffle-manager-3/pom.xml b/client-spark/shuffle-manager-3/pom.xml index 0fabc2a42ef..30bebe65376 100644 --- a/client-spark/shuffle-manager-3/pom.xml +++ b/client-spark/shuffle-manager-3/pom.xml @@ -21,7 +21,7 @@ com.aliyun.emr remote-shuffle-service - 0.1.3 + ${project.version} ../../pom.xml @@ -120,6 +120,10 @@ io.netty com.aliyun.emr.io.netty + + org.apache.commons + org.apache.celeborn.org.apache.commons + @@ -132,6 +136,7 @@ com.google.protobuf:protobuf-java com.google.guava:guava io.netty:* + org.apache.commons:commons-lang3 diff --git a/client-spark/shuffle-manager-common/pom.xml b/client-spark/shuffle-manager-common/pom.xml index f1abb916b2e..991ce66455c 100644 --- a/client-spark/shuffle-manager-common/pom.xml +++ b/client-spark/shuffle-manager-common/pom.xml @@ -21,7 +21,7 @@ com.aliyun.emr remote-shuffle-service - 0.1.3 + ${project.version} ../../pom.xml diff --git a/client/pom.xml b/client/pom.xml index 6de6a1cc8bd..0ed9a8141b4 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -23,7 +23,7 @@ com.aliyun.emr remote-shuffle-service - 0.1.3 + ${project.version} ../pom.xml diff --git a/client/src/main/java/com/aliyun/emr/rss/client/read/RssInputStream.java b/client/src/main/java/com/aliyun/emr/rss/client/read/RssInputStream.java index 947d6ca7544..822fb63f2fd 100644 --- a/client/src/main/java/com/aliyun/emr/rss/client/read/RssInputStream.java +++ b/client/src/main/java/com/aliyun/emr/rss/client/read/RssInputStream.java @@ -33,6 +33,7 @@ import java.util.concurrent.atomic.LongAdder; import io.netty.buffer.ByteBuf; +import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -164,8 +165,12 @@ private boolean skipLocation(int startMapIndex, int endMapIndex, PartitionLocati if (endMapIndex == Integer.MAX_VALUE) { return false; } + RoaringBitmap bitmap = location.getMapIdBitMap(); + if (bitmap == null && location.getPeer() != null) { + bitmap = location.getPeer().getMapIdBitMap(); + } for (int i = startMapIndex; i < endMapIndex; i++) { - if (location.getMapIdBitMap().contains(i)) { + if (bitmap.contains(i)) { return false; } } diff --git a/client/src/main/scala/com/aliyun/emr/rss/client/write/LifecycleManager.scala b/client/src/main/scala/com/aliyun/emr/rss/client/write/LifecycleManager.scala index 45cb82295cc..42de53014ce 100644 --- a/client/src/main/scala/com/aliyun/emr/rss/client/write/LifecycleManager.scala +++ b/client/src/main/scala/com/aliyun/emr/rss/client/write/LifecycleManager.scala @@ -17,15 +17,16 @@ package com.aliyun.emr.rss.client.write +import java.nio.ByteBuffer import java.util -import java.util.concurrent.{ConcurrentHashMap, ScheduledFuture, TimeUnit} -import java.util.concurrent.atomic.LongAdder +import java.util.concurrent.{Callable, ConcurrentHashMap, ScheduledFuture, TimeUnit} import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ListBuffer import scala.util.Random +import com.google.common.cache.{Cache, CacheBuilder} import io.netty.util.internal.ConcurrentSet import org.roaringbitmap.RoaringBitmap @@ -38,7 +39,7 @@ import com.aliyun.emr.rss.common.protocol.RpcNameConstants.WORKER_EP import com.aliyun.emr.rss.common.protocol.message.ControlMessages._ import com.aliyun.emr.rss.common.protocol.message.StatusCode import com.aliyun.emr.rss.common.rpc._ -import com.aliyun.emr.rss.common.rpc.netty.{NettyRpcEndpointRef, NettyRpcEnv} +import com.aliyun.emr.rss.common.rpc.netty.{LocalNettyRpcCallContext, NettyRpcEndpointRef, NettyRpcEnv, RemoteNettyRpcCallContext} import com.aliyun.emr.rss.common.util.{ThreadUtils, Utils} class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint with Logging { @@ -52,6 +53,9 @@ class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint wit private val splitMode = RssConf.partitionSplitMode(conf) private val storageHint = RssConf.storageHint(conf) private val rangeReadFilter = RssConf.rangeReadFilterEnabled(conf) + private val rpcCacheSize = RssConf.rpcCacheSize(conf) + private val rpcCacheConcurrentLevel = RssConf.rpcCacheConcurrentLevel(conf) + private val rpcCacheExpireMs = RssConf.rpcCacheExpireTimeMs(conf) private val unregisterShuffleTime = new ConcurrentHashMap[Int, Long]() @@ -67,6 +71,11 @@ class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint wit // shuffle id -> (partitionId -> newest PartitionLocation) private val latestPartitionLocation = new ConcurrentHashMap[Int, ConcurrentHashMap[Int, PartitionLocation]]() + private val getReducerFileGroupRpcCache: Cache[Int, ByteBuffer] = CacheBuilder.newBuilder() + .concurrencyLevel(rpcCacheConcurrentLevel) + .expireAfterWrite(rpcCacheExpireMs, TimeUnit.MILLISECONDS) + .maximumSize(rpcCacheSize) + .build().asInstanceOf[Cache[Int, ByteBuffer]] val newMapFunc = new util.function.Function[Int, ConcurrentHashMap[Int, PartitionLocation]]() { @@ -582,12 +591,26 @@ class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint wit if (dataLostShuffleSet.contains(shuffleId)) { context.reply(GetReducerFileGroupResponse(StatusCode.Failed, null, null)) } else { - val shuffleFileGroup = reducerFileGroupsMap.get(shuffleId) - context.reply(GetReducerFileGroupResponse( - StatusCode.Success, - shuffleFileGroup, - shuffleMapperAttempts.get(shuffleId) - )) + if (context.isInstanceOf[LocalNettyRpcCallContext]) { + // This branch is for the UTs + context.reply(GetReducerFileGroupResponse( + StatusCode.Success, + reducerFileGroupsMap.getOrDefault(shuffleId, Array.empty), + shuffleMapperAttempts.getOrDefault(shuffleId, Array.empty))) + } else { + val cachedMsg = getReducerFileGroupRpcCache.get( + shuffleId, + new Callable[ByteBuffer]() { + override def call(): ByteBuffer = { + val returnedMsg = GetReducerFileGroupResponse( + StatusCode.Success, + reducerFileGroupsMap.getOrDefault(shuffleId, Array.empty), + shuffleMapperAttempts.getOrDefault(shuffleId, Array.empty)) + context.asInstanceOf[RemoteNettyRpcCallContext].nettyEnv.serialize(returnedMsg) + } + }) + context.asInstanceOf[RemoteNettyRpcCallContext].callback.onSuccess(cachedMsg) + } } } @@ -708,7 +731,6 @@ class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint wit } committedSlaveIds.asScala.foreach { id => val slavePartition = slavePartMap.get(id) - slavePartition.setMapIdBitMap(committedMapIdBitmap.get(id)) val masterPartition = committedPartitions.get(id) if (masterPartition ne null) { masterPartition.setPeer(slavePartition) @@ -716,6 +738,7 @@ class LifecycleManager(appId: String, val conf: RssConf) extends RpcEndpoint wit } else { logWarning(s"Shuffle $shuffleId partition $id: master lost, " + s"use slave $slavePartition.") + slavePartition.setMapIdBitMap(committedMapIdBitmap.get(id)) committedPartitions.put(id, slavePartition) } } diff --git a/common/pom.xml b/common/pom.xml index cc78e6ae37a..01f7fe18d22 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ com.aliyun.emr remote-shuffle-service - 0.1.3 + ${project.version} ../pom.xml diff --git a/common/src/main/scala/com/aliyun/emr/rss/common/RssConf.scala b/common/src/main/scala/com/aliyun/emr/rss/common/RssConf.scala index d494f8761f9..91209338b84 100644 --- a/common/src/main/scala/com/aliyun/emr/rss/common/RssConf.scala +++ b/common/src/main/scala/com/aliyun/emr/rss/common/RssConf.scala @@ -823,6 +823,18 @@ object RssConf extends Logging { conf.getBoolean("rss.range.read.filter.enabled", false) } + def rpcCacheSize(conf: RssConf): Int = { + conf.getInt("rss.rpc.cache.size", 256) + } + + def rpcCacheConcurrentLevel(conf: RssConf): Int = { + conf.getInt("rss.rpc.cache.concurrent.level", 32) + } + + def rpcCacheExpireTimeMs(conf: RssConf): Long = { + conf.getTimeAsMs("rss.rpc.cache.expire", "15s") + } + val WorkingDirName = "hadoop/rss-worker/shuffle_data" // If we want to use multi-raft group we can diff --git a/common/src/main/scala/com/aliyun/emr/rss/common/rpc/netty/NettyRpcCallContext.scala b/common/src/main/scala/com/aliyun/emr/rss/common/rpc/netty/NettyRpcCallContext.scala index e84c57c1d15..eb4f1c651dd 100644 --- a/common/src/main/scala/com/aliyun/emr/rss/common/rpc/netty/NettyRpcCallContext.scala +++ b/common/src/main/scala/com/aliyun/emr/rss/common/rpc/netty/NettyRpcCallContext.scala @@ -55,8 +55,8 @@ private[rss] class LocalNettyRpcCallContext( * A [[RpcCallContext]] that will call [[RpcResponseCallback]] to send the reply back. */ private[rss] class RemoteNettyRpcCallContext( - nettyEnv: NettyRpcEnv, - callback: RpcResponseCallback, + val nettyEnv: NettyRpcEnv, + val callback: RpcResponseCallback, senderAddress: RpcAddress) extends NettyRpcCallContext(senderAddress) { diff --git a/common/src/main/scala/com/aliyun/emr/rss/common/util/Utils.scala b/common/src/main/scala/com/aliyun/emr/rss/common/util/Utils.scala index 89a108d6c47..f1a6d173072 100644 --- a/common/src/main/scala/com/aliyun/emr/rss/common/util/Utils.scala +++ b/common/src/main/scala/com/aliyun/emr/rss/common/util/Utils.scala @@ -21,8 +21,8 @@ import java.io.{File, FileInputStream, InputStreamReader, IOException} import java.lang.management.ManagementFactory import java.math.{MathContext, RoundingMode} import java.net._ -import java.nio.charset.StandardCharsets import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets import java.text.SimpleDateFormat import java.util.{Locale, Properties, UUID} import java.util diff --git a/pom.xml b/pom.xml index 88cb8eebe69..f43630c0dc2 100644 --- a/pom.xml +++ b/pom.xml @@ -23,12 +23,13 @@ com.aliyun.emr remote-shuffle-service - 0.1.3 + ${project.version} pom Aliyun E-MapReduce Shuffle Service Project Parent POM + 0.1.4 UTF-8 UTF-8 1.8 diff --git a/server-common/pom.xml b/server-common/pom.xml index d57af6df476..1ef87a9cede 100644 --- a/server-common/pom.xml +++ b/server-common/pom.xml @@ -22,7 +22,7 @@ com.aliyun.emr remote-shuffle-service - 0.1.3 + ${project.version} ../pom.xml diff --git a/server-master/pom.xml b/server-master/pom.xml index 96ec6ff222b..2b61d0c6781 100644 --- a/server-master/pom.xml +++ b/server-master/pom.xml @@ -5,7 +5,7 @@ remote-shuffle-service com.aliyun.emr - 0.1.3 + ${project.version} 4.0.0 diff --git a/server-worker/pom.xml b/server-worker/pom.xml index 1ffa9a51e3f..74347b1d9db 100644 --- a/server-worker/pom.xml +++ b/server-worker/pom.xml @@ -5,7 +5,7 @@ remote-shuffle-service com.aliyun.emr - 0.1.3 + ${project.version} 4.0.0 diff --git a/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/Controller.scala b/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/Controller.scala index a085f19a33f..dd519fee613 100644 --- a/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/Controller.scala +++ b/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/Controller.scala @@ -18,7 +18,7 @@ package com.aliyun.emr.rss.service.deploy.worker import java.io.IOException -import java.util.{ArrayList => jArrayList, List => jList, HashMap => jHashMap} +import java.util.{ArrayList => jArrayList, HashMap => jHashMap, List => jList} import java.util.concurrent._ import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} import java.util.function.BiFunction diff --git a/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/LocalStorageManager.scala b/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/LocalStorageManager.scala index 94604647971..038bdb81196 100644 --- a/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/LocalStorageManager.scala +++ b/server-worker/src/main/scala/com/aliyun/emr/rss/service/deploy/worker/LocalStorageManager.scala @@ -346,7 +346,7 @@ private[worker] final class LocalStorageManager( @throws[IOException] def createWriter(appId: String, shuffleId: Int, location: PartitionLocation, - splitThreshold: Long, splitMode: PartitionSplitMode,rangeReadFilter: Boolean): FileWriter = { + splitThreshold: Long, splitMode: PartitionSplitMode, rangeReadFilter: Boolean): FileWriter = { if (!hasAvailableWorkingDirs()) { throw new IOException("No available working dirs!") }