diff --git a/dashboards/operator.json b/dashboards/operator.json
index 4f9cc6c7ab..df0f71fce3 100644
--- a/dashboards/operator.json
+++ b/dashboards/operator.json
@@ -6,13 +6,7 @@
"type": "grafana",
"id": "grafana",
"name": "Grafana",
- "version": "9.2.2"
- },
- {
- "type": "panel",
- "id": "graph",
- "name": "Graph (old)",
- "version": ""
+ "version": "10.4.0"
},
{
"type": "datasource",
@@ -31,6 +25,12 @@
"id": "text",
"name": "Text",
"version": ""
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
}
],
"annotations": {
@@ -67,7 +67,7 @@
"collapsed": false,
"datasource": {
"type": "prometheus",
- "uid": "$ds"
+ "uid": "PB894574A363DF0AF"
},
"gridPos": {
"h": 1,
@@ -81,7 +81,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "$ds"
+ "uid": "PB894574A363DF0AF"
},
"refId": "A"
}
@@ -110,7 +110,7 @@
"content": "
$version
",
"mode": "markdown"
},
- "pluginVersion": "9.2.2",
+ "pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
@@ -170,10 +170,12 @@
"fields": "",
"values": false
},
+ "showPercentChange": false,
"text": {},
- "textMode": "auto"
+ "textMode": "auto",
+ "wideLayout": true
},
- "pluginVersion": "9.2.2",
+ "pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
@@ -205,12 +207,12 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
+ "color": "red",
"value": null
},
{
- "color": "red",
- "value": 80
+ "color": "green",
+ "value": 1800
}
]
},
@@ -237,9 +239,11 @@
"fields": "",
"values": false
},
- "textMode": "auto"
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
},
- "pluginVersion": "9.2.2",
+ "pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
@@ -261,49 +265,88 @@
"type": "stat"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
- "fill": 1,
- "fillGradient": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 13,
"w": 12,
"x": 0,
"y": 8
},
- "hiddenSeries": false,
"id": 12,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": false,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
},
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
@@ -311,87 +354,99 @@
"uid": "$ds"
},
"editorMode": "code",
- "expr": "sum(rate(controller_runtime_reconcile_total{job=~\"$job\",instance=~\"$instance\",result=~\"requeue_after|requeue|success\"}[$__rate_interval])) by(controller)",
+ "expr": "sum(rate(controller_runtime_reconcile_total{job=~\"$job\",instance=~\"$instance\",result=~\"requeue_after|requeue|success\"}[$__rate_interval])) by(controller) > 0",
"legendFormat": "{{controller}}",
"range": true,
"refId": "A"
}
],
- "thresholds": [],
- "timeRegions": [],
"title": "Reconciliation rate by controller",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "",
- "fill": 1,
- "fillGradient": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 13,
"w": 12,
"x": 12,
"y": 8
},
- "hiddenSeries": false,
"id": 16,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": false,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
},
- "percentage": false,
- "pluginVersion": "9.2.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
@@ -405,41 +460,14 @@
"refId": "A"
}
],
- "thresholds": [],
- "timeRegions": [],
"title": "Log message rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
+ "type": "timeseries"
},
{
- "collapsed": false,
+ "collapsed": true,
"datasource": {
"type": "prometheus",
- "uid": "$ds"
+ "uid": "PB894574A363DF0AF"
},
"gridPos": {
"h": 1,
@@ -448,402 +476,442 @@
"y": 21
},
"id": 6,
- "panels": [],
- "targets": [
+ "panels": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
+ "description": "Non zero metrics indicates about error with CR object definition (typos or incorrect values) or errors with kubernetes API connection.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 2
+ },
+ "id": 10,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by(controller) > 0 ",
+ "instant": false,
+ "legendFormat": "{{controller}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(controller_runtime_reconcile_total{job=~\"$job\",instance=~\"$instance\",result=\"error\"}[$__rate_interval])) by(controller) > 0",
+ "hide": false,
+ "legendFormat": "{{label_name}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "reconcile errors by controller",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Operator limits number of reconcilation events to 5 events per 2 seconds.\n For now, this limit is applied only for vmalert and vmagent controllers.\n It should reduce load at kubernetes cluster and increase operator performance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 2
+ },
+ "id": 18,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "10.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(operator_reconcile_throttled_events_total[$__rate_interval])) by(controller)",
+ "legendFormat": "{{controller}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "throttled reconcilation events",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Number of objects waiting in the queue for reconciliation. Non-zero values indicate that operator cannot process CR objects changes with the given resources.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 0,
+ "y": 10
+ },
+ "id": 20,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "10.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "max(workqueue_depth{job=~\"$job\",instance=~\"$instance\"}) by (name)",
+ "legendFormat": "{{label_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Wokring queue depth",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": " For controllers with StatefulSet it's ok to see latency greater then 3 seconds. It could be vmalertmanager,vmcluster or vmagent in statefulMode.\n\n For other controllers, latency greater then 1 second may indicate issues with kubernetes cluster or operator's performance.\n ",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 12,
+ "y": 10
+ },
+ "id": 26,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "10.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket[$__rate_interval])) by(le,controller))",
+ "legendFormat": "q.99 {{controller}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Reconcilation latency by controller",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PB894574A363DF0AF"
+ },
"refId": "A"
}
],
"title": "Troubleshooting",
"type": "row"
},
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "Non zero metrics indicates about error with CR object definition (typos or incorrect values) or errors with kubernetes API connection.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 22
- },
- "hiddenSeries": false,
- "id": 10,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by(controller) > 0 ",
- "instant": false,
- "legendFormat": "{{controller}}",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "expr": "sum(rate(controller_runtime_reconcile_total{job=~\"$job\",instance=~\"$instance\",result=\"error\"}[$__rate_interval])) by(controller) > 0",
- "hide": false,
- "legendFormat": "{{label_name}}",
- "range": true,
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "reconcile errors by controller",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "Operator limits number of reconcilation events to 5 events per 2 seconds.\n For now, this limit is applied only for vmalert and vmagent controllers.\n It should reduce load at kubernetes cluster and increase operator performance.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 22
- },
- "hiddenSeries": false,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "expr": "sum(rate(operator_reconcile_throttled_events_total[$__rate_interval])) by(controller)",
- "legendFormat": "{{controller}}",
- "range": true,
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "throttled reconcilation events",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "Number of objects waiting in the queue for reconciliation. Non-zero values indicate that operator cannot process CR objects changes with the given resources.",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 0,
- "y": 30
- },
- "hiddenSeries": false,
- "id": 20,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "expr": "max(workqueue_depth{job=~\"$job\",instance=~\"$instance\"}) by (name)",
- "legendFormat": "{{label_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Wokring queue depth",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": " For controllers with StatefulSet it's ok to see latency greater then 3 seconds. It could be vmalertmanager,vmcluster or vmagent in statefulMode.\n\n For other controllers, latency greater then 1 second may indicate issues with kubernetes cluster or operator's performance.\n ",
- "fieldConfig": {
- "defaults": {
- "unit": "s"
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 12,
- "y": 30
- },
- "hiddenSeries": false,
- "id": 26,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": false,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.3.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99,sum(rate(controller_runtime_reconcile_time_seconds_bucket[$__rate_interval])) by(le,controller) )",
- "legendFormat": "q.99 {{controller}}",
- "range": true,
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Reconcilation latency by controller",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
{
"collapsed": false,
"datasource": {
"type": "prometheus",
- "uid": "$ds"
+ "uid": "PB894574A363DF0AF"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 41
+ "y": 22
},
"id": 4,
"panels": [],
@@ -851,7 +919,7 @@
{
"datasource": {
"type": "prometheus",
- "uid": "$ds"
+ "uid": "PB894574A363DF0AF"
},
"refId": "A"
}
@@ -860,55 +928,89 @@
"type": "row"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
"unit": "bytes"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
- "y": 42
+ "y": 23
},
- "hiddenSeries": false,
"id": 28,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
},
- "percentage": false,
- "pluginVersion": "8.3.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
@@ -958,79 +1060,90 @@
"refId": "D"
}
],
- "thresholds": [],
- "timeRegions": [],
"title": "Memory usage ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "bytes",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
- "fill": 1,
- "fillGradient": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMin": 0,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
- "y": 42
+ "y": 23
},
- "hiddenSeries": false,
"id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
},
- "percentage": false,
- "pluginVersion": "8.3.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
@@ -1044,79 +1157,89 @@
"refId": "A"
}
],
- "thresholds": [],
- "timeRegions": [],
"title": "CPU ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
- "fill": 1,
- "fillGradient": 0,
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
- "y": 50
+ "y": 31
},
- "hiddenSeries": false,
"id": 32,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
},
- "percentage": false,
- "pluginVersion": "8.3.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
@@ -1130,86 +1253,92 @@
"refId": "A"
}
],
- "thresholds": [],
- "timeRegions": [],
"title": "Goroutines ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
"unit": "s"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
- "y": 50
+ "y": 31
},
- "hiddenSeries": false,
"id": 34,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": false,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
},
- "percentage": false,
- "pluginVersion": "8.3.2",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
@@ -1223,40 +1352,12 @@
"refId": "A"
}
],
- "thresholds": [],
- "timeRegions": [],
"title": "GC duration ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
+ "type": "timeseries"
}
],
"refresh": "",
- "schemaVersion": 37,
- "style": "dark",
+ "schemaVersion": 39,
"tags": [
"operator",
"VictoriaMetrics"
@@ -1265,9 +1366,9 @@
"list": [
{
"current": {
- "selected": false,
- "text": "cloud-c15",
- "value": "cloud-c15"
+ "selected": true,
+ "text": "VictoriaMetrics",
+ "value": "PF64AB64142051B50"
},
"hide": 0,
"includeAll": false,
@@ -1331,15 +1432,16 @@
"type": "prometheus",
"uid": "$ds"
},
- "definition": "label_values(vm_app_version{job=\"$job\", instance=\"$instance\"}, version)",
+ "definition": "label_values(vm_app_version{job=\"$job\", instance=~\"$instance\"},version)",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "version",
"options": [],
"query": {
- "query": "label_values(vm_app_version{job=\"$job\", instance=\"$instance\"}, version)",
- "refId": "StandardVariableQuery"
+ "qryType": 1,
+ "query": "label_values(vm_app_version{job=\"$job\", instance=~\"$instance\"},version)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8eb10a4c1f..95f46be284 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -33,6 +33,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
* FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): support selecting of multiple instances on the dashboard. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5869) for details.
* FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): properly display version in the Stats row for the custom builds of VictoriaMetrics.
* FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): add `Network Usage` panel to `Resource Usage` row.
+* FEATURE: [dashboards/operator](https://grafana.com/grafana/dashboards/17869-victoriametrics-operator/): update dashboard to be compatible with Grafana 10+ version.
## [v1.101.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.101.0)