diff --git a/dashboards/victoriametrics-cluster.json b/dashboards/victoriametrics-cluster.json index 5bb50de31..deab75e2a 100644 --- a/dashboards/victoriametrics-cluster.json +++ b/dashboards/victoriametrics-cluster.json @@ -6,7 +6,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.2.6" + "version": "9.2.7" }, { "type": "datasource", @@ -196,7 +196,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -265,7 +265,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -336,7 +336,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -406,7 +406,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -476,7 +476,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -546,7 +546,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -616,7 +616,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -685,7 +685,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -787,7 +787,7 @@ } ] }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -1501,7 +1501,13 @@ "mode": "off" } }, - "links": [], + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=203&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], "mappings": [], "min": 0, "thresholds": { @@ -1637,7 +1643,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1653,7 +1660,7 @@ "h": 8, "w": 12, "x": 0, - "y": 30 + "y": 38 }, "id": 66, "links": [], @@ -1748,7 +1755,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1764,7 +1772,7 @@ "h": 8, "w": 12, "x": 12, - "y": 30 + "y": 38 }, "id": 138, "links": [], @@ -1858,7 +1866,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1874,7 +1883,7 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 46 }, "id": 64, "links": [], @@ -1964,7 +1973,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1993,7 +2003,7 @@ "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 46 }, "id": 122, "links": [], @@ -2101,7 +2111,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2133,7 +2144,7 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 54 }, "id": 117, "links": [], @@ -2181,6 +2192,7 @@ "type": "prometheus", "uid": "$ds" }, + "description": "Shows the number of read/write syscalls such as read, pread, write, pwrite.", "fieldConfig": { "defaults": { "color": { @@ -2216,12 +2228,14 @@ "mode": "off" } }, + "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2231,15 +2245,29 @@ }, "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read .*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 54 }, - "id": 119, + "id": 204, + "links": [], "options": { "legend": { "calcs": [ @@ -2267,14 +2295,32 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(max_over_time(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", + "expr": "sum(rate(process_io_read_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, "interval": "", - "legendFormat": "{{job}}", + "intervalFactor": 1, + "legendFormat": "read {{job}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_write_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{job}}", + "range": true, + "refId": "B" } ], - "title": "TCP connections ($instance)", + "title": "Disk write/read calls ($instance)", "type": "timeseries" }, { @@ -2325,7 +2371,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2341,7 +2388,7 @@ "h": 8, "w": 12, "x": 0, - "y": 54 + "y": 62 }, "id": 68, "links": [], @@ -2429,7 +2476,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2445,9 +2493,9 @@ "h": 8, "w": 12, "x": 12, - "y": 54 + "y": 62 }, - "id": 120, + "id": 119, "options": { "legend": { "calcs": [ @@ -2475,14 +2523,14 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "expr": "sum(max_over_time(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", "interval": "", - "legendFormat": "__auto", + "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "TCP connections rate ($instance)", + "title": "TCP connections ($instance)", "type": "timeseries" }, { @@ -2533,7 +2581,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2549,7 +2598,7 @@ "h": 8, "w": 12, "x": 0, - "y": 62 + "y": 70 }, "id": 70, "links": [], @@ -2591,6 +2640,108 @@ ], "title": "Threads ($instance)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 120, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TCP connections rate ($instance)", + "type": "timeseries" } ], "title": "Resource usage ($job)", @@ -4350,8 +4501,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4462,8 +4612,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4575,8 +4724,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4721,8 +4869,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4859,8 +5006,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4963,8 +5109,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5106,8 +5251,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5210,8 +5354,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5319,8 +5462,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5453,8 +5595,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5566,8 +5707,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -5682,8 +5822,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5818,8 +5957,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7980,7 +8118,7 @@ "h": 2, "w": 24, "x": 0, - "y": 9 + "y": 84 }, "id": 198, "options": { @@ -7992,7 +8130,7 @@ "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", "mode": "markdown" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "transparent": true, "type": "text" }, @@ -8057,7 +8195,7 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 86 }, "id": 189, "links": [], @@ -8159,7 +8297,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 86 }, "id": 190, "links": [], @@ -8261,7 +8399,7 @@ "h": 7, "w": 12, "x": 0, - "y": 19 + "y": 94 }, "id": 192, "links": [], @@ -8369,7 +8507,7 @@ "h": 7, "w": 12, "x": 12, - "y": 19 + "y": 94 }, "id": 196, "links": [], @@ -8472,7 +8610,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 101 }, "id": 200, "links": [], @@ -8574,7 +8712,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 101 }, "id": 201, "links": [], @@ -8628,6 +8766,115 @@ ], "title": "Disk space usage by type ($instance)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 109 + }, + "id": 203, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, location) > 0", + "format": "time_series", + "hide": false, + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Logging rate", + "type": "timeseries" } ], "title": "Drilldown", diff --git a/dashboards/victoriametrics.json b/dashboards/victoriametrics.json index b64e344a7..b3d0641c0 100644 --- a/dashboards/victoriametrics.json +++ b/dashboards/victoriametrics.json @@ -6,7 +6,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.2.6" + "version": "9.2.7" }, { "type": "datasource", @@ -89,7 +89,7 @@ "editable": true, "fiscalYearStartMonth": 0, "gnetId": 10229, - "graphTooltip": 0, + "graphTooltip": 1, "id": null, "links": [ { @@ -168,7 +168,7 @@ "content": "
$version
", "mode": "markdown" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -230,7 +230,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -299,7 +299,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -369,7 +369,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -443,7 +443,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -513,7 +513,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -581,7 +581,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -650,7 +650,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -720,7 +720,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -790,7 +790,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -1456,9 +1456,9 @@ "uid": "$ds" }, "editorMode": "code", - "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, level) > 0", + "expr": "sum(rate(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level, location) > 0", "interval": "5m", - "legendFormat": "{{job}} - {{level}}", + "legendFormat": "{{level}}: {{location}}", "range": true, "refId": "A" } @@ -1576,7 +1576,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)", "interval": "", "legendFormat": "__auto", "range": true, @@ -1586,113 +1586,6 @@ "title": "RSS memory % usage ($instance)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 114, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "CPU % usage ($instance)", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -1757,8 +1650,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 39 + "x": 12, + "y": 31 }, "id": 44, "links": [], @@ -1854,7 +1747,7 @@ "type": "prometheus", "uid": "$ds" }, - "description": "", + "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%", "fieldConfig": { "defaults": { "color": { @@ -1906,33 +1799,17 @@ } ] }, - "unit": "short" + "unit": "percentunit" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, + "x": 0, "y": 39 }, - "id": 57, + "id": 123, "links": [], "options": { "legend": { @@ -1959,29 +1836,123 @@ "type": "prometheus", "uid": "$ds" }, - "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", + "editorMode": "code", + "exemplar": false, + "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)", "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU cores used", + "legendFormat": "__auto", + "range": true, "refId": "A" + } + ], + "title": "RSS anonymous memory % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 114, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, + "editorMode": "code", "exemplar": false, - "expr": "process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}", + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Limit", - "refId": "B" + "legendFormat": "__auto", + "range": true, + "refId": "A" } ], - "title": "CPU ($instance)", + "title": "CPU % usage ($instance)", "type": "timeseries" }, { @@ -2101,7 +2072,7 @@ "hide": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "{{instance}}", "range": true, "refId": "A" } @@ -2109,6 +2080,247 @@ "title": "Open FDs ($instance)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 57, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU cores used", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": false, + "expr": "process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Limit", + "refId": "B" + } + ], + "title": "CPU ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 47, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"}) by(instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines ($instance)", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -2186,7 +2398,7 @@ "h": 8, "w": 12, "x": 12, - "y": 47 + "y": 55 }, "id": 76, "links": [], @@ -2240,215 +2452,6 @@ "title": "Disk writes/reads ($instance)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 55 - }, - "id": 47, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "gc duration", - "refId": "A" - } - ], - "title": "Goroutines ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 55 - }, - "id": 37, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "connections", - "refId": "A" - } - ], - "title": "TCP connections ($instance)", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -2543,16 +2546,260 @@ "type": "prometheus", "uid": "$ds" }, - "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"})", + "editorMode": "code", + "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"}) by(instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "threads", + "legendFormat": "{{instance}}", + "range": true, "refId": "A" } ], "title": "Threads ($instance)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of read/write syscalls such as read, pread, write, pwrite.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "read calls" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 124, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_read_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read calls", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_write_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write calls", + "range": true, + "refId": "B" + } + ], + "title": "Disk write/read calls ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 49, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP connections rate ($instance)", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -2618,9 +2865,9 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 71 }, - "id": 49, + "id": 37, "links": [], "options": { "legend": { @@ -2648,16 +2895,16 @@ "uid": "$ds" }, "editorMode": "code", - "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}) by(instance)", "format": "time_series", "hide": false, "intervalFactor": 1, - "legendFormat": "connections", + "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], - "title": "TCP connections rate ($instance)", + "title": "TCP connections ($instance)", "type": "timeseries" } ], @@ -3628,7 +3875,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3644,7 +3892,7 @@ "h": 8, "w": 12, "x": 0, - "y": 106 + "y": 33 }, "id": 10, "links": [], @@ -3734,7 +3982,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3750,7 +3999,7 @@ "h": 8, "w": 12, "x": 12, - "y": 106 + "y": 33 }, "id": 73, "links": [], @@ -3841,7 +4090,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3857,7 +4107,7 @@ "h": 8, "w": 12, "x": 0, - "y": 114 + "y": 41 }, "id": 53, "links": [], @@ -3974,7 +4224,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4007,7 +4258,7 @@ "h": 8, "w": 12, "x": 12, - "y": 114 + "y": 41 }, "id": 34, "links": [], @@ -4107,7 +4358,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4140,7 +4392,7 @@ "h": 8, "w": 12, "x": 0, - "y": 122 + "y": 49 }, "id": 30, "links": [], @@ -4241,7 +4493,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4257,7 +4510,7 @@ "h": 8, "w": 12, "x": 12, - "y": 122 + "y": 49 }, "id": 36, "links": [], @@ -4344,7 +4597,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4360,7 +4614,7 @@ "h": 8, "w": 12, "x": 0, - "y": 130 + "y": 57 }, "id": 58, "links": [], @@ -4451,7 +4705,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4467,7 +4722,7 @@ "h": 8, "w": 12, "x": 12, - "y": 130 + "y": 57 }, "id": 62, "options": { @@ -4550,7 +4805,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4582,7 +4838,7 @@ "h": 8, "w": 12, "x": 0, - "y": 138 + "y": 65 }, "id": 59, "links": [], @@ -4681,7 +4937,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4697,7 +4954,7 @@ "h": 8, "w": 12, "x": 12, - "y": 138 + "y": 65 }, "id": 64, "options": { @@ -4782,7 +5039,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4798,7 +5056,7 @@ "h": 8, "w": 12, "x": 0, - "y": 146 + "y": 73 }, "id": 99, "links": [], @@ -4827,11 +5085,13 @@ "type": "prometheus", "uid": "$ds" }, - "expr": "histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (vmrange))", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", + "range": true, "refId": "A" } ], @@ -4887,7 +5147,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4903,7 +5164,7 @@ "h": 8, "w": 12, "x": 12, - "y": 146 + "y": 73 }, "id": 103, "links": [], @@ -4933,11 +5194,11 @@ "uid": "$ds" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (vmrange))", + "expr": "histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{label_name}}", + "legendFormat": "{{instance}}", "range": true, "refId": "A" } @@ -4994,7 +5255,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5010,7 +5272,7 @@ "h": 8, "w": 12, "x": 0, - "y": 154 + "y": 81 }, "id": 122, "links": [], @@ -5040,11 +5302,11 @@ "uid": "$ds" }, "editorMode": "code", - "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "expr": "histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "datapoints", + "legendFormat": "{{instance}}", "range": true, "refId": "A" } @@ -5101,7 +5363,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5117,7 +5380,7 @@ "h": 8, "w": 12, "x": 12, - "y": 154 + "y": 81 }, "id": 105, "links": [], @@ -5146,11 +5409,13 @@ "type": "prometheus", "uid": "$ds" }, - "expr": "histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (vmrange))", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", + "range": true, "refId": "A" } ], @@ -5182,7 +5447,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "VictoriaMetrics", "value": "VictoriaMetrics" }, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a276802e8..70a708d43 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -29,6 +29,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): Adds `enable_http2` on scrape configuration level. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4283). Thanks to @Haleygo for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4295). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add verbose output for docker installations or when TTY isn't available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4081). * FEATURE: vmstorage: suppress "broken pipe" errors for search queries on vmstorage side. See [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4418/commits/a6a7795b9e1f210d614a2c5f9a3016b97ded4792). +* FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add panel for tracking rate of syscalls while writing or reading from disk via `process_io_(read|write)_syscalls_total` metrics. * BUGFIX: add the following command-line flags, which can be used for limiting Graphite API calls: `--search.maxGraphiteTagKeys` for limiting the number of tag keys returned from Graphite `/tags`, `/tags/autoComplete/*`, `/tags/findSeries` API.