diff --git a/dashboards/victoriametrics.json b/dashboards/victoriametrics.json index a9bc56ba03..7089677962 100644 --- a/dashboards/victoriametrics.json +++ b/dashboards/victoriametrics.json @@ -45,12 +45,12 @@ } ] }, - "description": "Overview for cluster VictoriaMetrics v1.57.0 or higher", + "description": "Overview for cluster VictoriaMetrics v1.64.0 or higher", "editable": true, "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1624970989155, + "iteration": 1630495970262, "links": [ { "icon": "doc", @@ -5997,6 +5997,122 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "description": "Shows the saturation level of connection between vminsert and vmstorage components. If the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that more vminsert or vmstorage nodes must be added to the cluster in order to increase the total number of vminsert -> vmstorage links.\n", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 139, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(vm_rpc_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} => {{addr}}", + "refId": "A" + } + ], + "thresholds": [ + { + "$$hashKey": "object:234", + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.9, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Storage connection saturation ($instance)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:156", + "decimals": 0, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:157", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, diff --git a/deployment/docker/alerts.yml b/deployment/docker/alerts.yml index e8c5025b51..3913367323 100644 --- a/deployment/docker/alerts.yml +++ b/deployment/docker/alerts.yml @@ -221,11 +221,24 @@ groups: severity: warning annotations: dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=116&var-instance={{ $labels.instance }}" - summary: "Metrics ingested to vminsert on ({{ $labels.instance }}) are exceeding labels limit" + summary: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding labels limit" description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured correctly or that clients which send these metrics aren't misbehaving." + - alert: VminsertVmstorageConnectionIsSaturated + expr: rate(vm_rpc_send_duration_seconds_total[5m]) > 0.9 + for: 15m + labels: + severity: warning + annotations: + dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=139&var-instance={{ $labels.instance }}" + summary: "Connection between vminsert on {{ $labels.instance }} and vmstorage on {{ $labels.addr }} is saturated" + description: "The connection between vminsert (instance {{ $labels.instance }}) and vmstorage (instance {{ $labels.addr }}) + is saturated by more than 90% and vminsert won't be able to keep up.\n + This usually means that more vminsert or vmstorage nodes must be added to the cluster in order to increase + the total number of vminsert -> vmstorage links." + # Alerts group for vmagent assumes that Grafana dashboard # https://grafana.com/grafana/dashboards/12683 is installed.