From e07f23a1b9e0a9c28e7a9fc14514c3f4396661bb Mon Sep 17 00:00:00 2001
From: Max Golionko <8kirk8@gmail.com>
Date: Tue, 6 Sep 2022 22:19:43 +0800
Subject: [PATCH] moved cluster dashboard to master (#3074)
dashboards: move cluster dashboard to master branch
This change should simplify dashboards management.
---
dashboards/victoriametrics-cluster.json | 7483 ++++++++++++++++++++
dashboards/victoriametrics.json | 8345 +++++++----------------
deployment/docker/docker-compose.yml | 2 +-
docs/PerTenantStatistic.md | 2 +-
4 files changed, 10057 insertions(+), 5775 deletions(-)
create mode 100644 dashboards/victoriametrics-cluster.json
diff --git a/dashboards/victoriametrics-cluster.json b/dashboards/victoriametrics-cluster.json
new file mode 100644
index 0000000000..35b1f8a560
--- /dev/null
+++ b/dashboards/victoriametrics-cluster.json
@@ -0,0 +1,7483 @@
+{
+ "__inputs": [],
+ "__elements": {},
+ "__requires": [{
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "9.0.3"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph (old)",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "stat",
+ "name": "Stat",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "table",
+ "name": "Table",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [{
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }]
+ },
+ "description": "Overview for cluster VictoriaMetrics v1.77.0 or higher",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 1,
+ "id": null,
+ "iteration": 1659947950093,
+ "links": [{
+ "icon": "doc",
+ "tags": [],
+ "targetBlank": true,
+ "title": "Cluster Wiki",
+ "type": "link",
+ "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Cluster-VictoriaMetrics"
+ },
+ {
+ "icon": "external link",
+ "tags": [],
+ "targetBlank": true,
+ "title": "Found a bug?",
+ "type": "link",
+ "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues"
+ },
+ {
+ "icon": "external link",
+ "tags": [],
+ "targetBlank": true,
+ "title": "New releases",
+ "type": "link",
+ "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases"
+ }
+ ],
+ "liveNow": false,
+ "panels": [{
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P4169E866C3094E38"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 137,
+ "panels": [{
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "How many datapoints are in storage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{
+ "color": "green"
+ }]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 6,
+ "x": 0,
+ "y": 1
+ },
+ "id": 131,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "8.5.3",
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_rows{job=~\"$job_storage\", type!=\"indexdb\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }],
+ "title": "Total datapoints",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "The minimum free disk space left among all storage nodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{
+ "color": "green"
+ }]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 6,
+ "x": 6,
+ "y": 1
+ },
+ "id": 124,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "8.5.3",
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "min(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }],
+ "title": "Min free disk space",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Average disk usage per datapoint.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 2,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{
+ "color": "green"
+ }]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 6,
+ "x": 12,
+ "y": 1
+ },
+ "id": 112,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "8.5.3",
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\", type!=\"indexdb\"}) / sum(vm_rows{job=~\"$job_storage\", type!=\"indexdb\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }],
+ "title": "Bytes per point",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Total size of allowed memory via flag `-memory.allowedPercent` for all VM components",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{
+ "color": "green"
+ }]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 6,
+ "x": 18,
+ "y": 1
+ },
+ "id": 130,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "8.5.3",
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_allowed_memory_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }],
+ "title": "Allowed memory",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "How many entries inverted index contains. This value is proportional to the number of unique timeseries in storage(cardinality).",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{
+ "color": "green"
+ }]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 6,
+ "x": 0,
+ "y": 3
+ },
+ "id": 34,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "8.5.3",
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_rows{job=~\"$job_storage\", type=\"indexdb\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }],
+ "title": "Index size",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Total amount of used disk space",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{
+ "color": "green"
+ }]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 6,
+ "x": 6,
+ "y": 3
+ },
+ "id": 35,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "8.5.3",
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": false,
+ "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }],
+ "title": "Disk space usage",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Total number of available CPUs for all VM components. ",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{
+ "color": "green"
+ }]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 6,
+ "x": 12,
+ "y": 3
+ },
+ "id": 126,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "8.5.3",
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }],
+ "title": "Available CPU",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Total size of available memory for all VM components.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{
+ "color": "green"
+ }]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 6,
+ "x": 18,
+ "y": 3
+ },
+ "id": 128,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "8.5.3",
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }],
+ "title": "Available memory",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "inspect": false,
+ "minWidth": 50
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [{
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [{
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [{
+ "id": "custom.hidden",
+ "value": true
+ }]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Value"
+ },
+ "properties": [{
+ "id": "displayName",
+ "value": "Count"
+ }]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 0,
+ "y": 5
+ },
+ "id": 149,
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true
+ },
+ "pluginVersion": "8.5.3",
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
+ "format": "table",
+ "instant": true,
+ "range": false,
+ "refId": "A"
+ }],
+ "type": "table"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 16,
+ "x": 8,
+ "y": 5
+ },
+ "hiddenSeries": false,
+ "id": 62,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "current",
+ "sortDesc": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sort(sum(up{job=~\"$job\", instance=~\"$instance\"}) by (job, instance))",
+ "format": "time_series",
+ "instant": false,
+ "legendFormat": "{{instance}}({{job}})",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Uptime",
+ "tooltip": {
+ "shared": true,
+ "sort": 1,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:1177",
+ "decimals": 0,
+ "format": "none",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1178",
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": 2
+ }
+ }
+ ],
+ "title": "Stats",
+ "type": "row"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 1
+ },
+ "id": 10,
+ "panels": [],
+ "title": "Overview",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "How many datapoints are inserted into storage per second by protocol and accountID",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 2
+ },
+ "hiddenSeries": false,
+ "id": 2,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type,accountID) > 0 ",
+ "interval": "",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Datapoints ingestion rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 2
+ },
+ "hiddenSeries": false,
+ "id": 6,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance.*\", path!~\"/favicon.ico\"}[$__rate_interval])) by (path) > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{path}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Requests rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 10
+ },
+ "hiddenSeries": false,
+ "id": 52,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_http_request_errors_total{job=~\"$job\", instance=~\"$instance.*\"}[$__rate_interval])) by (path) > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{path}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Requests error rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "The less time it takes is better.\n* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 10
+ },
+ "hiddenSeries": false,
+ "id": 8,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "max(vm_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=~\"(0.5|0.99)\"}) by (path, quantile) > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{quantile}} ({{path}})",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Query duration ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "s",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "RPC errors are interconnection errors between cluster components. Errors rate should be 0 if network connection is stable and all components are up and operational.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 44,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 1,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Connection",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Dial",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Handshake",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "RPC errors ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "rps",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 104,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance.*\", level!=\"info\"}[$__rate_interval])) by (job, level) ",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}} - {{level}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Logging rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows the percentage of remaining disk space at `-storageDataPath` by instance",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 26
+ },
+ "hiddenSeries": false,
+ "id": 110,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) by(instance) /\n(\n sum(vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) by(instance) +\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) by(instance)\n)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }],
+ "thresholds": [{
+ "$$hashKey": "object:89",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 0.8,
+ "yaxis": "left"
+ }],
+ "timeRegions": [],
+ "title": "Disk space used ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "percentunit",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows the number of active time series with new data points inserted during the last hour. High value may result in ingestion slowdown. \n\nSee following link for details:",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 26
+ },
+ "hiddenSeries": false,
+ "id": 12,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [{
+ "targetBlank": true,
+ "title": "troubleshooting",
+ "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#troubleshooting"
+ }],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_cache_entries{job=~\"$job\", instance=~\"$instance.*\", type=\"storage/hour_metric_ids\"})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Active time series",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Active time series ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 34
+ },
+ "id": 46,
+ "panels": [{
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Amount of used memory (resident)\nContains memory share which can be freed by OS when needed.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 3
+ },
+ "hiddenSeries": false,
+ "id": 66,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)",
+ "interval": "",
+ "legendFormat": "{{instance}} ({{job}})",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "RSS memory usage ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 3
+ },
+ "hiddenSeries": false,
+ "id": 138,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)",
+ "interval": "",
+ "legendFormat": "{{instance}} ({{job}})",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "RSS anonymous memory usage ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:271",
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:272",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 11
+ },
+ "hiddenSeries": false,
+ "id": 64,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}} ({{job}})",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "CPU ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Shows the CPU usage in the percentage from the limit.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 11
+ },
+ "hiddenSeries": false,
+ "id": 146,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": false,
+ "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance) / sum(process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}} ({{job}})",
+ "refId": "A"
+ }],
+ "thresholds": [{
+ "$$hashKey": "object:195",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 0.9,
+ "yaxis": "left"
+ }],
+ "timeRegions": [],
+ "title": "CPU percentage ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:75",
+ "format": "percentunit",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:76",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Panel shows the number of open file descriptors in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 19
+ },
+ "hiddenSeries": false,
+ "id": 117,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "alias": "/max.*/",
+ "color": "#C4162A"
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(process_open_fds{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}} ({{job}})",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(process_max_fds{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "max {{instance}} ({{job}})",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Open FDs ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "decimals": 0,
+ "format": "short",
+ "logBase": 2,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows average GC duration by instance",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 19
+ },
+ "hiddenSeries": false,
+ "id": 72,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(go_gc_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)\n/\nsum(rate(go_gc_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}} ({{job}})",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "GC duration ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "s",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 68,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}} ({{job}})",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Goroutines ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows the number of bytes read/write from the storage layer.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 122,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "alias": "/read .*/",
+ "transform": "negative-Y"
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(instance)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "read {{instance}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(instance)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "write {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Disk writes/reads ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "bytes",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 70,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}} ({{job}})",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Threads ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 119,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
+ "interval": "",
+ "legendFormat": "{{instance}} ({{job}})",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "TCP connections ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 120,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)",
+ "interval": "",
+ "legendFormat": "{{instance}} ({{job}})",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "TCP connections rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "title": "Resource usage ($job)",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 35
+ },
+ "id": 106,
+ "panels": [{
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows the rate and total number of new series created over last 24h.\n\nHigh churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nTo investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 4
+ },
+ "hiddenSeries": false,
+ "id": 102,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "alias": "new series over 24h",
+ "yaxis": 2
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "churn rate",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[24h]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "new series over 24h",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Churn rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Shows the rate of adding new items to the index. It should correlate with `Slow inserts` and `Churn rate` graphs and could help to determine the pressure on indexdb.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 4
+ },
+ "hiddenSeries": false,
+ "id": 147,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(vm_indexdb_items_added_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "items",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "IndexDB items rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:92",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:93",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 12
+ },
+ "hiddenSeries": false,
+ "id": 108,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_slow_row_inserts_total{job=~\"$job_storage\"}[$__rate_interval])) / sum(rate(vm_rows_inserted_total{job=~\"$job_insert\"}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "slow inserts",
+ "refId": "A"
+ }],
+ "thresholds": [{
+ "$$hashKey": "object:72",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 0.1,
+ "yaxis": "left"
+ }],
+ "timeRegions": [],
+ "title": "Slow inserts",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:738",
+ "format": "percentunit",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:739",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 12
+ },
+ "hiddenSeries": false,
+ "id": 107,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "slow queries rate",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Slow queries rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:892",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:893",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "The value is above 0 when the vmstorage at the given `addr` communicates to the given vminsert node that it cannot accept new data because it is in the read-only mode.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 20
+ },
+ "hiddenSeries": false,
+ "id": 142,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [{
+ "targetBlank": true,
+ "title": "Readonly mode",
+ "url": "https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode"
+ }],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 1,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_rpc_vmstorage_is_read_only{job=~\"$job_insert\", instance=~\"$instance\"}) by(instance, addr)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}} => {{addr}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Storage in readonly status for vminsert ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n\nThis prevents from ingesting metrics with too many labels. The value of `maxLabelsPerTimeseries` must be adjusted for your workload.\n\nWhen limit is exceeded (graph is > 0) - extra labels are dropped, which could result in unexpected identical time series.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 20
+ },
+ "hiddenSeries": false,
+ "id": 116,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "metrics with dropped labels",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Labels limit exceeded ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:1046",
+ "decimals": 2,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1047",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Merge assist happens when vmstorage can't keep up with merging parts. This is usually a sign of overload for vmstorage.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 28
+ },
+ "hiddenSeries": false,
+ "id": 170,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 1,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "sum(increase(vm_assisted_merges_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type, instance) > 0",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Assisted merges ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:536",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.",
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 28
+ },
+ "hiddenSeries": false,
+ "id": 144,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "vm_cache_size_bytes{job=~\"$job_storage\", instance=~\"$instance\"} / vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{ instance }} / {{ type }}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache usage % by vmstorage ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:107",
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:108",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "title": "Troubleshooting",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 36
+ },
+ "id": 48,
+ "panels": [{
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes\n\nPlease note, it could be that `Sent > Pushed` because of the replication factor.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 5
+ },
+ "hiddenSeries": false,
+ "id": 76,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 1,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_rpc_rows_pushed_total{job=~\"$job\",instance=~\"$instance\"}[2m]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Pushed",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_rpc_rows_sent_total{job=~\"$job\",instance=~\"$instance\"}[2m]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Sent",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Rows ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:1108",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1109",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Rate of RPC errors by type:\n* `Connection` - the number of connection errors to vmstorage node\n* `Dial` - the number of dial errors to vmstorage node.\n* `Handshake` - the number of handshake errors to vmstorage node\n* `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 5
+ },
+ "hiddenSeries": false,
+ "id": 86,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 1,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Connection",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Dial",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Handshake",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "RPC errors ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 14
+ },
+ "hiddenSeries": false,
+ "id": 80,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 1,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{addr}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Rows ($instance) rerouted to ",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "rps",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 14
+ },
+ "hiddenSeries": false,
+ "id": 78,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 1,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{addr}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Rows ($instance) rerouted from",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "rps",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "The number of rows or bytes that vminesrt internal buffer contains at the moment.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 22
+ },
+ "hiddenSeries": false,
+ "id": 82,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "alias": "bytes",
+ "yaxis": 2
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_rpc_buf_pending_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "legendFormat": "bytes",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_rpc_rows_pending{job=~\"$job\", instance=~\"$instance\"})",
+ "legendFormat": "rows",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Pending",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Network usage by internal VictoriaMetrics RPC protocol",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 22
+ },
+ "hiddenSeries": false,
+ "id": 74,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) * 8",
+ "legendFormat": "network usage",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "RPC network usage ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "bps",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "title": "Interconnection ($job)",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 37
+ },
+ "id": 60,
+ "panels": [{
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited with -`memory.allowedPercent` flag. Line `max allowed` shows max allowed memory size for cache.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 6
+ },
+ "hiddenSeries": false,
+ "id": 57,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "alias": "max allowed",
+ "color": "#C4162A",
+ "fill": 0,
+ "stack": false
+ }],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(type)",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_allowed_memory_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "max allowed",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache size ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.",
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 14
+ },
+ "hiddenSeries": false,
+ "id": 172,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(type) / \nsum(vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}) by(type)",
+ "interval": "",
+ "legendFormat": "{{ type }}",
+ "range": true,
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache usage % ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:107",
+ "format": "percentunit",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:108",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited with -`memory.allowedPercent` flag. Line `max allowed` shows max allowed memory size for cache.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 23
+ },
+ "hiddenSeries": false,
+ "id": 58,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "1 - (\n sum(rate(vm_cache_misses_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type) /\n sum(rate(vm_cache_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type)\n)",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Cache hit ratio ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "percentunit",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "title": "Caches ($job)",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 38
+ },
+ "id": 24,
+ "panels": [{
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows how many rows per second every storage node accepts. This metric doesn't show all stored rows since some of them may be dropped because of wrong timestamps or decode errors.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 7
+ },
+ "hiddenSeries": false,
+ "id": 100,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(instance)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Ingestion rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows the time needed to reach the 100% of disk capacity based on the following params:\n* free disk space;\n* row ingestion rate;\n* dedup rate;\n* compression.\n\nUse this panel for capacity planning in order to estimate the time remaining for running out of the disk space.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 7
+ },
+ "hiddenSeries": false,
+ "id": 113,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": false,
+ "min": true,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"} / ignoring(path) ((rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) - ignoring(type) rate(vm_deduplicated_samples_total{job=~\"$job_storage\", instance=~\"$instance\", type=\"merge\"}[1d])) * scalar(sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"}) / sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"})))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Storage full ETA ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:87",
+ "format": "s",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:88",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows how many datapoints are in the storage.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 15
+ },
+ "hiddenSeries": false,
+ "id": 16,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"}) by(instance)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Datapoints ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second. The index datapoints value in general is much lower.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 15
+ },
+ "hiddenSeries": false,
+ "id": 14,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "alias": "pending index entries",
+ "yaxis": 2
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"storage\"})",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "pending datapoints",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb\"})",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "pending index entries",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Pending datapoints ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "decimals": 3,
+ "format": "none",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows amount of on-disk space occupied by data points.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 23
+ },
+ "hiddenSeries": false,
+ "id": 18,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"}) by(instance)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Disk space usage (datapoints) ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows amount of on-disk space occupied by inverted index.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 23
+ },
+ "hiddenSeries": false,
+ "id": 20,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb\"}) by(instance)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Disk space usage (index) ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "The number of on-going merges in storage nodes. It is expected to have high numbers for `storage/small` metric.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 54,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_active_merges{job=~\"$job_storage\", instance=~\"$instance\"}) by(type)",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Active merges ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "The number of rows merged per second by storage nodes.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 55,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_rows_merged_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Merge speed",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 135,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_rows_ignored_total{job=~\"$job_storage\", instance=~\"$instance\"}) by (reason)",
+ "interval": "",
+ "legendFormat": "{{reason}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Rows ignored ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Data parts of LSM tree.\nHigh number of parts could be an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage(hot data)\n* `storage/big` - small parts gradually merged into big parts (cold data)",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 22,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_parts{job=~\"$job_storage\", instance=~\"$instance\"}) by (type)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "LSM parts ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Value > 0 means vmstorage is in readonly mode.",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 141,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [{
+ "targetBlank": true,
+ "title": "Readonly mode",
+ "url": "https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode"
+ }],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "vm_storage_is_read_only{job=~\"$job_storage\", instance=~\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{ instance }}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Readonly mode",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:131",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:132",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU.",
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 133,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [{
+ "targetBlank": true,
+ "title": "Related discussion",
+ "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632"
+ }],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "alias": "max",
+ "color": "#C4162A"
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "max(vm_concurrent_addrows_capacity{job=~\"$job_storage\", instance=~\"$instance\"})",
+ "interval": "",
+ "legendFormat": "max",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(avg_over_time(vm_concurrent_addrows_current{job=~\"$job_storage\", instance=~\"$instance\"}[1m])) by(instance)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Concurrent flushes on disk ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 55
+ },
+ "hiddenSeries": false,
+ "id": 151,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "$$hashKey": "object:186",
+ "alias": "max",
+ "color": "#C4162A"
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "cores used",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"})",
+ "hide": false,
+ "legendFormat": "max",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "CPU ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 55
+ },
+ "hiddenSeries": false,
+ "id": 167,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "$$hashKey": "object:186",
+ "alias": "max",
+ "color": "#C4162A"
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) ",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "mem used",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"})",
+ "hide": false,
+ "legendFormat": "max",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:164",
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:165",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "title": "vmstorage ($instance)",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 39
+ },
+ "id": 42,
+ "panels": [{
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Request rate accepted by vmselect nodes",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "hiddenSeries": false,
+ "id": 92,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance.*\", path!~\"/favicon.ico\"}[$__rate_interval])) by (path) > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{path}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Requests rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows how many ongoing insertions are taking place.\n* `max` - equal to number of CPU * 2 by default. May be configured with `search.maxConcurrentRequests` flag\n* `current` - current number of goroutines busy with processing requests\n\nWhen `current` hits `max` constantly, it means vmselect node is overloaded and require more CPU or higher limits.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 8
+ },
+ "hiddenSeries": false,
+ "id": 95,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "alias": "max",
+ "color": "#C4162A",
+ "fill": 0
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(max_over_time(vm_concurrent_select_current{job=~\"$job_select\", instance=~\"$instance\"}[1m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "current",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_concurrent_select_capacity{job=~\"$job_select\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "max",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Concurrent selects ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 6,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 24,
+ "x": 0,
+ "y": 16
+ },
+ "hiddenSeries": false,
+ "id": 93,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "alias": "/read.*/",
+ "transform": "negative-Y"
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance) * 8 > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "read - {{instance}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance) * 8 > 0",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "write - {{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Network usage ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "bps",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 23
+ },
+ "hiddenSeries": false,
+ "id": 163,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "$$hashKey": "object:186",
+ "alias": "max",
+ "color": "#C4162A"
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "cores used",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"})",
+ "hide": false,
+ "legendFormat": "max",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "CPU ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 23
+ },
+ "hiddenSeries": false,
+ "id": 165,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "$$hashKey": "object:186",
+ "alias": "max",
+ "color": "#C4162A"
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job_select\", instance=~\"$instance\"}) ",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "mem used",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"})",
+ "hide": false,
+ "legendFormat": "max",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:164",
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:165",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "title": "vmselect ($instance)",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 40
+ },
+ "id": 40,
+ "panels": [{
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 160
+ },
+ "hiddenSeries": false,
+ "id": 97,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_http_requests_total{job=~\"$job_insert\", instance=~\"$instance.*\", path!~\"/favicon.ico\"}[$__rate_interval])) by (path) > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{path}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Requests rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows how many ongoing insertions are taking place.\n* `max` - equal to number of CPU * 4 by default. May be configured with `maxConcurrentInserts` flag;\n* `current` - current number of goroutines busy with processing requests.\n\n`-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate.\n\nWhen `current` hits `max` constantly, it means vminsert node is overloaded and requires more CPU or higher limits.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 160
+ },
+ "hiddenSeries": false,
+ "id": 99,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "alias": "max",
+ "color": "#C4162A",
+ "fill": 0
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_concurrent_insert_current{job=~\"$job_insert\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "current",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_concurrent_insert_capacity{job=~\"$job_insert\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "max",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Concurrent inserts ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 168
+ },
+ "hiddenSeries": false,
+ "id": 90,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (instance) * 8 > 0",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Network usage ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "format": "bps",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 168
+ },
+ "hiddenSeries": false,
+ "id": 88,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Rows per insert ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "decimals": 2,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows the saturation level of connection between vminsert and vmstorage components. If the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that more vminsert or vmstorage nodes must be added to the cluster in order to increase the total number of vminsert -> vmstorage links.\n",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 176
+ },
+ "hiddenSeries": false,
+ "id": 139,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "rate(vm_rpc_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}} => {{addr}}",
+ "refId": "A"
+ }],
+ "thresholds": [{
+ "$$hashKey": "object:234",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 0.9,
+ "yaxis": "left"
+ }],
+ "timeRegions": [],
+ "title": "Storage connection saturation ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:156",
+ "decimals": 0,
+ "format": "s",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:157",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows if vmstorage node is reachable for vminsert. If below 1 means vmstorage is not reachable at this moment of time. ",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 176
+ },
+ "hiddenSeries": false,
+ "id": 114,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "expr": "vm_rpc_vmstorage_is_reachable{job=~\"$job\", instance=~\"$instance\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}} => {{addr}}",
+ "refId": "A"
+ }],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Storage reachability ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:1534",
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1535",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 184
+ },
+ "hiddenSeries": false,
+ "id": 164,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "$$hashKey": "object:186",
+ "alias": "max",
+ "color": "#C4162A"
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "cores used",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"})",
+ "hide": false,
+ "legendFormat": "max",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "CPU ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:887",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:888",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 184
+ },
+ "hiddenSeries": false,
+ "id": 169,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [{
+ "$$hashKey": "object:186",
+ "alias": "max",
+ "color": "#C4162A"
+ }],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job_insert\", instance=~\"$instance\"}) ",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "mem used",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"})",
+ "hide": false,
+ "legendFormat": "max",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [{
+ "$$hashKey": "object:164",
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:165",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "title": "vminsert ($instance)",
+ "type": "row"
+ }
+ ],
+ "schemaVersion": 36,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [{
+ "current": {
+ "selected": false,
+ "text": "VictoriaMetrics",
+ "value": "VictoriaMetrics"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "ds",
+ "options": [],
+ "query": "prometheus",
+ "queryValue": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "uid": "$ds"
+ },
+ "definition": "label_values(vm_app_version{version=~\"^vminsert.*\"}, job)",
+ "hide": 2,
+ "includeAll": false,
+ "multi": false,
+ "name": "job_insert",
+ "options": [],
+ "query": {
+ "query": "label_values(vm_app_version{version=~\"^vminsert.*\"}, job)",
+ "refId": "VictoriaMetrics-job_insert-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {},
+ "datasource": {
+ "uid": "$ds"
+ },
+ "definition": "label_values(vm_app_version{version=~\"^vmselect.*\"}, job)",
+ "hide": 2,
+ "includeAll": false,
+ "multi": false,
+ "name": "job_select",
+ "options": [],
+ "query": {
+ "query": "label_values(vm_app_version{version=~\"^vmselect.*\"}, job)",
+ "refId": "VictoriaMetrics-job_select-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {},
+ "datasource": {
+ "uid": "$ds"
+ },
+ "definition": "label_values(vm_app_version{version=~\"^vmstorage.*\"}, job)",
+ "hide": 2,
+ "includeAll": false,
+ "multi": false,
+ "name": "job_storage",
+ "options": [],
+ "query": {
+ "query": "label_values(vm_app_version{version=~\"^vmstorage.*\"}, job)",
+ "refId": "VictoriaMetrics-job_storage-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {},
+ "datasource": {
+ "uid": "$ds"
+ },
+ "definition": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "job",
+ "options": [],
+ "query": {
+ "query": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)",
+ "refId": "VictoriaMetrics-job-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": ".*",
+ "current": {},
+ "datasource": {
+ "uid": "$ds"
+ },
+ "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "instance",
+ "options": [],
+ "query": {
+ "query": "label_values(vm_app_version{job=~\"$job\"}, instance)",
+ "refId": "VictoriaMetrics-instance-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "type": "query"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${ds}"
+ },
+ "filters": [],
+ "hide": 0,
+ "name": "adhoc",
+ "skipUrlSync": false,
+ "type": "adhoc"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "VictoriaMetrics - cluster",
+ "uid": "oS7Bi_0Wz",
+ "version": 1,
+ "weekStart": ""
+}
diff --git a/dashboards/victoriametrics.json b/dashboards/victoriametrics.json
index 4608f147f2..5efea695e9 100644
--- a/dashboards/victoriametrics.json
+++ b/dashboards/victoriametrics.json
@@ -28,8 +28,8 @@
},
{
"type": "panel",
- "id": "table",
- "name": "Table",
+ "id": "text",
+ "name": "Text",
"version": ""
}
],
@@ -55,20 +55,21 @@
}
]
},
- "description": "Overview for cluster VictoriaMetrics v1.77.0 or higher",
+ "description": "Overview for single node VictoriaMetrics v1.77.0 or higher",
"editable": true,
"fiscalYearStartMonth": 0,
- "graphTooltip": 1,
+ "gnetId": 10229,
+ "graphTooltip": 0,
"id": null,
- "iteration": 1659947950093,
+ "iteration": 1659966607833,
"links": [
{
"icon": "doc",
"tags": [],
"targetBlank": true,
- "title": "Cluster Wiki",
+ "title": "Single server Wiki",
"type": "link",
- "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Cluster-VictoriaMetrics"
+ "url": "https://docs.victoriametrics.com/"
},
{
"icon": "external link",
@@ -83,6 +84,7 @@
"tags": [],
"targetBlank": true,
"title": "New releases",
+ "tooltip": "",
"type": "link",
"url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases"
}
@@ -90,10 +92,9 @@
"liveNow": false,
"panels": [
{
- "collapsed": true,
+ "collapsed": false,
"datasource": {
- "type": "prometheus",
- "uid": "P4169E866C3094E38"
+ "uid": "$ds"
},
"gridPos": {
"h": 1,
@@ -101,745 +102,655 @@
"x": 0,
"y": 0
},
- "id": 137,
- "panels": [
+ "id": 6,
+ "panels": [],
+ "targets": [
{
"datasource": {
"uid": "$ds"
},
- "description": "How many datapoints are in storage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 2,
- "w": 6,
- "x": 0,
- "y": 1
- },
- "id": 131,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "8.5.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(vm_rows{job=~\"$job_storage\", type!=\"indexdb\"})",
- "format": "time_series",
- "instant": true,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Total datapoints",
- "type": "stat"
- },
- {
- "datasource": {
- "uid": "$ds"
- },
- "description": "The minimum free disk space left among all storage nodes.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 2,
- "w": 6,
- "x": 6,
- "y": 1
- },
- "id": 124,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "8.5.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "min(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"})",
- "format": "time_series",
- "instant": true,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Min free disk space",
- "type": "stat"
- },
- {
- "datasource": {
- "uid": "$ds"
- },
- "description": "Average disk usage per datapoint.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 2,
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 2,
- "w": 6,
- "x": 12,
- "y": 1
- },
- "id": 112,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "8.5.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\", type!=\"indexdb\"}) / sum(vm_rows{job=~\"$job_storage\", type!=\"indexdb\"})",
- "format": "time_series",
- "instant": true,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Bytes per point",
- "type": "stat"
- },
- {
- "datasource": {
- "uid": "$ds"
- },
- "description": "Total size of allowed memory via flag `-memory.allowedPercent` for all VM components",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 2,
- "w": 6,
- "x": 18,
- "y": 1
- },
- "id": 130,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "8.5.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(vm_allowed_memory_bytes{job=~\"$job\", instance=~\"$instance\"})",
- "format": "time_series",
- "instant": true,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Allowed memory",
- "type": "stat"
- },
- {
- "datasource": {
- "uid": "$ds"
- },
- "description": "How many entries inverted index contains. This value is proportional to the number of unique timeseries in storage(cardinality).",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 2,
- "w": 6,
- "x": 0,
- "y": 3
- },
- "id": 34,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "8.5.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(vm_rows{job=~\"$job_storage\", type=\"indexdb\"})",
- "format": "time_series",
- "instant": true,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Index size",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "Total amount of used disk space",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 2,
- "w": 6,
- "x": 6,
- "y": 3
- },
- "id": 35,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "8.5.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": false,
- "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"})",
- "format": "time_series",
- "instant": true,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Disk space usage",
- "type": "stat"
- },
- {
- "datasource": {
- "uid": "$ds"
- },
- "description": "Total number of available CPUs for all VM components. ",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 2,
- "w": 6,
- "x": 12,
- "y": 3
- },
- "id": 126,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "8.5.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"})",
- "format": "time_series",
- "instant": true,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Available CPU",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "Total size of available memory for all VM components.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 2,
- "w": 6,
- "x": 18,
- "y": 3
- },
- "id": 128,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "8.5.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"})",
- "format": "time_series",
- "instant": true,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Available memory",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "displayMode": "auto",
- "inspect": false,
- "minWidth": 50
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Time"
- },
- "properties": [
- {
- "id": "custom.hidden",
- "value": true
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Value"
- },
- "properties": [
- {
- "id": "displayName",
- "value": "Count"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 5,
- "w": 8,
- "x": 0,
- "y": 5
- },
- "id": 149,
- "options": {
- "footer": {
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
- },
- "showHeader": true
- },
- "pluginVersion": "8.5.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
- "format": "table",
- "instant": true,
- "range": false,
- "refId": "A"
- }
- ],
- "type": "table"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 5,
- "w": 16,
- "x": 8,
- "y": 5
- },
- "hiddenSeries": false,
- "id": 62,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "hideEmpty": false,
- "hideZero": false,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": true,
- "sort": "current",
- "sortDesc": false,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": true,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sort(sum(up{job=~\"$job\", instance=~\"$instance\"}) by (job, instance))",
- "format": "time_series",
- "instant": false,
- "legendFormat": "{{instance}}({{job}})",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Uptime",
- "tooltip": {
- "shared": true,
- "sort": 1,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:1177",
- "decimals": 0,
- "format": "none",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:1178",
- "format": "short",
- "label": "",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": 2
- }
+ "refId": "A"
}
],
"title": "Stats",
"type": "row"
},
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "",
+ "gridPos": {
+ "h": 2,
+ "w": 4,
+ "x": 0,
+ "y": 1
+ },
+ "id": 85,
+ "options": {
+ "content": "<div style=\"text-align: center;\">$version</div>",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Version",
+ "type": "text"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "How many datapoints are in storage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 5,
+ "x": 4,
+ "y": 1
+ },
+ "id": 26,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type!=\"indexdb\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Total datapoints",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Total amount of used disk space",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 5,
+ "x": 9,
+ "y": 1
+ },
+ "id": 81,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": false,
+ "expr": "sum(vm_data_size_bytes{job=~\"$job\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Disk space usage",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Average disk usage per datapoint.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 5,
+ "x": 14,
+ "y": 1
+ },
+ "id": 82,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_data_size_bytes{job=~\"$job\", type!=\"indexdb\"}) / sum(vm_rows{job=~\"$job\", type!=\"indexdb\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Bytes per point",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Total size of allowed memory via flag `-memory.allowedPercent`",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 5,
+ "x": 19,
+ "y": 1
+ },
+ "id": 79,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_allowed_memory_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Allowed memory",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ },
+ {
+ "color": "green",
+ "value": 1800
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 4,
+ "x": 0,
+ "y": 3
+ },
+ "id": 87,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "vm_app_uptime_seconds{job=~\"$job\", instance=~\"$instance\"}",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Uptime",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "How many entries inverted index contains. This value is proportional to the number of unique timeseries in storage(cardinality).",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 5,
+ "x": 4,
+ "y": 3
+ },
+ "id": 38,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type=\"indexdb\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Index size",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "The minimum free disk space left",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "percentage",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 5,
+ "x": 9,
+ "y": 3
+ },
+ "id": 80,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "min(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Min free disk space",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Total number of available CPUs for VM process",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 5,
+ "x": 14,
+ "y": 3
+ },
+ "id": 77,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Available CPU",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Total size of available memory for VM process",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 5,
+ "x": 19,
+ "y": 3
+ },
+ "id": 78,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Available memory",
+ "type": "stat"
+ },
{
"collapsed": false,
"datasource": {
@@ -849,110 +760,21 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 1
+ "y": 5
},
- "id": 10,
+ "id": 24,
"panels": [],
- "title": "Overview",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Performance",
"type": "row"
},
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "How many datapoints are inserted into storage per second by protocol and accountID",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 2
- },
- "hiddenSeries": false,
- "id": 2,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type,accountID) > 0 ",
- "interval": "",
- "legendFormat": "{{type}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Datapoints ingestion rate ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
{
"aliasColors": {},
"bars": false,
@@ -973,11 +795,11 @@
"gridPos": {
"h": 8,
"w": 12,
- "x": 12,
- "y": 2
+ "x": 0,
+ "y": 6
},
"hiddenSeries": false,
- "id": 6,
+ "id": 12,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -1009,11 +831,11 @@
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance.*\", path!~\"/favicon.ico\"}[$__rate_interval])) by (path) > 0",
+ "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\", path!~\"/favicon.ico\"}[$__interval])) by (path) > 0",
"format": "time_series",
+ "interval": "",
"intervalFactor": 1,
"legendFormat": "{{path}}",
"refId": "A"
@@ -1051,104 +873,6 @@
"align": false
}
},
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 10
- },
- "hiddenSeries": false,
- "id": 52,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_http_request_errors_total{job=~\"$job\", instance=~\"$instance.*\"}[$__rate_interval])) by (path) > 0",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{path}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Requests error rate ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
{
"aliasColors": {},
"bars": false,
@@ -1170,10 +894,10 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 10
+ "y": 6
},
"hiddenSeries": false,
- "id": 8,
+ "id": 22,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -1205,7 +929,6 @@
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
"expr": "max(vm_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=~\"(0.5|0.99)\"}) by (path, quantile) > 0",
@@ -1247,335 +970,6 @@
"align": false
}
},
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "RPC errors are interconnection errors between cluster components. Errors rate should be 0 if network connection is stable and all components are up and operational.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 18
- },
- "hiddenSeries": false,
- "id": 44,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 1,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Connection",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Dial",
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance.*\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Handshake",
- "refId": "E"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "RPC errors ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "rps",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 18
- },
- "hiddenSeries": false,
- "id": 104,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance.*\", level!=\"info\"}[$__rate_interval])) by (job, level) ",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}} - {{level}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Logging rate",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Shows the percentage of remaining disk space at `-storageDataPath` by instance",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 26
- },
- "hiddenSeries": false,
- "id": 110,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) by(instance) /\n(\n sum(vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) by(instance) +\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) by(instance)\n)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:89",
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 0.8,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "Disk space used ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
{
"aliasColors": {},
"bars": false,
@@ -1596,11 +990,11 @@
"gridPos": {
"h": 8,
"w": 12,
- "x": 12,
- "y": 26
+ "x": 0,
+ "y": 14
},
"hiddenSeries": false,
- "id": 12,
+ "id": 51,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -1627,7 +1021,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "9.0.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -1638,10 +1032,9 @@
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(vm_cache_entries{job=~\"$job\", instance=~\"$instance.*\", type=\"storage/hour_metric_ids\"})",
+ "expr": "vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "Active time series",
@@ -1681,1264 +1074,237 @@
}
},
{
- "collapsed": true,
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
"datasource": {
"uid": "$ds"
},
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 34
+ "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
},
- "id": 46,
- "panels": [
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 14
+ },
+ "hiddenSeries": false,
+ "id": 35,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": {
"uid": "$ds"
},
- "description": "Amount of used memory (resident)\nContains memory share which can be freed by OS when needed.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 3
- },
- "hiddenSeries": false,
- "id": 66,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)",
- "interval": "",
- "legendFormat": "{{instance}} ({{job}})",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "RSS memory usage ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "bytes",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 3
- },
- "hiddenSeries": false,
- "id": 138,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)",
- "interval": "",
- "legendFormat": "{{instance}} ({{job}})",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "RSS anonymous memory usage ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:271",
- "format": "bytes",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:272",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 11
- },
- "hiddenSeries": false,
- "id": 64,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{instance}} ({{job}})",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "CPU ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "Shows the CPU usage in the percentage from the limit.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 11
- },
- "hiddenSeries": false,
- "id": 146,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": false,
- "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance) / sum(process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{instance}} ({{job}})",
- "refId": "A"
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:195",
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 0.9,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "CPU percentage ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:75",
- "format": "percentunit",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:76",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Panel shows the number of open file descriptors in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 19
- },
- "hiddenSeries": false,
- "id": 117,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": false,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/max.*/",
- "color": "#C4162A"
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(process_open_fds{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}} ({{job}})",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(process_max_fds{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "max {{instance}} ({{job}})",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Open FDs ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": 0,
- "format": "short",
- "logBase": 2,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Shows average GC duration by instance",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 19
- },
- "hiddenSeries": false,
- "id": 72,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(go_gc_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)\n/\nsum(rate(go_gc_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}} ({{job}})",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "GC duration ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 27
- },
- "hiddenSeries": false,
- "id": 68,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}} ({{job}})",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Goroutines ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Shows the number of bytes read/write from the storage layer.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 27
- },
- "hiddenSeries": false,
- "id": 122,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/read .*/",
- "transform": "negative-Y"
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(instance)",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "read {{instance}}",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(instance)",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "write {{instance}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Disk writes/reads ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "bytes",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 35
- },
- "hiddenSeries": false,
- "id": 70,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}} ({{job}})",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Threads ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 35
- },
- "hiddenSeries": false,
- "id": 119,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)",
- "interval": "",
- "legendFormat": "{{instance}} ({{job}})",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "TCP connections ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 43
- },
- "hiddenSeries": false,
- "id": 120,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)",
- "interval": "",
- "legendFormat": "{{instance}} ({{job}})",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "TCP connections rate ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
+ "exemplar": false,
+ "expr": "sum(rate(vm_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by (path) > 0",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{path}}",
+ "refId": "A"
}
],
- "title": "Resource usage ($job)",
- "type": "row"
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Requests error rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. However, these two actions are separated and controlled via different concurrency limiters. The `max` on this panel can't be changed and always equal to number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 22
+ },
+ "hiddenSeries": false,
+ "id": 59,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "max",
+ "color": "#C4162A"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_concurrent_addrows_capacity{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "max",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_concurrent_addrows_current{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "current",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Concurrent flushes on disk ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:425",
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:426",
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
},
{
"collapsed": true,
"datasource": {
- "uid": "$ds"
+ "type": "prometheus",
+ "uid": "P4169E866C3094E38"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 35
+ "y": 30
},
- "id": 106,
+ "id": 92,
"panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Shows the rate and total number of new series created over last 24h.\n\nHigh churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nTo investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 4
- },
- "hiddenSeries": false,
- "id": 102,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "new series over 24h",
- "yaxis": 2
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))",
- "interval": "",
- "legendFormat": "churn rate",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(increase(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[24h]))",
- "hide": false,
- "interval": "",
- "legendFormat": "new series over 24h",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Churn rate ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
{
"aliasColors": {},
"bars": false,
@@ -2948,1400 +1314,17 @@
"type": "prometheus",
"uid": "$ds"
},
- "description": "Shows the rate of adding new items to the index. It should correlate with `Slow inserts` and `Churn rate` graphs and could help to determine the pressure on indexdb.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 4
- },
- "hiddenSeries": false,
- "id": 147,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(vm_indexdb_items_added_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))",
- "interval": "",
- "legendFormat": "items",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "IndexDB items rate ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:92",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:93",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 12
- },
- "hiddenSeries": false,
- "id": 108,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_slow_row_inserts_total{job=~\"$job_storage\"}[$__rate_interval])) / sum(rate(vm_rows_inserted_total{job=~\"$job_insert\"}[$__rate_interval]))",
- "interval": "",
- "legendFormat": "slow inserts",
- "refId": "A"
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:72",
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 0.1,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "Slow inserts",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:738",
- "format": "percentunit",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:739",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 12
- },
- "hiddenSeries": false,
- "id": 107,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))",
- "interval": "",
- "legendFormat": "slow queries rate",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Slow queries rate ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:892",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:893",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "The value is above 0 when the vmstorage at the given `addr` communicates to the given vminsert node that it cannot accept new data because it is in the read-only mode.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 20
- },
- "hiddenSeries": false,
- "id": 142,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [
- {
- "targetBlank": true,
- "title": "Readonly mode",
- "url": "https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode"
- }
- ],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 1,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(vm_rpc_vmstorage_is_read_only{job=~\"$job_insert\", instance=~\"$instance\"}) by(instance, addr)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{instance}} => {{addr}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Storage in readonly status for vminsert ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n\nThis prevents from ingesting metrics with too many labels. The value of `maxLabelsPerTimeseries` must be adjusted for your workload.\n\nWhen limit is exceeded (graph is > 0) - extra labels are dropped, which could result in unexpected identical time series.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 20
- },
- "hiddenSeries": false,
- "id": 116,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))",
- "format": "time_series",
- "hide": false,
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "metrics with dropped labels",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Labels limit exceeded ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:1046",
- "decimals": 2,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:1047",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "Merge assist happens when vmstorage can't keep up with merging parts. This is usually a sign of overload for vmstorage.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 28
- },
- "hiddenSeries": false,
- "id": 170,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 1,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "exemplar": true,
- "expr": "sum(increase(vm_assisted_merges_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type, instance) > 0",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Assisted merges ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:536",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:537",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.",
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 28
- },
- "hiddenSeries": false,
- "id": 144,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "vm_cache_size_bytes{job=~\"$job_storage\", instance=~\"$instance\"} / vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}",
- "interval": "",
- "legendFormat": "{{ instance }} / {{ type }}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Cache usage % by vmstorage ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:107",
- "format": "percentunit",
- "logBase": 1,
- "show": true
- },
- {
- "$$hashKey": "object:108",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "title": "Troubleshooting",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "uid": "$ds"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 36
- },
- "id": 48,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes\n\nPlease note, it could be that `Sent > Pushed` because of the replication factor.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 5
- },
- "hiddenSeries": false,
- "id": 76,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 1,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_rpc_rows_pushed_total{job=~\"$job\",instance=~\"$instance\"}[2m]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Pushed",
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_rpc_rows_sent_total{job=~\"$job\",instance=~\"$instance\"}[2m]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Sent",
- "refId": "E"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Rows ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:1108",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:1109",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Rate of RPC errors by type:\n* `Connection` - the number of connection errors to vmstorage node\n* `Dial` - the number of dial errors to vmstorage node.\n* `Handshake` - the number of handshake errors to vmstorage node\n* `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 5
- },
- "hiddenSeries": false,
- "id": 86,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 1,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Connection",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Dial",
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Handshake",
- "refId": "E"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "RPC errors ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 14
- },
- "hiddenSeries": false,
- "id": 80,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 1,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{addr}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Rows ($instance) rerouted to ",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "rps",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 14
- },
- "hiddenSeries": false,
- "id": 78,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "9.0.3",
- "pointradius": 1,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr)",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{addr}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Rows ($instance) rerouted from",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "rps",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "The number of rows or bytes that vminesrt internal buffer contains at the moment.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 22
- },
- "hiddenSeries": false,
- "id": 82,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "bytes",
- "yaxis": 2
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(vm_rpc_buf_pending_bytes{job=~\"$job\", instance=~\"$instance\"})",
- "legendFormat": "bytes",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(vm_rpc_rows_pending{job=~\"$job\", instance=~\"$instance\"})",
- "legendFormat": "rows",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Pending",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "bytes",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Network usage by internal VictoriaMetrics RPC protocol",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 22
- },
- "hiddenSeries": false,
- "id": 74,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) * 8",
- "legendFormat": "network usage",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "RPC network usage ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "bps",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "title": "Interconnection ($job)",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "uid": "$ds"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 37
- },
- "id": 60,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
"description": "VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited with -`memory.allowedPercent` flag. Line `max allowed` shows max allowed memory size for cache.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
- "y": 6
+ "y": 7
},
"hiddenSeries": false,
- "id": 57,
+ "id": 94,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -4357,7 +1340,6 @@
},
"lines": true,
"linewidth": 1,
- "links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
@@ -4369,14 +1351,15 @@
"renderer": "flot",
"seriesOverrides": [
{
+ "$$hashKey": "object:85",
"alias": "max allowed",
- "color": "#C4162A",
+ "color": "#F2495C",
"fill": 0,
"stack": false
}
],
"spaceLength": 10,
- "stack": true,
+ "stack": false,
"steppedLine": false,
"targets": [
{
@@ -4384,10 +1367,9 @@
"type": "prometheus",
"uid": "$ds"
},
+ "exemplar": true,
"expr": "sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(type)",
- "format": "time_series",
- "hide": false,
- "intervalFactor": 1,
+ "interval": "",
"legendFormat": "{{type}}",
"refId": "A"
},
@@ -4396,10 +1378,10 @@
"type": "prometheus",
"uid": "$ds"
},
+ "exemplar": false,
"expr": "sum(vm_allowed_memory_bytes{job=~\"$job\", instance=~\"$instance\"})",
- "format": "time_series",
"hide": false,
- "intervalFactor": 1,
+ "interval": "",
"legendFormat": "max allowed",
"refId": "B"
}
@@ -4420,15 +1402,15 @@
},
"yaxes": [
{
+ "$$hashKey": "object:111",
"format": "bytes",
"logBase": 1,
- "min": "0",
"show": true
},
{
+ "$$hashKey": "object:112",
"format": "short",
"logBase": 1,
- "min": "0",
"show": true
}
],
@@ -4449,25 +1431,21 @@
"fill": 0,
"fillGradient": 0,
"gridPos": {
- "h": 9,
+ "h": 7,
"w": 24,
"x": 0,
- "y": 14
+ "y": 15
},
"hiddenSeries": false,
- "id": 172,
+ "id": 97,
"legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
+ "avg": false,
+ "current": false,
+ "max": false,
"min": false,
- "rightSide": true,
- "show": true,
- "sort": "current",
- "sortDesc": true,
+ "show": false,
"total": false,
- "values": true
+ "values": false
},
"lines": true,
"linewidth": 1,
@@ -4490,12 +1468,10 @@
"type": "prometheus",
"uid": "$ds"
},
- "editorMode": "code",
"exemplar": true,
- "expr": "sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(type) / \nsum(vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}) by(type)",
+ "expr": "vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"} / vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}",
"interval": "",
- "legendFormat": "{{ type }}",
- "range": true,
+ "legendFormat": "{{type}}",
"refId": "A"
}
],
@@ -4515,13 +1491,13 @@
},
"yaxes": [
{
- "$$hashKey": "object:107",
+ "$$hashKey": "object:235",
"format": "percentunit",
"logBase": 1,
"show": true
},
{
- "$$hashKey": "object:108",
+ "$$hashKey": "object:236",
"format": "short",
"logBase": 1,
"show": true
@@ -4537,30 +1513,25 @@
"dashLength": 10,
"dashes": false,
"datasource": {
+ "type": "prometheus",
"uid": "$ds"
},
- "description": "VictoriaMetrics stores various caches in RAM. Memory size for these caches may be limited with -`memory.allowedPercent` flag. Line `max allowed` shows max allowed memory size for cache.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
+ "description": "Cache hit ratio shows cache efficiency. The higher is hit rate the better.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
- "y": 23
+ "y": 22
},
"hiddenSeries": false,
- "id": 58,
+ "id": 95,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
- "max": true,
+ "max": false,
"min": false,
"rightSide": true,
"show": true,
@@ -4571,7 +1542,6 @@
},
"lines": true,
"linewidth": 1,
- "links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
@@ -4581,7 +1551,15 @@
"pointradius": 2,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:85",
+ "alias": "max allowed",
+ "color": "#F2495C",
+ "fill": 0,
+ "stack": false
+ }
+ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
@@ -4593,10 +1571,7 @@
},
"exemplar": true,
"expr": "1 - (\n sum(rate(vm_cache_misses_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type) /\n sum(rate(vm_cache_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type)\n)",
- "format": "time_series",
- "hide": false,
"interval": "",
- "intervalFactor": 1,
"legendFormat": "{{type}}",
"refId": "A"
}
@@ -4617,16 +1592,15 @@
},
"yaxes": [
{
+ "$$hashKey": "object:111",
"format": "percentunit",
"logBase": 1,
- "max": "1",
- "min": "0",
"show": true
},
{
+ "$$hashKey": "object:112",
"format": "short",
"logBase": 1,
- "min": "0",
"show": true
}
],
@@ -4635,7 +1609,16 @@
}
}
],
- "title": "Caches ($job)",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P4169E866C3094E38"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Caches",
"type": "row"
},
{
@@ -4647,9 +1630,9 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 38
+ "y": 31
},
- "id": 24,
+ "id": 14,
"panels": [
{
"aliasColors": {},
@@ -4659,7 +1642,7 @@
"datasource": {
"uid": "$ds"
},
- "description": "Shows how many rows per second every storage node accepts. This metric doesn't show all stored rows since some of them may be dropped because of wrong timestamps or decode errors.",
+ "description": "How many datapoints are inserted into storage per second",
"fieldConfig": {
"defaults": {
"links": []
@@ -4672,15 +1655,16 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 7
+ "y": 3
},
"hiddenSeries": false,
- "id": 100,
+ "id": 10,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
- "max": true,
+ "hideZero": true,
+ "max": false,
"min": false,
"show": true,
"sort": "current",
@@ -4691,35 +1675,35 @@
"lines": true,
"linewidth": 1,
"links": [],
- "nullPointMode": "null",
+ "nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
- "stack": true,
+ "stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(instance)",
+ "expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by (type) > 0",
"format": "time_series",
+ "hide": false,
"intervalFactor": 1,
- "legendFormat": "{{instance}}",
+ "legendFormat": "{{type}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Ingestion rate ($instance)",
+ "title": "Datapoints ingestion rate ($instance)",
"tooltip": {
"shared": true,
"sort": 2,
@@ -4757,27 +1741,28 @@
"datasource": {
"uid": "$ds"
},
- "description": "Shows the time needed to reach the 100% of disk capacity based on the following params:\n* free disk space;\n* row ingestion rate;\n* dedup rate;\n* compression.\n\nUse this panel for capacity planning in order to estimate the time remaining for running out of the disk space.",
+ "description": "Shows the time needed to reach the 100% of disk capacity based on the following params:\n* free disk space;\n* row ingestion rate;\n* dedup rate;\n* compression.\n\nUse this panel for capacity planning in order to estimate the time remaining for running out of the disk space.\n\n",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
- "fill": 0,
+ "fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
- "y": 7
+ "y": 3
},
"hiddenSeries": false,
- "id": 113,
+ "id": 73,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
+ "hideZero": true,
"max": false,
"min": true,
"show": true,
@@ -4789,12 +1774,12 @@
"lines": true,
"linewidth": 1,
"links": [],
- "nullPointMode": "null",
+ "nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -4805,14 +1790,14 @@
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"} / ignoring(path) ((rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) - ignoring(type) rate(vm_deduplicated_samples_total{job=~\"$job_storage\", instance=~\"$instance\", type=\"merge\"}[1d])) * scalar(sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"}) / sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"})))",
+ "expr": "vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"} / ignoring(path) ((rate(vm_rows_added_to_storage_total{job=~\"$job\", instance=~\"$instance\"}[1d]) - ignoring(type) rate(vm_deduplicated_samples_total{job=~\"$job\", instance=~\"$instance\", type=\"merge\"}[1d])) * scalar(sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!=\"indexdb\"}) / sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type!=\"indexdb\"})))",
"format": "time_series",
+ "hide": false,
"interval": "",
"intervalFactor": 1,
- "legendFormat": "{{instance}}",
+ "legendFormat": "",
"refId": "A"
}
],
@@ -4821,7 +1806,7 @@
"title": "Storage full ETA ($instance)",
"tooltip": {
"shared": true,
- "sort": 0,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -4832,16 +1817,15 @@
},
"yaxes": [
{
- "$$hashKey": "object:87",
"format": "s",
"logBase": 1,
"min": "0",
"show": true
},
{
- "$$hashKey": "object:88",
"format": "short",
"logBase": 1,
+ "min": "0",
"show": true
}
],
@@ -4857,7 +1841,7 @@
"datasource": {
"uid": "$ds"
},
- "description": "Shows how many datapoints are in the storage.",
+ "description": "Shows how many datapoints are in the storage and what is average disk usage per datapoint.",
"fieldConfig": {
"defaults": {
"links": []
@@ -4870,10 +1854,10 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 15
+ "y": 11
},
"hiddenSeries": false,
- "id": 16,
+ "id": 30,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -4894,26 +1878,41 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "alias": "bytes-per-datapoint",
+ "yaxis": 2
+ }
+ ],
"spaceLength": 10,
- "stack": true,
+ "stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"}) by(instance)",
+ "expr": "sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type != \"indexdb\"})",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
- "legendFormat": "{{instance}}",
+ "legendFormat": "total datapoints",
"refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!=\"indexdb\"}) / sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type != \"indexdb\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "bytes-per-datapoint",
+ "refId": "B"
}
],
"thresholds": [],
@@ -4938,7 +1937,8 @@
"show": true
},
{
- "format": "short",
+ "decimals": 2,
+ "format": "bytes",
"logBase": 1,
"min": "0",
"show": true
@@ -4956,7 +1956,7 @@
"datasource": {
"uid": "$ds"
},
- "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second. The index datapoints value in general is much lower.",
+ "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second.",
"fieldConfig": {
"defaults": {
"links": []
@@ -4969,17 +1969,16 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 15
+ "y": 11
},
"hiddenSeries": false,
- "id": 14,
+ "id": 34,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": false,
- "rightSide": false,
"show": true,
"sort": "current",
"sortDesc": true,
@@ -4994,7 +1993,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -5010,10 +2009,9 @@
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"storage\"})",
+ "expr": "vm_pending_rows{job=~\"$job\", instance=~\"$instance\", type=\"storage\"}",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
@@ -5022,10 +2020,9 @@
},
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb\"})",
+ "expr": "vm_pending_rows{job=~\"$job\", instance=~\"$instance\", type=\"indexdb\"}",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
@@ -5074,7 +2071,7 @@
"datasource": {
"uid": "$ds"
},
- "description": "Shows amount of on-disk space occupied by data points.",
+ "description": "Shows amount of on-disk space occupied by data points and the remaining disk space at `-storageDataPath`",
"fieldConfig": {
"defaults": {
"links": []
@@ -5087,16 +2084,17 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 23
+ "y": 19
},
"hiddenSeries": false,
- "id": 18,
+ "id": 53,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": false,
+ "rightSide": false,
"show": true,
"sort": "current",
"sortDesc": true,
@@ -5111,30 +2109,41 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
- "stack": true,
+ "stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!=\"indexdb\"}) by(instance)",
+ "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!=\"indexdb\"})",
"format": "time_series",
+ "interval": "",
"intervalFactor": 1,
- "legendFormat": "{{instance}}",
+ "legendFormat": "Used",
"refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Free",
+ "refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Disk space usage (datapoints) ($instance)",
+ "title": "Disk space usage - datapoints ($instance)",
"tooltip": {
"shared": true,
"sort": 0,
@@ -5172,7 +2181,7 @@
"datasource": {
"uid": "$ds"
},
- "description": "Shows amount of on-disk space occupied by inverted index.",
+ "description": "Data parts of LSM tree.\nHigh number of parts could be an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage(hot data)\n* `storage/big` - small parts gradually merged into big parts (cold data)",
"fieldConfig": {
"defaults": {
"links": []
@@ -5185,10 +2194,10 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 23
+ "y": 19
},
"hiddenSeries": false,
- "id": 20,
+ "id": 36,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -5209,30 +2218,128 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
- "stack": true,
+ "stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb\"}) by(instance)",
+ "expr": "sum(vm_parts{job=~\"$job\", instance=~\"$instance\"}) by (type)",
"format": "time_series",
"intervalFactor": 1,
- "legendFormat": "{{instance}}",
+ "legendFormat": "{{type}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Disk space usage (index) ($instance)",
+ "title": "LSM parts ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows amount of on-disk space occupied by inverted index.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 27
+ },
+ "hiddenSeries": false,
+ "id": 55,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=\"indexdb\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "disk space used",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Disk space usage - index ($instance)",
"tooltip": {
"shared": true,
"sort": 0,
@@ -5282,11 +2389,11 @@
"gridPos": {
"h": 8,
"w": 12,
- "x": 0,
- "y": 31
+ "x": 12,
+ "y": 27
},
"hiddenSeries": false,
- "id": 54,
+ "id": 62,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -5306,7 +2413,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -5317,10 +2424,9 @@
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(vm_active_merges{job=~\"$job_storage\", instance=~\"$instance\"}) by(type)",
+ "expr": "sum(vm_active_merges{job=~\"$job\", instance=~\"$instance\"}) by(type)",
"legendFormat": "{{type}}",
"refId": "A"
}
@@ -5366,7 +2472,7 @@
"datasource": {
"uid": "$ds"
},
- "description": "The number of rows merged per second by storage nodes.",
+ "description": "Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps.",
"fieldConfig": {
"defaults": {
"links": []
@@ -5375,104 +2481,14 @@
},
"fill": 1,
"fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 31
- },
- "hiddenSeries": false,
- "id": 55,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_rows_merged_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)",
- "legendFormat": "{{type}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Merge speed",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps.",
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
- "y": 39
+ "y": 35
},
"hiddenSeries": false,
- "id": 135,
+ "id": 58,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -5487,12 +2503,13 @@
},
"lines": true,
"linewidth": 1,
+ "links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -5503,12 +2520,14 @@
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
"exemplar": true,
- "expr": "sum(vm_rows_ignored_total{job=~\"$job_storage\", instance=~\"$instance\"}) by (reason)",
+ "expr": "sum(vm_rows_ignored_total{job=~\"$job\", instance=~\"$instance\"}) by (reason)",
+ "format": "time_series",
+ "hide": false,
"interval": "",
+ "intervalFactor": 1,
"legendFormat": "{{reason}}",
"refId": "A"
}
@@ -5531,11 +2550,13 @@
{
"format": "short",
"logBase": 1,
+ "min": "0",
"show": true
},
{
"format": "short",
"logBase": 1,
+ "min": "0",
"show": true
}
],
@@ -5551,7 +2572,7 @@
"datasource": {
"uid": "$ds"
},
- "description": "Data parts of LSM tree.\nHigh number of parts could be an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage(hot data)\n* `storage/big` - small parts gradually merged into big parts (cold data)",
+ "description": "The number of rows merged per second by storage nodes.",
"fieldConfig": {
"defaults": {
"links": []
@@ -5564,10 +2585,107 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 39
+ "y": 35
},
"hiddenSeries": false,
- "id": 22,
+ "id": 64,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_rows_merged_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(type)",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Merge speed ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:867",
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:868",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 43
+ },
+ "hiddenSeries": false,
+ "id": 67,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -5588,7 +2706,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -5599,23 +2717,156 @@
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(vm_parts{job=~\"$job_storage\", instance=~\"$instance\"}) by (type)",
+ "expr": "sum(rate(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (level) ",
"format": "time_series",
- "interval": "",
+ "hide": false,
"intervalFactor": 1,
- "legendFormat": "{{type}}",
+ "legendFormat": "{{level}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "LSM parts ($instance)",
+ "title": "Logging rate ($instance)",
"tooltip": {
"shared": true,
- "sort": 2,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Storage",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 32
+ },
+ "id": 71,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows the rate and total number of new series created over last 24h.\n\nHigh churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 32
+ },
+ "hiddenSeries": false,
+ "id": 66,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "new series over 24h",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "churn rate",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(increase(vm_new_timeseries_created_total{job=~\"$job\", instance=~\"$instance\"}[24h]))",
+ "interval": "",
+ "legendFormat": "new series over 24h",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Churn rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
"value_type": "individual"
},
"type": "graph",
@@ -5651,17 +2902,325 @@
"type": "prometheus",
"uid": "$ds"
},
- "description": "Value > 0 means vmstorage is in readonly mode.",
+ "description": "Shows the rate of adding new items to the index. It should correlate with `Slow inserts` and `Churn rate` graphs and could help to determine the pressure on indexdb.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 32
+ },
+ "hiddenSeries": false,
+ "id": 96,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(vm_indexdb_items_added_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+ "interval": "",
+ "legendFormat": "items",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "IndexDB items rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:76",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:77",
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
- "y": 47
+ "y": 40
},
"hiddenSeries": false,
- "id": 141,
+ "id": 68,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_slow_row_inserts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) / sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "slow inserts percentage",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:60",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 0.1,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Slow inserts ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 2,
+ "format": "percentunit",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Slow queries rate according to `search.logSlowQueryDuration` flag, which is `5s` by default.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 60,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_slow_queries_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "slow queries rate",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Slow queries rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.",
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 48
+ },
+ "hiddenSeries": false,
+ "id": 90,
"legend": {
"avg": false,
"current": false,
@@ -5673,19 +3232,12 @@
},
"lines": true,
"linewidth": 1,
- "links": [
- {
- "targetBlank": true,
- "title": "Readonly mode",
- "url": "https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode"
- }
- ],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "9.0.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -5700,15 +3252,15 @@
"uid": "$ds"
},
"exemplar": true,
- "expr": "vm_storage_is_read_only{job=~\"$job_storage\", instance=~\"$instance\"}",
+ "expr": "vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"} / vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}",
"interval": "",
- "legendFormat": "{{ instance }}",
+ "legendFormat": "{{type}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Readonly mode",
+ "title": "Cache usage % ($instance)",
"tooltip": {
"shared": true,
"sort": 2,
@@ -5722,13 +3274,13 @@
},
"yaxes": [
{
- "$$hashKey": "object:131",
- "format": "short",
+ "$$hashKey": "object:235",
+ "format": "percentunit",
"logBase": 1,
"show": true
},
{
- "$$hashKey": "object:132",
+ "$$hashKey": "object:236",
"format": "short",
"logBase": 1,
"show": true
@@ -5744,380 +3296,9 @@
"dashLength": 10,
"dashes": false,
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU.",
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 47
- },
- "hiddenSeries": false,
- "id": 133,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [
- {
- "targetBlank": true,
- "title": "Related discussion",
- "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632"
- }
- ],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "max",
- "color": "#C4162A"
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "max(vm_concurrent_addrows_capacity{job=~\"$job_storage\", instance=~\"$instance\"})",
- "interval": "",
- "legendFormat": "max",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(avg_over_time(vm_concurrent_addrows_current{job=~\"$job_storage\", instance=~\"$instance\"}[1m])) by(instance)",
- "hide": false,
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Concurrent flushes on disk ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 55
- },
- "hiddenSeries": false,
- "id": 151,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "$$hashKey": "object:186",
- "alias": "max",
- "color": "#C4162A"
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "exemplar": true,
- "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "cores used",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "expr": "sum(process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"})",
- "hide": false,
- "legendFormat": "max",
- "range": true,
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "CPU ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 55
- },
- "hiddenSeries": false,
- "id": 167,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "$$hashKey": "object:186",
- "alias": "max",
- "color": "#C4162A"
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "exemplar": true,
- "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) ",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "mem used",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "expr": "sum(vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"})",
- "hide": false,
- "legendFormat": "max",
- "range": true,
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Memory ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:164",
- "format": "bytes",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:165",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "title": "vmstorage ($instance)",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "uid": "$ds"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 39
- },
- "id": 42,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Request rate accepted by vmselect nodes",
+ "description": "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n\nThis prevents from ingesting metrics with too many labels. The value of `maxLabelsPerTimeseries` must be adjusted for your workload.\n\nWhen limit is exceeded (graph is > 0) - extra labels are dropped, which could result in unexpected identical time series.",
"fieldConfig": {
"defaults": {
"links": []
@@ -6129,32 +3310,30 @@
"gridPos": {
"h": 8,
"w": 12,
- "x": 0,
- "y": 8
+ "x": 12,
+ "y": 48
},
"hiddenSeries": false,
- "id": 92,
+ "id": 74,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
+ "show": false,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
- "nullPointMode": "null as zero",
+ "nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "9.0.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -6165,372 +3344,21 @@
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance.*\", path!~\"/favicon.ico\"}[$__rate_interval])) by (path) > 0",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{path}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Requests rate ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Shows how many ongoing insertions are taking place.\n* `max` - equal to number of CPU * 2 by default. May be configured with `search.maxConcurrentRequests` flag\n* `current` - current number of goroutines busy with processing requests\n\nWhen `current` hits `max` constantly, it means vmselect node is overloaded and require more CPU or higher limits.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 8
- },
- "hiddenSeries": false,
- "id": 95,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "hideEmpty": false,
- "hideZero": false,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "max",
- "color": "#C4162A",
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(max_over_time(vm_concurrent_select_current{job=~\"$job_select\", instance=~\"$instance\"}[1m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "current",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(vm_concurrent_select_capacity{job=~\"$job_select\", instance=~\"$instance\"})",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "max",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Concurrent selects ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 6,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 16
- },
- "hiddenSeries": false,
- "id": 93,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": false,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/read.*/",
- "transform": "negative-Y"
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance) * 8 > 0",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "read - {{instance}}",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance) * 8 > 0",
- "format": "time_series",
- "hide": false,
- "intervalFactor": 1,
- "legendFormat": "write - {{instance}}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Network usage ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "bps",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 23
- },
- "hiddenSeries": false,
- "id": 163,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "$$hashKey": "object:186",
- "alias": "max",
- "color": "#C4162A"
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
"exemplar": true,
- "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))",
+ "expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "time_series",
+ "hide": false,
"interval": "",
"intervalFactor": 1,
- "legendFormat": "cores used",
- "range": true,
+ "legendFormat": "limit exceeded",
"refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "expr": "sum(process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"})",
- "hide": false,
- "legendFormat": "max",
- "range": true,
- "refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "CPU ($instance)",
+ "title": "Labels limit exceeded ($instance)",
"tooltip": {
"shared": true,
"sort": 0,
@@ -6542,562 +3370,6 @@
"show": true,
"values": []
},
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 23
- },
- "hiddenSeries": false,
- "id": 165,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "$$hashKey": "object:186",
- "alias": "max",
- "color": "#C4162A"
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "exemplar": true,
- "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job_select\", instance=~\"$instance\"}) ",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "mem used",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "expr": "sum(vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"})",
- "hide": false,
- "legendFormat": "max",
- "range": true,
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Memory ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:164",
- "format": "bytes",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:165",
- "format": "short",
- "logBase": 1,
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- }
- ],
- "title": "vmselect ($instance)",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "uid": "$ds"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 40
- },
- "id": 40,
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 160
- },
- "hiddenSeries": false,
- "id": 97,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(rate(vm_http_requests_total{job=~\"$job_insert\", instance=~\"$instance.*\", path!~\"/favicon.ico\"}[$__rate_interval])) by (path) > 0",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{path}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Requests rate ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Shows how many ongoing insertions are taking place.\n* `max` - equal to number of CPU * 4 by default. May be configured with `maxConcurrentInserts` flag;\n* `current` - current number of goroutines busy with processing requests.\n\n`-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate.\n\nWhen `current` hits `max` constantly, it means vminsert node is overloaded and requires more CPU or higher limits.",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 160
- },
- "hiddenSeries": false,
- "id": 99,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "hideEmpty": false,
- "hideZero": false,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "max",
- "color": "#C4162A",
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "sum(vm_concurrent_insert_current{job=~\"$job_insert\", instance=~\"$instance\"})",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "current",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(vm_concurrent_insert_capacity{job=~\"$job_insert\", instance=~\"$instance\"})",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "max",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Concurrent inserts ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 168
- },
- "hiddenSeries": false,
- "id": 90,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (instance) * 8 > 0",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Network usage ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "bps",
- "logBase": 1,
- "show": true
- },
- {
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 168
- },
- "hiddenSeries": false,
- "id": 88,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Rows per insert ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
"yaxes": [
{
"decimals": 2,
@@ -7116,7 +3388,32 @@
"yaxis": {
"align": false
}
- },
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Troubleshooting",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 33
+ },
+ "id": 46,
+ "panels": [
{
"aliasColors": {},
"bars": false,
@@ -7125,7 +3422,7 @@
"datasource": {
"uid": "$ds"
},
- "description": "Shows the saturation level of connection between vminsert and vmstorage components. If the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that more vminsert or vmstorage nodes must be added to the cluster in order to increase the total number of vminsert -> vmstorage links.\n",
+ "description": "",
"fieldConfig": {
"defaults": {
"links": []
@@ -7138,225 +3435,10 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 176
+ "y": 5
},
"hiddenSeries": false,
- "id": 139,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": false,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "exemplar": true,
- "expr": "rate(vm_rpc_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{instance}} => {{addr}}",
- "refId": "A"
- }
- ],
- "thresholds": [
- {
- "$$hashKey": "object:234",
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 0.9,
- "yaxis": "left"
- }
- ],
- "timeRegions": [],
- "title": "Storage connection saturation ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:156",
- "decimals": 0,
- "format": "s",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:157",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "uid": "$ds"
- },
- "description": "Shows if vmstorage node is reachable for vminsert. If below 1 means vmstorage is not reachable at this moment of time. ",
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 176
- },
- "hiddenSeries": false,
- "id": 114,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": false,
- "min": false,
- "show": true,
- "sort": "current",
- "sortDesc": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null as zero",
- "options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "8.5.3",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "expr": "vm_rpc_vmstorage_is_reachable{job=~\"$job\", instance=~\"$instance\"}",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{instance}} => {{addr}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeRegions": [],
- "title": "Storage reachability ($instance)",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "mode": "time",
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:1534",
- "decimals": 0,
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:1535",
- "format": "short",
- "logBase": 1,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "fieldConfig": {
- "defaults": {
- "links": []
- },
- "overrides": []
- },
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 184
- },
- "hiddenSeries": false,
- "id": 164,
+ "id": 44,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -7377,15 +3459,161 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "requested from system",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "heap inuse",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "stack inuse",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "resident",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "exemplar": true,
+ "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "resident anonymous",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory usage ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 5
+ },
+ "hiddenSeries": false,
+ "id": 57,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.5",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
- "$$hashKey": "object:186",
- "alias": "max",
- "color": "#C4162A"
+ "$$hashKey": "object:85",
+ "alias": "Limit",
+ "color": "#F2495C"
}
],
"spaceLength": 10,
@@ -7397,14 +3625,11 @@
"type": "prometheus",
"uid": "$ds"
},
- "editorMode": "code",
- "exemplar": true,
- "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))",
+ "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
- "legendFormat": "cores used",
- "range": true,
+ "legendFormat": "CPU cores used",
"refId": "A"
},
{
@@ -7412,11 +3637,13 @@
"type": "prometheus",
"uid": "$ds"
},
- "editorMode": "code",
- "expr": "sum(process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"})",
+ "exemplar": false,
+ "expr": "process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}",
+ "format": "time_series",
"hide": false,
- "legendFormat": "max",
- "range": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Limit",
"refId": "B"
}
],
@@ -7436,16 +3663,15 @@
},
"yaxes": [
{
- "$$hashKey": "object:887",
"format": "short",
"logBase": 1,
"min": "0",
"show": true
},
{
- "$$hashKey": "object:888",
"format": "short",
"logBase": 1,
+ "min": "0",
"show": true
}
],
@@ -7459,25 +3685,25 @@
"dashLength": 10,
"dashes": false,
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
+ "description": "Panel shows the number of open file descriptors in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a",
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
- "fill": 0,
+ "fill": 1,
"fillGradient": 0,
"gridPos": {
- "h": 7,
+ "h": 8,
"w": 12,
- "x": 12,
- "y": 184
+ "x": 0,
+ "y": 13
},
"hiddenSeries": false,
- "id": 169,
+ "id": 75,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -7498,13 +3724,12 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "8.5.3",
+ "pluginVersion": "8.3.5",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
- "$$hashKey": "object:186",
"alias": "max",
"color": "#C4162A"
}
@@ -7515,35 +3740,147 @@
"targets": [
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "editorMode": "code",
- "exemplar": true,
- "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job_insert\", instance=~\"$instance\"}) ",
+ "expr": "sum(process_open_fds{job=~\"$job\", instance=~\"$instance\"})",
"format": "time_series",
"interval": "",
- "intervalFactor": 1,
- "legendFormat": "mem used",
- "range": true,
+ "intervalFactor": 2,
+ "legendFormat": "open",
"refId": "A"
},
{
"datasource": {
- "type": "prometheus",
"uid": "$ds"
},
- "editorMode": "code",
- "expr": "sum(vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"})",
- "hide": false,
+ "expr": "min(process_max_fds{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
"legendFormat": "max",
- "range": true,
"refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
- "title": "Memory ($instance)",
+ "title": "Open FDs ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 2,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows the number of bytes read/write from the storage layer.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 13
+ },
+ "hiddenSeries": false,
+ "id": 76,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "read",
+ "transform": "negative-Y"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "read",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "write",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Disk writes/reads ($instance)",
"tooltip": {
"shared": true,
"sort": 0,
@@ -7557,16 +3894,501 @@
},
"yaxes": [
{
- "$$hashKey": "object:164",
"format": "bytes",
"logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 21
+ },
+ "hiddenSeries": false,
+ "id": 47,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "gc duration",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Goroutines ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
"min": "0",
"show": true
},
{
- "$$hashKey": "object:165",
"format": "short",
"logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "Shows avg GC duration",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 21
+ },
+ "hiddenSeries": false,
+ "id": 42,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(go_gc_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))\n/\nsum(rate(go_gc_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "avg gc duration",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "GC duration ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 48,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "threads",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Threads ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 0,
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 37,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"})",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "connections",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "TCP connections ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$ds"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 37
+ },
+ "hiddenSeries": false,
+ "id": 49,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": false,
+ "show": true,
+ "sort": "current",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.3.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__interval]))",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "connections",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "TCP connections rate ($instance)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
"show": true
}
],
@@ -7575,13 +4397,25 @@
}
}
],
- "title": "vminsert ($instance)",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$ds"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Resource usage",
"type": "row"
}
],
+ "refresh": "30s",
"schemaVersion": 36,
"style": "dark",
- "tags": [],
+ "tags": [
+ "victoriametrics",
+ "vmsingle"
+ ],
"templating": {
"list": [
{
@@ -7607,86 +4441,14 @@
"datasource": {
"uid": "$ds"
},
- "definition": "label_values(vm_app_version{version=~\"^vminsert.*\"}, job)",
- "hide": 2,
- "includeAll": false,
- "multi": false,
- "name": "job_insert",
- "options": [],
- "query": {
- "query": "label_values(vm_app_version{version=~\"^vminsert.*\"}, job)",
- "refId": "VictoriaMetrics-job_insert-Variable-Query"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {},
- "datasource": {
- "uid": "$ds"
- },
- "definition": "label_values(vm_app_version{version=~\"^vmselect.*\"}, job)",
- "hide": 2,
- "includeAll": false,
- "multi": false,
- "name": "job_select",
- "options": [],
- "query": {
- "query": "label_values(vm_app_version{version=~\"^vmselect.*\"}, job)",
- "refId": "VictoriaMetrics-job_select-Variable-Query"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {},
- "datasource": {
- "uid": "$ds"
- },
- "definition": "label_values(vm_app_version{version=~\"^vmstorage.*\"}, job)",
- "hide": 2,
- "includeAll": false,
- "multi": false,
- "name": "job_storage",
- "options": [],
- "query": {
- "query": "label_values(vm_app_version{version=~\"^vmstorage.*\"}, job)",
- "refId": "VictoriaMetrics-job_storage-Variable-Query"
- },
- "refresh": 1,
- "regex": "",
- "skipUrlSync": false,
- "sort": 0,
- "tagValuesQuery": "",
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {},
- "datasource": {
- "uid": "$ds"
- },
- "definition": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)",
+ "definition": "label_values(vm_app_version{version=~\"victoria-metrics-.*\"}, job)",
"hide": 0,
- "includeAll": true,
- "multi": true,
+ "includeAll": false,
+ "multi": false,
"name": "job",
"options": [],
"query": {
- "query": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)",
+ "query": "label_values(vm_app_version{version=~\"victoria-metrics-.*\"}, job)",
"refId": "VictoriaMetrics-job-Variable-Query"
},
"refresh": 1,
@@ -7699,15 +4461,38 @@
"useTags": false
},
{
- "allValue": ".*",
+ "current": {},
+ "datasource": {
+ "uid": "$ds"
+ },
+ "definition": "label_values(vm_app_version{job=~\"$job\", instance=~\"$instance\"}, version)",
+ "hide": 2,
+ "includeAll": false,
+ "multi": false,
+ "name": "version",
+ "options": [],
+ "query": {
+ "query": "label_values(vm_app_version{job=~\"$job\", instance=~\"$instance\"}, version)",
+ "refId": "VictoriaMetrics-version-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "/.*-tags-(v\\d+\\.\\d+\\.\\d+)/",
+ "skipUrlSync": false,
+ "sort": 2,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
"current": {},
"datasource": {
"uid": "$ds"
},
"definition": "label_values(vm_app_version{job=~\"$job\"}, instance)",
"hide": 0,
- "includeAll": true,
- "multi": true,
+ "includeAll": false,
+ "multi": false,
"name": "instance",
"options": [],
"query": {
@@ -7718,7 +4503,10 @@
"regex": "",
"skipUrlSync": false,
"sort": 0,
- "type": "query"
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
},
{
"datasource": {
@@ -7748,11 +4536,22 @@
"1h",
"2h",
"1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
]
},
"timezone": "",
- "title": "VictoriaMetrics - cluster",
- "uid": "oS7Bi_0Wz",
+ "title": "VictoriaMetrics",
+ "uid": "wNf0q_kZk",
"version": 1,
"weekStart": ""
}
\ No newline at end of file
diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml
index 24eb69f778..6c00ed7df7 100644
--- a/deployment/docker/docker-compose.yml
+++ b/deployment/docker/docker-compose.yml
@@ -26,7 +26,7 @@ services:
volumes:
- grafanadata:/var/lib/grafana
- ./provisioning/:/etc/grafana/provisioning/
- - ./../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json
+ - ./../../dashboards/victoriametrics-cluster.json:/var/lib/grafana/dashboards/vm.json
- ./../../dashboards/vmagent.json:/var/lib/grafana/dashboards/vmagent.json
- ./../../dashboards/vmalert.json:/var/lib/grafana/dashboards/vmalert.json
diff --git a/docs/PerTenantStatistic.md b/docs/PerTenantStatistic.md
index e798a7e119..e97ffb09e2 100644
--- a/docs/PerTenantStatistic.md
+++ b/docs/PerTenantStatistic.md
@@ -46,7 +46,7 @@ scrape_configs:
## Visualization
Visualisation of statistics can be done in Grafana using the following
-[dashboard](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster/dashboards/clusterbytenant.json).
+[dashboard](https://grafana.com/grafana/dashboards/16399-victoriametrics-cluster-per-tenant-statistic/).
## Integration with vmgateway