{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "- if not filtered, shows global percentage of anomalies\n-if filtered by Node, shows % of anomalous indicators", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "noValue": "No anomalies found", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 0 }, "id": 145, "options": { "legend": { "calcs": [ "last" ], "displayMode": "table", "placement": "right", "showLegend": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "count((min(anomaly_score{preset=\"node-exporter\",instance=~\"$node\"}) by (for, instance))>1)/count((min(anomaly_score{preset=\"node-exporter\",instance=~\"$node\"}) by (for, instance)))", "hide": false, "legendFormat": "Percentage", "range": true, "refId": "A" } ], "title": "Percentage of Anomalous Nodes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Count of anomaly scores greater than threshold", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [], "noValue": "No anomalies found", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 0 }, "id": 146, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false }, "mergeValues": true, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "WITH(f={preset=\"node-exporter\"}) count(count(min(anomaly_score{f}) without (model_alias,scheduler_alias)>=1.0) by (for)) by (for)/count(count by (for) (anomaly_score{f})) by (for)", "format": "time_series", "hide": false, "legendFormat": "{{for}}", "range": true, "refId": "A" } ], "title": "Anomalies per Indicator", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Number of anomalous indicators per node", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 10 }, "id": 224, "options": { "legend": { "calcs": [ "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "count((min(anomaly_score{preset=\"node-exporter\",instance=~\"$node\"}) without (model_alias,scheduler_alias))>=1.0) by (instance)", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Numbers of Anomalous Indicators by Node", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 19 }, "id": 88, "panels": [], "title": "Basic CPU / Mem / Net / Disk", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 20 }, "id": 6, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "WITH(f={instance=~\"$node\",for=\"receive_bytes\",preset=\"node-exporter\"}) count(min(anomaly_score{f}) without (model_alias,scheduler_alias)>=1.0) by (instance)", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Anomalies: Network Receive Bytes", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 20 }, "id": 94, "options": { "legend": { "calcs": [ "lastNotNull", "mean", "min", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "irate(node_network_receive_bytes_total{instance=~\"$node\",job=~\"$job\"}[$__rate_interval])*8", "legendFormat": "{{instance}},{{device}}", "range": true, "refId": "A" } ], "title": "Network Traffic- Receive Bytes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 30 }, "id": 156, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "WITH(f={instance=~\"$node\",for=\"transmit_bytes\",preset=\"node-exporter\"}) count(min(anomaly_score{f}) without (model_alias,scheduler_alias)>=1.0) by (instance)", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Anomalies: Network Transmit Bytes", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 30 }, "id": 157, "options": { "legend": { "calcs": [ "lastNotNull", "mean", "min", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Name", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "irate(node_network_transmit_bytes_total{instance=~\"$node\",job=~\"$job\"}[$__rate_interval])*8", "hide": false, "legendFormat": "{{instance}},{{device}}", "range": true, "refId": "B" } ], "title": "Network Traffic - Transmit Bytes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, "id": 111, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "# Title\n\nFor markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)", "mode": "markdown" }, "pluginVersion": "10.4.0", "title": "CPU seconds", "type": "text" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, "id": 158, "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Sudden spike of Node CPU seconds", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 }, "id": 22, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "WITH(f={instance=~\"$node\",for=\"cpu_seconds_total\",preset=\"node-exporter\",mode=~\"$cpu_mode\"}) count(min(anomaly_score{f}) without (model_alias,scheduler_alias)>=1.0) by (instance)", "hide": false, "legendFormat": "{{instance}}", "range": true, "refId": "B" } ], "title": "Anomalies: CPU Seconds: Mode $cpu_mode", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "percent" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 }, "id": 84, "options": { "legend": { "calcs": [ "lastNotNull", "mean", "min", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=~\"$node\",job=~\"$job\", mode=~\"$cpu_mode\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=~\"$node\",job=~\"$job\"}[$__rate_interval])))", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "CPU Basic", "type": "timeseries" } ], "repeat": "cpu_mode", "repeatDirection": "h", "title": "CPU seconds. Mode: $cpu_mode", "type": "row" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 49 }, "id": 78, "panels": [], "title": "Storage disk", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, "id": 114, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "# Title\n\nFor markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)", "mode": "markdown" }, "pluginVersion": "10.4.0", "title": "Latency", "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Sudden drop or spike of Read latency", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "dark-red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 51 }, "id": 18, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "WITH(f={instance=~\"$node\",for=\"read_latency\",preset=\"node-exporter\"}) count(min(anomaly_score{f}) without (model_alias,scheduler_alias)>=1.0) by (instance)", "hide": false, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Anoamlies: Read Latency", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 51 }, "id": 74, "options": { "legend": { "calcs": [ "lastNotNull", "mean", "max", "min" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "irate(node_disk_read_time_seconds_total{instance=~\"$node\",job=~\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=~\"$node\",job=~\"$job\"}[$__rate_interval])", "legendFormat": "{{instance}},{{device}}", "range": true, "refId": "A" } ], "title": "Disk Average Wait Time: Read", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Sudden drop or spike of Write latency", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 61 }, "id": 20, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "WITH(f={instance=~\"$node\",for=\"write_latency\",preset=\"node-exporter\"}) count(min(anomaly_score{f}) without (model_alias,scheduler_alias)>=1.0) by (instance)", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Anomalies: Write Latency", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 61 }, "id": 180, "options": { "legend": { "calcs": [ "lastNotNull", "mean", "max", "min" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "irate(node_disk_write_time_seconds_total{instance=~\"$node\",job=~\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=~\"$node\",job=~\"$job\"}[$__rate_interval])", "hide": false, "legendFormat": "{{instance}},{{device}}", "range": true, "refId": "B" } ], "title": "Disk Average Wait Time: Write", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 71 }, "id": 62, "panels": [], "title": "System Misc", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Sudden spike of Context Switch", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 72 }, "id": 24, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "WITH(f={for=\"context_switch\",preset=\"node-exporter\",instance=~\"$node\",job=~\"$job\"}) count(min(anomaly_score{f}) without (model_alias,scheduler_alias)>=1.0) by (instance)", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Anomalies: Context Switch", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 72 }, "id": 66, "options": { "legend": { "calcs": [ "lastNotNull", "mean", "max", "min" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "irate(node_context_switches_total{instance=~\"$node\",job=~\"$job\"}[$__rate_interval])", "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Context Switches", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 82 }, "id": 40, "panels": [], "title": "Network Traffic", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Sudden drop or spike of Host Network Transmit Errors", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 83 }, "id": 32, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "WITH(f={for=\"host_network_transmit_errors\",preset=\"node-exporter\",instance=~\"$node\",job=~\"$job\"}) count(min(anomaly_score{f}) without (model_alias,scheduler_alias)>=1.0) by (instance)", "legendFormat": "Value - {{instance}}", "range": true, "refId": "A" } ], "title": "Anomalies: Host Network Transmit Errors", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "packets out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "pps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 83 }, "id": 52, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max", "min" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "irate(node_network_receive_errs_total{instance=~\"$node\",job=~\"$job\"}[$__rate_interval])", "legendFormat": "{{instance}},{{device}}", "range": true, "refId": "A" } ], "title": "Network Receive Traffic Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Sudden drop or spike of Host Network Receive Errors", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 93 }, "id": 30, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "WITH(f={for=\"host_network_receive_errors\",preset=\"node-exporter\",instance=~\"$node\",job=~\"$job\"}) count(min(anomaly_score{f}) without (model_alias,scheduler_alias)>=1.0) by (instance)", "legendFormat": "Value - {{instance}}", "range": true, "refId": "A" } ], "title": "Anomalies: Host Network Receive Errors", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "packets out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "pps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 93 }, "id": 202, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max", "min" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "irate(node_network_transmit_errs_total{instance=~\"$node\",job=~\"$job\"}[$__rate_interval])", "hide": false, "legendFormat": "{{instance}},{{device}}", "range": true, "refId": "B" } ], "title": "Network Transmit Traffic Errors", "type": "timeseries" } ], "refresh": "", "schemaVersion": 39, "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "sandbox-monitoring", "value": "PB894574A363DF0AF" }, "hide": 0, "includeAll": false, "label": "datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".*", "current": {}, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(node_uname_info{job=~\"$job\"}, instance)", "hide": 0, "includeAll": true, "label": "Host", "multi": true, "name": "node", "options": [], "query": { "query": "label_values(node_uname_info{job=~\"$job\"}, instance)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "allValue": ".*", "current": {}, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(node_uname_info, job)", "hide": 0, "includeAll": true, "label": "Job", "multi": true, "name": "job", "options": [], "query": { "query": "label_values(node_uname_info, job)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": { "selected": false, "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" }, "hide": 2, "includeAll": false, "multi": false, "name": "diskdevices", "options": [ { "selected": true, "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" } ], "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", "skipUrlSync": false, "type": "custom" }, { "allValue": ".*", "current": {}, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(anomaly_score{for=\"cpu_seconds_total\"},mode)", "hide": 2, "includeAll": true, "label": "cpu_mode", "multi": true, "name": "cpu_mode", "options": [], "query": { "qryType": 1, "query": "label_values(anomaly_score{for=\"cpu_seconds_total\"},mode)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" } ] }, "time": { "from": "now-2d", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Node Exporter - vmanomaly", "uid": "feceb53e-c252-44aa-ae7a-51c20f58cd241", "version": 21, "weekStart": "" }