diff --git a/dashboards/vm/vmagent.json b/dashboards/vm/vmagent.json index a15e64299..3928c3248 100644 --- a/dashboards/vm/vmagent.json +++ b/dashboards/vm/vmagent.json @@ -1271,7 +1271,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1373,7 +1374,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2388,7 +2390,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 36 }, "id": 92, "options": { @@ -2491,7 +2493,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 36 }, "id": 95, "options": { @@ -2597,7 +2599,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 44 }, "id": 98, "options": { @@ -2703,7 +2705,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 44 }, "id": 99, "options": { @@ -2808,7 +2810,7 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 52 }, "id": 79, "links": [], @@ -2914,7 +2916,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 52 }, "id": 18, "links": [ @@ -3025,7 +3027,7 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 60 }, "id": 127, "links": [], @@ -3129,7 +3131,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 60 }, "id": 50, "options": { @@ -3232,7 +3234,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 68 }, "id": 130, "links": [], @@ -3284,110 +3286,6 @@ "title": "Concurrent inserts ($instance)", "type": "timeseries" }, - { - "datasource": { - "type": "victoriametrics-datasource", - "uid": "$ds" - }, - "description": "Shows the number of skipped scrapes.\n\nScrapes could be skipped due to\n1. vmagent overload;\n2. too small scrape interval for configured job or target.\n\nVerify that vmagent has enough CPU/Mem resources, network bandwidth (try increasing `-remoteWrite.queues`).\n\nCheck `Scrape duration` pane to measure the scrape duration. If vmagent has enough resources and scrape duration is very close or exceeds scrape interval - try increasing the scrape interval or debugging why scraped target is responding too slowly.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 36 - }, - "id": 132, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "victoriametrics-datasource", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(vm_promscrape_scrapes_skipped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", - "interval": "", - "legendFormat": "{{instance}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "Skipped scrapes ($instance)", - "type": "timeseries" - }, { "datasource": { "type": "victoriametrics-datasource", @@ -3449,7 +3347,7 @@ "h": 7, "w": 24, "x": 0, - "y": 44 + "y": 76 }, "id": 129, "options": { @@ -3586,8 +3484,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3603,7 +3500,7 @@ "h": 7, "w": 12, "x": 0, - "y": 52 + "y": 45 }, "id": 48, "options": { @@ -3691,8 +3588,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3708,7 +3604,7 @@ "h": 7, "w": 12, "x": 12, - "y": 52 + "y": 45 }, "id": 76, "options": { @@ -3795,8 +3691,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3812,7 +3707,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 52 }, "id": 20, "options": { @@ -3898,8 +3793,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3915,7 +3809,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 52 }, "id": 126, "options": { @@ -4000,8 +3894,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4017,7 +3910,7 @@ "h": 8, "w": 12, "x": 0, - "y": 67 + "y": 60 }, "id": 46, "options": { @@ -4055,109 +3948,6 @@ "title": "Scrape response size 0.99 quantile ($instance)", "type": "timeseries" }, - { - "datasource": { - "type": "victoriametrics-datasource", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 133, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "victoriametrics-datasource", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)", - "format": "time_series", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Scrape duration 0.99 quantile ($instance)", - "type": "timeseries" - }, { "datasource": { "type": "victoriametrics-datasource", @@ -4205,8 +3995,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4222,7 +4011,7 @@ "h": 8, "w": 12, "x": 12, - "y": 75 + "y": 60 }, "id": 31, "options": { @@ -4383,7 +4172,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4499,7 +4289,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4602,7 +4393,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", diff --git a/dashboards/vmagent.json b/dashboards/vmagent.json index d84d4ff62..de0502665 100644 --- a/dashboards/vmagent.json +++ b/dashboards/vmagent.json @@ -1270,7 +1270,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1372,7 +1373,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2387,7 +2389,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 36 }, "id": 92, "options": { @@ -2490,7 +2492,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 36 }, "id": 95, "options": { @@ -2596,7 +2598,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 44 }, "id": 98, "options": { @@ -2702,7 +2704,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 44 }, "id": 99, "options": { @@ -2807,7 +2809,7 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 52 }, "id": 79, "links": [], @@ -2913,7 +2915,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 52 }, "id": 18, "links": [ @@ -3024,7 +3026,7 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 60 }, "id": 127, "links": [], @@ -3128,7 +3130,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 60 }, "id": 50, "options": { @@ -3231,7 +3233,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 68 }, "id": 130, "links": [], @@ -3283,110 +3285,6 @@ "title": "Concurrent inserts ($instance)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of skipped scrapes.\n\nScrapes could be skipped due to\n1. vmagent overload;\n2. too small scrape interval for configured job or target.\n\nVerify that vmagent has enough CPU/Mem resources, network bandwidth (try increasing `-remoteWrite.queues`).\n\nCheck `Scrape duration` pane to measure the scrape duration. If vmagent has enough resources and scrape duration is very close or exceeds scrape interval - try increasing the scrape interval or debugging why scraped target is responding too slowly.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 36 - }, - "id": 132, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(vm_promscrape_scrapes_skipped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", - "interval": "", - "legendFormat": "{{instance}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "Skipped scrapes ($instance)", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -3448,7 +3346,7 @@ "h": 7, "w": 24, "x": 0, - "y": 44 + "y": 76 }, "id": 129, "options": { @@ -3585,8 +3483,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3602,7 +3499,7 @@ "h": 7, "w": 12, "x": 0, - "y": 52 + "y": 45 }, "id": 48, "options": { @@ -3690,8 +3587,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3707,7 +3603,7 @@ "h": 7, "w": 12, "x": 12, - "y": 52 + "y": 45 }, "id": 76, "options": { @@ -3794,8 +3690,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3811,7 +3706,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 52 }, "id": 20, "options": { @@ -3897,8 +3792,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3914,7 +3808,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 52 }, "id": 126, "options": { @@ -3999,8 +3893,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4016,7 +3909,7 @@ "h": 8, "w": 12, "x": 0, - "y": 67 + "y": 60 }, "id": 46, "options": { @@ -4054,109 +3947,6 @@ "title": "Scrape response size 0.99 quantile ($instance)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 133, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)", - "format": "time_series", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Scrape duration 0.99 quantile ($instance)", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -4204,8 +3994,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4221,7 +4010,7 @@ "h": 8, "w": 12, "x": 12, - "y": 75 + "y": 60 }, "id": 31, "options": { @@ -4382,7 +4171,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4498,7 +4288,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4601,7 +4392,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", diff --git a/deployment/docker/alerts-vmagent.yml b/deployment/docker/alerts-vmagent.yml index 220121107..b4f4ee9e0 100644 --- a/deployment/docker/alerts-vmagent.yml +++ b/deployment/docker/alerts-vmagent.yml @@ -132,19 +132,3 @@ groups: summary: "Configuration reload failed for vmagent instance {{ $labels.instance }}" description: "Configuration hot-reload failed for vmagent on instance {{ $labels.instance }}. Check vmagent's logs for detailed error message." - - - alert: TooManyScrapeSkips - expr: | - rate(vm_promscrape_scrapes_skipped_total[5m]) > 0 - for: 15m - labels: - severity: warning - annotations: - dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=132&var-instance={{ $labels.instance }}" - summary: "vmagent on instance {{ $labels.instance }} is skipping scrapes" - description: "Scrape intervals are skipped for vmagent on instance {{ $labels.instance }} - due to either overload or too small scrape interval. Verify that vmagent has enough CPU/Mem resources - and network bandwidth (try increasing -remoteWrite.queues). - Check vm_promscrape_scrape_duration_seconds metric to measure the scrape duration. If vmagent has enough - resources and scrape duration is very close or exceeds scrape interval - try increasing the scrape interval - or debuggin why scraped target is responding too slow;y." \ No newline at end of file diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7c7c1767e..8af3d72cd 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -35,7 +35,6 @@ The sandbox cluster installation is running under the constant load generated by * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not log `unexpected EOF` when reading incoming metrics, since this error is expected and is handled during metrics' parsing. This reduces the amounts of noisy logs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4817). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): retry failed write request on the closed connection immediately, without waiting for backoff. This should improve data delivery speed and reduce amount of error logs emitted by vmagent when using idle connections. See related [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): reduces load on Kubernetes control plane during initial service discovery. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4855) for details. -* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add metric `vm_promscrape_scrapes_skipped_total` to show whether vmagent skips the scrapes. This could happen if vmagent is overloaded or target is responding too slow for configured `scrape_interval`. * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): reduce the maximum recovery time at `vmselect` and `vminsert` when some of `vmstorage` nodes become unavailable because of networking issues from 60 seconds to 3 seconds by default. The recovery time can be tuned at `vmselect` and `vminsert` nodes with `-vmstorageUserTimeout` command-line flag if needed. Thanks to @wjordan for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4423). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add Prometheus data support to the "Explore cardinality" page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4320) for details. * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): make the warning message more noticeable for text fields. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4848). @@ -44,7 +43,6 @@ The sandbox cluster installation is running under the constant load generated by * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): organize `min`, `max`, `median` values on the chart legend and tooltips for better visibility. * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add explanation about [cardinality explorer](https://docs.victoriametrics.com/#cardinality-explorer) statistic inaccuracy in VictoriaMetrics cluster. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3070). * FEATURE: dashboards: provide copies of Grafana dashboards alternated with VictoriaMetrics datasource at [dashboards/vm](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards/vm). -* FEATURE: dashboards: add panels `Scrape duration 0.99 quantile` and `Skipped scrapes` to vmagent dashboard. * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): added ability to set, override and clear request and response headers on a per-user and per-path basis. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4825) and [these docs](https://docs.victoriametrics.com/vmauth.html#auth-config) for details. * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to retry requests to the [remaining backends](https://docs.victoriametrics.com/vmauth.html#load-balancing) if they return response status codes specified in the `retry_status_codes` list. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): expose metrics `vmauth_config_last_reload_*` for tracking the state of config reloads, similarly to vmagent/vmalert components. diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index 737e5a3dc..3a23c8655 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -326,7 +326,6 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}, globalStopCh <-chan struct{}) sw.scrapeAndLogError(timestamp, timestamp) } defer ticker.Stop() - for { timestamp += scrapeInterval.Milliseconds() select { @@ -351,16 +350,9 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}, globalStopCh <-chan struct{}) return case tt := <-ticker.C: t := tt.UnixNano() / 1e6 - if d := math.Abs(float64(t - timestamp)); d > 0 { - intervalDelay := d / float64(scrapeInterval.Milliseconds()) - if intervalDelay > 0.1 { - // Too big jitter. Adjust timestamp - timestamp = t - } - if intervalDelay >= 1 { - skipped := math.Floor(intervalDelay) - scrapesSkipped.Add(int(skipped)) - } + if d := math.Abs(float64(t - timestamp)); d > 0 && d/float64(scrapeInterval.Milliseconds()) > 0.1 { + // Too big jitter. Adjust timestamp + timestamp = t } sw.scrapeAndLogError(timestamp, t) } @@ -403,7 +395,6 @@ var ( scrapeDuration = metrics.NewHistogram("vm_promscrape_scrape_duration_seconds") scrapeResponseSize = metrics.NewHistogram("vm_promscrape_scrape_response_size_bytes") scrapedSamples = metrics.NewHistogram("vm_promscrape_scraped_samples") - scrapesSkipped = metrics.NewCounter("vm_promscrape_scrapes_skipped_total") scrapesSkippedBySampleLimit = metrics.NewCounter("vm_promscrape_scrapes_skipped_by_sample_limit_total") scrapesFailed = metrics.NewCounter("vm_promscrape_scrapes_failed_total") pushDataDuration = metrics.NewHistogram("vm_promscrape_push_data_duration_seconds")