diff --git a/app/vmagent/csvimport/request_handler.go b/app/vmagent/csvimport/request_handler.go index e17207c0d8..00d9846d75 100644 --- a/app/vmagent/csvimport/request_handler.go +++ b/app/vmagent/csvimport/request_handler.go @@ -9,6 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/metrics" ) @@ -25,7 +26,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error { if err != nil { return err } - return parser.ParseStream(req, func(rows []parser.Row) error { + return stream.Parse(req, func(rows []parser.Row) error { return insertRows(at, rows, extraLabels) }) } diff --git a/app/vmagent/datadog/request_handler.go b/app/vmagent/datadog/request_handler.go index adac86be3c..056d3bcb0b 100644 --- a/app/vmagent/datadog/request_handler.go +++ b/app/vmagent/datadog/request_handler.go @@ -9,6 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/metrics" ) @@ -28,7 +29,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error { return err } ce := req.Header.Get("Content-Encoding") - return parser.ParseStream(req.Body, ce, func(series []parser.Series) error { + return stream.Parse(req.Body, ce, func(series []parser.Series) error { return insertRows(at, series, extraLabels) }) } diff --git a/app/vmagent/graphite/request_handler.go b/app/vmagent/graphite/request_handler.go index d9ff2a1b20..2c81806677 100644 --- a/app/vmagent/graphite/request_handler.go +++ b/app/vmagent/graphite/request_handler.go @@ -7,6 +7,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream" "github.com/VictoriaMetrics/metrics" ) @@ -19,7 +20,7 @@ var ( // // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol func InsertHandler(r io.Reader) error { - return parser.ParseStream(r, insertRows) + return stream.Parse(r, insertRows) } func insertRows(rows []parser.Row) error { diff --git a/app/vmagent/influx/request_handler.go b/app/vmagent/influx/request_handler.go index d7211e385b..7ff651d5b9 100644 --- a/app/vmagent/influx/request_handler.go +++ b/app/vmagent/influx/request_handler.go @@ -15,6 +15,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/metrics" ) @@ -36,7 +37,7 @@ var ( // // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/ func InsertHandlerForReader(r io.Reader, isGzipped bool) error { - return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error { + return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error { return insertRows(nil, db, rows, nil) }) } @@ -54,7 +55,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error { precision := q.Get("precision") // Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint db := q.Get("db") - return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error { + return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error { return insertRows(at, db, rows, extraLabels) }) } diff --git a/app/vmagent/native/request_handler.go b/app/vmagent/native/request_handler.go index 898394ae83..60347b0dde 100644 --- a/app/vmagent/native/request_handler.go +++ b/app/vmagent/native/request_handler.go @@ -10,7 +10,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" - parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/metrics" ) @@ -30,12 +30,12 @@ func InsertHandler(at *auth.Token, req *http.Request) error { return err } isGzip := req.Header.Get("Content-Encoding") == "gzip" - return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error { + return stream.Parse(req.Body, isGzip, func(block *stream.Block) error { return insertRows(at, block, extraLabels) }) } -func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal.Label) error { +func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error { ctx := common.GetPushCtx() defer common.PutPushCtx(ctx) diff --git a/app/vmagent/opentsdb/request_handler.go b/app/vmagent/opentsdb/request_handler.go index e1e42ba4bc..8388a5238a 100644 --- a/app/vmagent/opentsdb/request_handler.go +++ b/app/vmagent/opentsdb/request_handler.go @@ -7,6 +7,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream" "github.com/VictoriaMetrics/metrics" ) @@ -19,7 +20,7 @@ var ( // // See http://opentsdb.net/docs/build/html/api_telnet/put.html func InsertHandler(r io.Reader) error { - return parser.ParseStream(r, insertRows) + return stream.Parse(r, insertRows) } func insertRows(rows []parser.Row) error { diff --git a/app/vmagent/opentsdbhttp/request_handler.go b/app/vmagent/opentsdbhttp/request_handler.go index 968ea936ba..7fecea6b34 100644 --- a/app/vmagent/opentsdbhttp/request_handler.go +++ b/app/vmagent/opentsdbhttp/request_handler.go @@ -9,6 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream" "github.com/VictoriaMetrics/metrics" ) @@ -24,7 +25,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error { if err != nil { return err } - return parser.ParseStream(req, func(rows []parser.Row) error { + return stream.Parse(req, func(rows []parser.Row) error { return insertRows(at, rows, extraLabels) }) } diff --git a/app/vmagent/prometheusimport/request_handler.go b/app/vmagent/prometheusimport/request_handler.go index ed5d53a850..2f450e1c29 100644 --- a/app/vmagent/prometheusimport/request_handler.go +++ b/app/vmagent/prometheusimport/request_handler.go @@ -10,6 +10,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/metrics" ) @@ -31,7 +32,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error { return err } isGzipped := req.Header.Get("Content-Encoding") == "gzip" - return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error { + return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error { return insertRows(at, rows, extraLabels) }, func(s string) { httpserver.LogError(req, s) diff --git a/app/vmagent/promremotewrite/request_handler.go b/app/vmagent/promremotewrite/request_handler.go index e3e08abe6b..acba222a99 100644 --- a/app/vmagent/promremotewrite/request_handler.go +++ b/app/vmagent/promremotewrite/request_handler.go @@ -10,7 +10,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" - parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/metrics" ) @@ -27,7 +27,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error { if err != nil { return err } - return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error { + return stream.Parse(req.Body, func(tss []prompb.TimeSeries) error { return insertRows(at, tss, extraLabels) }) } diff --git a/app/vmagent/vmimport/request_handler.go b/app/vmagent/vmimport/request_handler.go index f0eb010ba6..838f7a6ca9 100644 --- a/app/vmagent/vmimport/request_handler.go +++ b/app/vmagent/vmimport/request_handler.go @@ -11,6 +11,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/metrics" ) @@ -30,7 +31,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error { return err } isGzipped := req.Header.Get("Content-Encoding") == "gzip" - return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error { + return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error { return insertRows(at, rows, extraLabels) }) } diff --git a/app/vmalert/README.md b/app/vmalert/README.md index b2db420f95..3aee5da5bc 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -725,31 +725,42 @@ a review to the dashboard. ## Troubleshooting -vmalert executes configured rules within certain intervals. It is expected that at the moment when rule is executed, -the data is already present in configured `-datasource.url`: +### Data delay + +Data delay is one of the most common issues with rules execution. +vmalert executes configured rules within certain intervals at specifics timestamps. +It expects that the data is already present in configured `-datasource.url` at the moment of time when rule is executed: vmalert expected evaluation Usually, troubles start to appear when data in `-datasource.url` is delayed or absent. In such cases, evaluations -may get empty response from datasource and produce empty recording rules or reset alerts state: +may get empty response from the datasource and produce empty recording rules or reset alerts state: vmalert evaluation when data is delayed -By default, recently written samples to VictoriaMetrics aren't visible for queries for up to 30s. -This behavior is controlled by `-search.latencyOffset` command-line flag and the `latency_offset` query ag at `vmselect`. -Usually, this results into a 30s shift for recording rules results. -Note that too small value passed to `-search.latencyOffset` or to `latency_offest` query arg may lead to incomplete query results. +Try the following recommendations to reduce the chance of hitting the data delay issue: -Try the following recommendations in such cases: - -* Always configure group's `evaluationInterval` to be bigger or equal to `scrape_interval` at which metrics -are delivered to the datasource; +* Always configure group's `evaluationInterval` to be bigger or at least equal to +[time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution); +* Ensure that `[duration]` value is at least twice bigger than +[time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution). For example, +if expression is `rate(my_metric[2m]) > 0` then ensure that `my_metric` resolution is at least `1m` or better `30s`. +If you use VictoriaMetrics as datasource, `[duration]` can be omitted and VictoriaMetrics will adjust it automatically. * If you know in advance, that data in datasource is delayed - try changing vmalert's `-datasource.lookback` -command-line flag to add a time shift for evaluations; -* If time intervals between datapoints in datasource are irregular or `>=5min` - try changing vmalert's -`-datasource.queryStep` command-line flag to specify how far search query can lookback for the recent datapoint. -The recommendation is to have the step at least two times bigger than `scrape_interval`, since -there are no guarantees that scrape will not fail. +command-line flag to add a time shift for evaluations. Or extend `[duration]` to tolerate the delay. +For example, `max_over_time(errors_total[10m]) > 0` will be active even if there is no data in datasource for last `9m`. +* If [time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution) +in datasource is inconsistent or `>=5min` - try changing vmalert's `-datasource.queryStep` command-line flag to specify +how far search query can lookback for the recent datapoint. The recommendation is to have the step +at least two times bigger than the resolution. + +> Please note, data delay is inevitable in distributed systems. And it is better to account for it instead of ignoring. + +By default, recently written samples to VictoriaMetrics aren't visible for queries for up to 30s +(see `-search.latencyOffset` command-line flag at vmselect). Such delay is needed to eliminate risk of incomplete +data on the moment of querying, since metrics collectors won't be able to deliver the data in time. + +### Alerts state Sometimes, it is not clear why some specific alert fired or didn't fire. It is very important to remember, that alerts with `for: 0` fire immediately when their expression becomes true. And alerts with `for > 0` will fire only @@ -772,6 +783,8 @@ HTTP request sent by vmalert to the `-datasource.url` during evaluation. If spec no samples returned and curl command returns data - then it is very likely there was no data in datasource on the moment when rule was evaluated. +### Debug mode + vmalert allows configuring more detailed logging for specific alerting rule. Just set `debug: true` in rule's configuration and vmalert will start printing additional log messages: ```terminal diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go index 35a68c54ce..f26cfcfa32 100644 --- a/app/vmalert/notifier/alert.go +++ b/app/vmalert/notifier/alert.go @@ -41,7 +41,7 @@ type Alert struct { LastSent time.Time // Value stores the value returned from evaluating expression from Expr field Value float64 - // ID is the unique identifer for the Alert + // ID is the unique identifier for the Alert ID uint64 // Restored is true if Alert was restored after restart Restored bool diff --git a/app/vmalert/notifier/config.go b/app/vmalert/notifier/config.go index ac60a49652..d6419b900e 100644 --- a/app/vmalert/notifier/config.go +++ b/app/vmalert/notifier/config.go @@ -29,7 +29,7 @@ type Config struct { // ConsulSDConfigs contains list of settings for service discovery via Consul // see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"` - // DNSSDConfigs ontains list of settings for service discovery via DNS. + // DNSSDConfigs contains list of settings for service discovery via DNS. // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"` diff --git a/app/vmalert/templates/template.go b/app/vmalert/templates/template.go index c932b5777b..045a154ffb 100644 --- a/app/vmalert/templates/template.go +++ b/app/vmalert/templates/template.go @@ -225,7 +225,7 @@ func templateFuncs() textTpl.FuncMap { "toLower": strings.ToLower, // crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`. - // This funcion is deprectated. + // This function is deprecated. // // It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead - // these functions properly escape `\n` and `\r` chars according to their purpose. diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index 5031c67a06..0c02a9d1c8 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -110,7 +110,7 @@ type SrcPath struct { re *regexp.Regexp } -// URLPrefix represents pased `url_prefix` +// URLPrefix represents passed `url_prefix` type URLPrefix struct { n uint32 bus []*backendURL diff --git a/app/vmauth/target_url.go b/app/vmauth/target_url.go index a2849b88ef..b456c554a0 100644 --- a/app/vmauth/target_url.go +++ b/app/vmauth/target_url.go @@ -50,7 +50,7 @@ func normalizeURL(uOrig *url.URL) *url.URL { // Prevent from attacks with using `..` in r.URL.Path u.Path = path.Clean(u.Path) if !strings.HasSuffix(u.Path, "/") && strings.HasSuffix(uOrig.Path, "/") { - // The path.Clean() removes traling slash. + // The path.Clean() removes trailing slash. // Return it back if needed. // This should fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1752 u.Path += "/" diff --git a/app/vmctl/main.go b/app/vmctl/main.go index acb2c2bd03..df9a211652 100644 --- a/app/vmctl/main.go +++ b/app/vmctl/main.go @@ -20,7 +20,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm" "github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo" "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" - parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream" ) func main() { @@ -247,7 +247,7 @@ func main() { return cli.Exit(fmt.Errorf("cannot open exported block at path=%q err=%w", blockPath, err), 1) } var blocksCount uint64 - if err := parser.ParseStream(f, isBlockGzipped, func(block *parser.Block) error { + if err := stream.Parse(f, isBlockGzipped, func(block *stream.Block) error { atomic.AddUint64(&blocksCount, 1) return nil }); err != nil { diff --git a/app/vminsert/common/insert_ctx.go b/app/vminsert/common/insert_ctx.go index 153e003bd2..67ed92d908 100644 --- a/app/vminsert/common/insert_ctx.go +++ b/app/vminsert/common/insert_ctx.go @@ -146,7 +146,7 @@ func (ctx *InsertCtx) FlushBufs() error { } // There is no need in limiting the number of concurrent calls to vmstorage.AddRows() here, // since the number of concurrent FlushBufs() calls should be already limited via writeconcurrencylimiter - // used at every ParseStream() call under lib/protoparser/*/streamparser.go + // used at every stream.Parse() call under lib/protoparser/* err := vmstorage.AddRows(ctx.mrs) ctx.Reset(0) if err == nil { diff --git a/app/vminsert/csvimport/request_handler.go b/app/vminsert/csvimport/request_handler.go index 971f3abbd9..d7dc794c24 100644 --- a/app/vminsert/csvimport/request_handler.go +++ b/app/vminsert/csvimport/request_handler.go @@ -8,6 +8,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream" "github.com/VictoriaMetrics/metrics" ) @@ -22,7 +23,7 @@ func InsertHandler(req *http.Request) error { if err != nil { return err } - return parser.ParseStream(req, func(rows []parser.Row) error { + return stream.Parse(req, func(rows []parser.Row) error { return insertRows(rows, extraLabels) }) } diff --git a/app/vminsert/datadog/request_handler.go b/app/vminsert/datadog/request_handler.go index 48dd6f83fd..9717b7ef5e 100644 --- a/app/vminsert/datadog/request_handler.go +++ b/app/vminsert/datadog/request_handler.go @@ -8,6 +8,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream" "github.com/VictoriaMetrics/metrics" ) @@ -25,7 +26,7 @@ func InsertHandlerForHTTP(req *http.Request) error { return err } ce := req.Header.Get("Content-Encoding") - return parser.ParseStream(req.Body, ce, func(series []parser.Series) error { + return stream.Parse(req.Body, ce, func(series []parser.Series) error { return insertRows(series, extraLabels) }) } diff --git a/app/vminsert/graphite/request_handler.go b/app/vminsert/graphite/request_handler.go index 5d2d0a504b..2bb28789a3 100644 --- a/app/vminsert/graphite/request_handler.go +++ b/app/vminsert/graphite/request_handler.go @@ -6,6 +6,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common" "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream" "github.com/VictoriaMetrics/metrics" ) @@ -18,7 +19,7 @@ var ( // // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol func InsertHandler(r io.Reader) error { - return parser.ParseStream(r, insertRows) + return stream.Parse(r, insertRows) } func insertRows(rows []parser.Row) error { diff --git a/app/vminsert/influx/request_handler.go b/app/vminsert/influx/request_handler.go index daa7334d77..34b5a5b86d 100644 --- a/app/vminsert/influx/request_handler.go +++ b/app/vminsert/influx/request_handler.go @@ -14,6 +14,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" ) @@ -34,7 +35,7 @@ var ( // // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/ func InsertHandlerForReader(r io.Reader) error { - return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error { + return stream.Parse(r, false, "", "", func(db string, rows []parser.Row) error { return insertRows(db, rows, nil) }) } @@ -52,7 +53,7 @@ func InsertHandlerForHTTP(req *http.Request) error { precision := q.Get("precision") // Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint db := q.Get("db") - return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error { + return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error { return insertRows(db, rows, extraLabels) }) } diff --git a/app/vminsert/native/request_handler.go b/app/vminsert/native/request_handler.go index 666848e4a8..f5e7cd714b 100644 --- a/app/vminsert/native/request_handler.go +++ b/app/vminsert/native/request_handler.go @@ -10,7 +10,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" - parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" ) @@ -27,12 +27,12 @@ func InsertHandler(req *http.Request) error { return err } isGzip := req.Header.Get("Content-Encoding") == "gzip" - return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error { + return stream.Parse(req.Body, isGzip, func(block *stream.Block) error { return insertRows(block, extraLabels) }) } -func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error { +func insertRows(block *stream.Block, extraLabels []prompbmarshal.Label) error { ctx := getPushCtx() defer putPushCtx(ctx) diff --git a/app/vminsert/opentsdb/request_handler.go b/app/vminsert/opentsdb/request_handler.go index 44bf3eb986..9f3f0fbdff 100644 --- a/app/vminsert/opentsdb/request_handler.go +++ b/app/vminsert/opentsdb/request_handler.go @@ -6,6 +6,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common" "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream" "github.com/VictoriaMetrics/metrics" ) @@ -18,7 +19,7 @@ var ( // // See http://opentsdb.net/docs/build/html/api_telnet/put.html func InsertHandler(r io.Reader) error { - return parser.ParseStream(r, insertRows) + return stream.Parse(r, insertRows) } func insertRows(rows []parser.Row) error { diff --git a/app/vminsert/opentsdbhttp/request_handler.go b/app/vminsert/opentsdbhttp/request_handler.go index 3dafe29f92..2213e7dee9 100644 --- a/app/vminsert/opentsdbhttp/request_handler.go +++ b/app/vminsert/opentsdbhttp/request_handler.go @@ -9,6 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream" "github.com/VictoriaMetrics/metrics" ) @@ -27,7 +28,7 @@ func InsertHandler(req *http.Request) error { if err != nil { return err } - return parser.ParseStream(req, func(rows []parser.Row) error { + return stream.Parse(req, func(rows []parser.Row) error { return insertRows(rows, extraLabels) }) default: diff --git a/app/vminsert/prometheusimport/request_handler.go b/app/vminsert/prometheusimport/request_handler.go index 1ef5abb084..09eefc7c40 100644 --- a/app/vminsert/prometheusimport/request_handler.go +++ b/app/vminsert/prometheusimport/request_handler.go @@ -9,6 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream" "github.com/VictoriaMetrics/metrics" ) @@ -28,7 +29,7 @@ func InsertHandler(req *http.Request) error { return err } isGzipped := req.Header.Get("Content-Encoding") == "gzip" - return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error { + return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error { return insertRows(rows, extraLabels) }, func(s string) { httpserver.LogError(req, s) diff --git a/app/vminsert/prompush/push.go b/app/vminsert/prompush/push.go index a51e77dc09..fdbbb347ba 100644 --- a/app/vminsert/prompush/push.go +++ b/app/vminsert/prompush/push.go @@ -21,7 +21,7 @@ func Push(wr *prompbmarshal.WriteRequest) { tss := wr.Timeseries for len(tss) > 0 { - // Process big tss in smaller blocks in order to reduce maxmimum memory usage + // Process big tss in smaller blocks in order to reduce maximum memory usage samplesCount := 0 i := 0 for i < len(tss) { diff --git a/app/vminsert/promremotewrite/request_handler.go b/app/vminsert/promremotewrite/request_handler.go index 8d94e3235a..9cda3c4d3e 100644 --- a/app/vminsert/promremotewrite/request_handler.go +++ b/app/vminsert/promremotewrite/request_handler.go @@ -8,7 +8,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" - parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream" "github.com/VictoriaMetrics/metrics" ) @@ -23,7 +23,7 @@ func InsertHandler(req *http.Request) error { if err != nil { return err } - return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error { + return stream.Parse(req.Body, func(tss []prompb.TimeSeries) error { return insertRows(tss, extraLabels) }) } diff --git a/app/vminsert/vmimport/request_handler.go b/app/vminsert/vmimport/request_handler.go index 2a3a450393..f5357cff8d 100644 --- a/app/vminsert/vmimport/request_handler.go +++ b/app/vminsert/vmimport/request_handler.go @@ -11,6 +11,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" ) @@ -29,7 +30,7 @@ func InsertHandler(req *http.Request) error { return err } isGzipped := req.Header.Get("Content-Encoding") == "gzip" - return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error { + return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error { return insertRows(rows, extraLabels) }) } diff --git a/app/vmselect/graphiteql/parser.go b/app/vmselect/graphiteql/parser.go index f24a2e8bf0..b827c0f0e1 100644 --- a/app/vmselect/graphiteql/parser.go +++ b/app/vmselect/graphiteql/parser.go @@ -164,7 +164,7 @@ func (p *parser) parseString() (*StringExpr, error) { return se, nil } -// StringExpr represents string contant. +// StringExpr represents string constant. type StringExpr struct { // S contains unquoted string contents. S string diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index a468526bef..2622e5cce2 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -194,7 +194,7 @@ func getTmpResult() *result { func putTmpResult(r *result) { currentTime := fasttime.UnixTimestamp() if cap(r.rs.Values) > 1024*1024 && 4*len(r.rs.Values) < cap(r.rs.Values) && currentTime-r.lastResetTime > 10 { - // Reset r.rs in order to preseve memory usage after processing big time series with millions of rows. + // Reset r.rs in order to preserve memory usage after processing big time series with millions of rows. r.rs = Result{} r.lastResetTime = currentTime } @@ -1015,7 +1015,7 @@ func TagValueSuffixes(qt *querytracer.Tracer, tr storage.TimeRange, tagKey, tagV // TSDBStatus returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats // -// It accepts aribtrary filters on time series in sq. +// It accepts arbitrary filters on time series in sq. func TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) { qt = qt.NewChild("get tsdb stats: %s, focusLabel=%q, topN=%d", sq, focusLabel, topN) defer qt.Done() diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go index 84c459915b..d432e58068 100644 --- a/app/vmselect/promql/aggr.go +++ b/app/vmselect/promql/aggr.go @@ -901,7 +901,7 @@ func quantileSorted(phi float64, values []float64) float64 { func aggrFuncMAD(tss []*timeseries) []*timeseries { // Calculate medians for each point across tss. medians := getPerPointMedians(tss) - // Calculate MAD values multipled by tolerance for each point across tss. + // Calculate MAD values multiplied by tolerance for each point across tss. // See https://en.wikipedia.org/wiki/Median_absolute_deviation mads := getPerPointMADs(tss, medians) tss[0].Values = append(tss[0].Values[:0], mads...) @@ -920,7 +920,7 @@ func aggrFuncOutliersMAD(afa *aggrFuncArg) ([]*timeseries, error) { afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries { // Calculate medians for each point across tss. medians := getPerPointMedians(tss) - // Calculate MAD values multipled by tolerance for each point across tss. + // Calculate MAD values multiplied by tolerance for each point across tss. // See https://en.wikipedia.org/wiki/Median_absolute_deviation mads := getPerPointMADs(tss, medians) for n := range mads { diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go index ac364adbe5..37bdaed025 100644 --- a/app/vmselect/promql/eval.go +++ b/app/vmselect/promql/eval.go @@ -466,7 +466,7 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec // 1) execute the exprFirst // 2) get common label filters for series returned at step 1 // 3) push down the found common label filters to exprSecond. This filters out unneeded series - // during exprSecond exection instead of spending compute resources on extracting and processing these series + // during exprSecond execution instead of spending compute resources on extracting and processing these series // before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching // 4) execute the exprSecond with possible additional filters found at step 3 // diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 993c247614..2b6dfe6b49 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -385,7 +385,7 @@ func getRollupFunc(funcName string) newRollupFunc { } type rollupFuncArg struct { - // The value preceeding values if it fits staleness interval. + // The value preceding values if it fits staleness interval. prevValue float64 // The timestamp for prevValue. @@ -397,7 +397,7 @@ type rollupFuncArg struct { // Timestamps for values. timestamps []int64 - // Real value preceeding values without restrictions on staleness interval. + // Real value preceding values without restrictions on staleness interval. realPrevValue float64 // Real value which goes after values. @@ -587,7 +587,7 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu if window <= 0 { window = rc.Step if rc.MayAdjustWindow && window < maxPrevInterval { - // Adjust lookbehind window only if it isn't set explicilty, e.g. rate(foo). + // Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo). // In the case of missing lookbehind window it should be adjusted in order to return non-empty graph // when the window doesn't cover at least two raw samples (this is what most users expect). // diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c9eed3b854..4ed3678ffe 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -25,6 +25,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): show `median` instead of `avg` in graph tooltip and line legend, since `median` is more tolerant against spikes. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3706). * BUGFIX: prevent from possible data ingestion slowdown and query performance slowdown during [background merges of big parts](https://docs.victoriametrics.com/#storage) on systems with small number of CPU cores (1 or 2 CPU cores). The issue has been introduced in [v1.85.0](https://docs.victoriametrics.com/CHANGELOG.html#v1850) when implementing [this feature](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337). See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3790). +* BUGFIX: properly parse timestamps in milliseconds when [ingesting data via OpenTSDB telnet put protocol](https://docs.victoriametrics.com/#sending-data-via-telnet-put-protocol). Previously timestamps in milliseconds were mistakenly multiplied by 1000. Thanks to @Droxenator for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3810). ## [v1.87.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.1) diff --git a/docs/keyConcepts.md b/docs/keyConcepts.md index 3673ef6798..d4d13aabc0 100644 --- a/docs/keyConcepts.md +++ b/docs/keyConcepts.md @@ -79,6 +79,30 @@ requests_total{path="/", code="200"} 123 4567890 - The `4567890` is an optional timestamp for the sample. If it is missing, then the current timestamp is used when storing the sample in VictoriaMetrics. +#### Time series resolution + +Resolution is the minimum interval between [raw samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples) +of the [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series). Consider the following example: +``` +---------------------------------------------------------------------- +|