diff --git a/app/vmagent/csvimport/request_handler.go b/app/vmagent/csvimport/request_handler.go
index e17207c0d8..00d9846d75 100644
--- a/app/vmagent/csvimport/request_handler.go
+++ b/app/vmagent/csvimport/request_handler.go
@@ -9,6 +9,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
"github.com/VictoriaMetrics/metrics"
)
@@ -25,7 +26,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
if err != nil {
return err
}
- return parser.ParseStream(req, func(rows []parser.Row) error {
+ return stream.Parse(req, func(rows []parser.Row) error {
return insertRows(at, rows, extraLabels)
})
}
diff --git a/app/vmagent/datadog/request_handler.go b/app/vmagent/datadog/request_handler.go
index adac86be3c..056d3bcb0b 100644
--- a/app/vmagent/datadog/request_handler.go
+++ b/app/vmagent/datadog/request_handler.go
@@ -9,6 +9,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
"github.com/VictoriaMetrics/metrics"
)
@@ -28,7 +29,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
return err
}
ce := req.Header.Get("Content-Encoding")
- return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+ return stream.Parse(req.Body, ce, func(series []parser.Series) error {
return insertRows(at, series, extraLabels)
})
}
diff --git a/app/vmagent/graphite/request_handler.go b/app/vmagent/graphite/request_handler.go
index d9ff2a1b20..2c81806677 100644
--- a/app/vmagent/graphite/request_handler.go
+++ b/app/vmagent/graphite/request_handler.go
@@ -7,6 +7,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
"github.com/VictoriaMetrics/metrics"
)
@@ -19,7 +20,7 @@ var (
//
// See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
func InsertHandler(r io.Reader) error {
- return parser.ParseStream(r, insertRows)
+ return stream.Parse(r, insertRows)
}
func insertRows(rows []parser.Row) error {
diff --git a/app/vmagent/influx/request_handler.go b/app/vmagent/influx/request_handler.go
index d7211e385b..7ff651d5b9 100644
--- a/app/vmagent/influx/request_handler.go
+++ b/app/vmagent/influx/request_handler.go
@@ -15,6 +15,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
"github.com/VictoriaMetrics/metrics"
)
@@ -36,7 +37,7 @@ var (
//
// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
- return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+ return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
return insertRows(nil, db, rows, nil)
})
}
@@ -54,7 +55,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
precision := q.Get("precision")
// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
db := q.Get("db")
- return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+ return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
return insertRows(at, db, rows, extraLabels)
})
}
diff --git a/app/vmagent/native/request_handler.go b/app/vmagent/native/request_handler.go
index 898394ae83..60347b0dde 100644
--- a/app/vmagent/native/request_handler.go
+++ b/app/vmagent/native/request_handler.go
@@ -10,7 +10,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
- parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
"github.com/VictoriaMetrics/metrics"
)
@@ -30,12 +30,12 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
return err
}
isGzip := req.Header.Get("Content-Encoding") == "gzip"
- return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
+ return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
return insertRows(at, block, extraLabels)
})
}
-func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error {
ctx := common.GetPushCtx()
defer common.PutPushCtx(ctx)
diff --git a/app/vmagent/opentsdb/request_handler.go b/app/vmagent/opentsdb/request_handler.go
index e1e42ba4bc..8388a5238a 100644
--- a/app/vmagent/opentsdb/request_handler.go
+++ b/app/vmagent/opentsdb/request_handler.go
@@ -7,6 +7,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
"github.com/VictoriaMetrics/metrics"
)
@@ -19,7 +20,7 @@ var (
//
// See http://opentsdb.net/docs/build/html/api_telnet/put.html
func InsertHandler(r io.Reader) error {
- return parser.ParseStream(r, insertRows)
+ return stream.Parse(r, insertRows)
}
func insertRows(rows []parser.Row) error {
diff --git a/app/vmagent/opentsdbhttp/request_handler.go b/app/vmagent/opentsdbhttp/request_handler.go
index 968ea936ba..7fecea6b34 100644
--- a/app/vmagent/opentsdbhttp/request_handler.go
+++ b/app/vmagent/opentsdbhttp/request_handler.go
@@ -9,6 +9,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
"github.com/VictoriaMetrics/metrics"
)
@@ -24,7 +25,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
if err != nil {
return err
}
- return parser.ParseStream(req, func(rows []parser.Row) error {
+ return stream.Parse(req, func(rows []parser.Row) error {
return insertRows(at, rows, extraLabels)
})
}
diff --git a/app/vmagent/prometheusimport/request_handler.go b/app/vmagent/prometheusimport/request_handler.go
index ed5d53a850..2f450e1c29 100644
--- a/app/vmagent/prometheusimport/request_handler.go
+++ b/app/vmagent/prometheusimport/request_handler.go
@@ -10,6 +10,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
"github.com/VictoriaMetrics/metrics"
)
@@ -31,7 +32,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
return err
}
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
- return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+ return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
return insertRows(at, rows, extraLabels)
}, func(s string) {
httpserver.LogError(req, s)
diff --git a/app/vmagent/promremotewrite/request_handler.go b/app/vmagent/promremotewrite/request_handler.go
index e3e08abe6b..acba222a99 100644
--- a/app/vmagent/promremotewrite/request_handler.go
+++ b/app/vmagent/promremotewrite/request_handler.go
@@ -10,7 +10,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
- parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
"github.com/VictoriaMetrics/metrics"
)
@@ -27,7 +27,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
if err != nil {
return err
}
- return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
+ return stream.Parse(req.Body, func(tss []prompb.TimeSeries) error {
return insertRows(at, tss, extraLabels)
})
}
diff --git a/app/vmagent/vmimport/request_handler.go b/app/vmagent/vmimport/request_handler.go
index f0eb010ba6..838f7a6ca9 100644
--- a/app/vmagent/vmimport/request_handler.go
+++ b/app/vmagent/vmimport/request_handler.go
@@ -11,6 +11,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
"github.com/VictoriaMetrics/metrics"
)
@@ -30,7 +31,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
return err
}
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
- return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
+ return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
return insertRows(at, rows, extraLabels)
})
}
diff --git a/app/vmalert/README.md b/app/vmalert/README.md
index b2db420f95..3aee5da5bc 100644
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -725,31 +725,42 @@ a review to the dashboard.
## Troubleshooting
-vmalert executes configured rules within certain intervals. It is expected that at the moment when rule is executed,
-the data is already present in configured `-datasource.url`:
+### Data delay
+
+Data delay is one of the most common issues with rules execution.
+vmalert executes configured rules within certain intervals at specifics timestamps.
+It expects that the data is already present in configured `-datasource.url` at the moment of time when rule is executed:
Usually, troubles start to appear when data in `-datasource.url` is delayed or absent. In such cases, evaluations
-may get empty response from datasource and produce empty recording rules or reset alerts state:
+may get empty response from the datasource and produce empty recording rules or reset alerts state:
-By default, recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
-This behavior is controlled by `-search.latencyOffset` command-line flag and the `latency_offset` query ag at `vmselect`.
-Usually, this results into a 30s shift for recording rules results.
-Note that too small value passed to `-search.latencyOffset` or to `latency_offest` query arg may lead to incomplete query results.
+Try the following recommendations to reduce the chance of hitting the data delay issue:
-Try the following recommendations in such cases:
-
-* Always configure group's `evaluationInterval` to be bigger or equal to `scrape_interval` at which metrics
-are delivered to the datasource;
+* Always configure group's `evaluationInterval` to be bigger or at least equal to
+[time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution);
+* Ensure that `[duration]` value is at least twice bigger than
+[time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution). For example,
+if expression is `rate(my_metric[2m]) > 0` then ensure that `my_metric` resolution is at least `1m` or better `30s`.
+If you use VictoriaMetrics as datasource, `[duration]` can be omitted and VictoriaMetrics will adjust it automatically.
* If you know in advance, that data in datasource is delayed - try changing vmalert's `-datasource.lookback`
-command-line flag to add a time shift for evaluations;
-* If time intervals between datapoints in datasource are irregular or `>=5min` - try changing vmalert's
-`-datasource.queryStep` command-line flag to specify how far search query can lookback for the recent datapoint.
-The recommendation is to have the step at least two times bigger than `scrape_interval`, since
-there are no guarantees that scrape will not fail.
+command-line flag to add a time shift for evaluations. Or extend `[duration]` to tolerate the delay.
+For example, `max_over_time(errors_total[10m]) > 0` will be active even if there is no data in datasource for last `9m`.
+* If [time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution)
+in datasource is inconsistent or `>=5min` - try changing vmalert's `-datasource.queryStep` command-line flag to specify
+how far search query can lookback for the recent datapoint. The recommendation is to have the step
+at least two times bigger than the resolution.
+
+> Please note, data delay is inevitable in distributed systems. And it is better to account for it instead of ignoring.
+
+By default, recently written samples to VictoriaMetrics aren't visible for queries for up to 30s
+(see `-search.latencyOffset` command-line flag at vmselect). Such delay is needed to eliminate risk of incomplete
+data on the moment of querying, since metrics collectors won't be able to deliver the data in time.
+
+### Alerts state
Sometimes, it is not clear why some specific alert fired or didn't fire. It is very important to remember, that
alerts with `for: 0` fire immediately when their expression becomes true. And alerts with `for > 0` will fire only
@@ -772,6 +783,8 @@ HTTP request sent by vmalert to the `-datasource.url` during evaluation. If spec
no samples returned and curl command returns data - then it is very likely there was no data in datasource on the
moment when rule was evaluated.
+### Debug mode
+
vmalert allows configuring more detailed logging for specific alerting rule. Just set `debug: true` in rule's configuration
and vmalert will start printing additional log messages:
```terminal
diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go
index 35a68c54ce..f26cfcfa32 100644
--- a/app/vmalert/notifier/alert.go
+++ b/app/vmalert/notifier/alert.go
@@ -41,7 +41,7 @@ type Alert struct {
LastSent time.Time
// Value stores the value returned from evaluating expression from Expr field
Value float64
- // ID is the unique identifer for the Alert
+ // ID is the unique identifier for the Alert
ID uint64
// Restored is true if Alert was restored after restart
Restored bool
diff --git a/app/vmalert/notifier/config.go b/app/vmalert/notifier/config.go
index ac60a49652..d6419b900e 100644
--- a/app/vmalert/notifier/config.go
+++ b/app/vmalert/notifier/config.go
@@ -29,7 +29,7 @@ type Config struct {
// ConsulSDConfigs contains list of settings for service discovery via Consul
// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
- // DNSSDConfigs ontains list of settings for service discovery via DNS.
+ // DNSSDConfigs contains list of settings for service discovery via DNS.
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
diff --git a/app/vmalert/templates/template.go b/app/vmalert/templates/template.go
index c932b5777b..045a154ffb 100644
--- a/app/vmalert/templates/template.go
+++ b/app/vmalert/templates/template.go
@@ -225,7 +225,7 @@ func templateFuncs() textTpl.FuncMap {
"toLower": strings.ToLower,
// crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`.
- // This funcion is deprectated.
+ // This function is deprecated.
//
// It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead -
// these functions properly escape `\n` and `\r` chars according to their purpose.
diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go
index 5031c67a06..0c02a9d1c8 100644
--- a/app/vmauth/auth_config.go
+++ b/app/vmauth/auth_config.go
@@ -110,7 +110,7 @@ type SrcPath struct {
re *regexp.Regexp
}
-// URLPrefix represents pased `url_prefix`
+// URLPrefix represents passed `url_prefix`
type URLPrefix struct {
n uint32
bus []*backendURL
diff --git a/app/vmauth/target_url.go b/app/vmauth/target_url.go
index a2849b88ef..b456c554a0 100644
--- a/app/vmauth/target_url.go
+++ b/app/vmauth/target_url.go
@@ -50,7 +50,7 @@ func normalizeURL(uOrig *url.URL) *url.URL {
// Prevent from attacks with using `..` in r.URL.Path
u.Path = path.Clean(u.Path)
if !strings.HasSuffix(u.Path, "/") && strings.HasSuffix(uOrig.Path, "/") {
- // The path.Clean() removes traling slash.
+ // The path.Clean() removes trailing slash.
// Return it back if needed.
// This should fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1752
u.Path += "/"
diff --git a/app/vmctl/main.go b/app/vmctl/main.go
index acb2c2bd03..df9a211652 100644
--- a/app/vmctl/main.go
+++ b/app/vmctl/main.go
@@ -20,7 +20,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
- parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
)
func main() {
@@ -247,7 +247,7 @@ func main() {
return cli.Exit(fmt.Errorf("cannot open exported block at path=%q err=%w", blockPath, err), 1)
}
var blocksCount uint64
- if err := parser.ParseStream(f, isBlockGzipped, func(block *parser.Block) error {
+ if err := stream.Parse(f, isBlockGzipped, func(block *stream.Block) error {
atomic.AddUint64(&blocksCount, 1)
return nil
}); err != nil {
diff --git a/app/vminsert/common/insert_ctx.go b/app/vminsert/common/insert_ctx.go
index 153e003bd2..67ed92d908 100644
--- a/app/vminsert/common/insert_ctx.go
+++ b/app/vminsert/common/insert_ctx.go
@@ -146,7 +146,7 @@ func (ctx *InsertCtx) FlushBufs() error {
}
// There is no need in limiting the number of concurrent calls to vmstorage.AddRows() here,
// since the number of concurrent FlushBufs() calls should be already limited via writeconcurrencylimiter
- // used at every ParseStream() call under lib/protoparser/*/streamparser.go
+ // used at every stream.Parse() call under lib/protoparser/*
err := vmstorage.AddRows(ctx.mrs)
ctx.Reset(0)
if err == nil {
diff --git a/app/vminsert/csvimport/request_handler.go b/app/vminsert/csvimport/request_handler.go
index 971f3abbd9..d7dc794c24 100644
--- a/app/vminsert/csvimport/request_handler.go
+++ b/app/vminsert/csvimport/request_handler.go
@@ -8,6 +8,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
"github.com/VictoriaMetrics/metrics"
)
@@ -22,7 +23,7 @@ func InsertHandler(req *http.Request) error {
if err != nil {
return err
}
- return parser.ParseStream(req, func(rows []parser.Row) error {
+ return stream.Parse(req, func(rows []parser.Row) error {
return insertRows(rows, extraLabels)
})
}
diff --git a/app/vminsert/datadog/request_handler.go b/app/vminsert/datadog/request_handler.go
index 48dd6f83fd..9717b7ef5e 100644
--- a/app/vminsert/datadog/request_handler.go
+++ b/app/vminsert/datadog/request_handler.go
@@ -8,6 +8,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog/stream"
"github.com/VictoriaMetrics/metrics"
)
@@ -25,7 +26,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
return err
}
ce := req.Header.Get("Content-Encoding")
- return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+ return stream.Parse(req.Body, ce, func(series []parser.Series) error {
return insertRows(series, extraLabels)
})
}
diff --git a/app/vminsert/graphite/request_handler.go b/app/vminsert/graphite/request_handler.go
index 5d2d0a504b..2bb28789a3 100644
--- a/app/vminsert/graphite/request_handler.go
+++ b/app/vminsert/graphite/request_handler.go
@@ -6,6 +6,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
"github.com/VictoriaMetrics/metrics"
)
@@ -18,7 +19,7 @@ var (
//
// See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
func InsertHandler(r io.Reader) error {
- return parser.ParseStream(r, insertRows)
+ return stream.Parse(r, insertRows)
}
func insertRows(rows []parser.Row) error {
diff --git a/app/vminsert/influx/request_handler.go b/app/vminsert/influx/request_handler.go
index daa7334d77..34b5a5b86d 100644
--- a/app/vminsert/influx/request_handler.go
+++ b/app/vminsert/influx/request_handler.go
@@ -14,6 +14,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/metrics"
)
@@ -34,7 +35,7 @@ var (
//
// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
func InsertHandlerForReader(r io.Reader) error {
- return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
+ return stream.Parse(r, false, "", "", func(db string, rows []parser.Row) error {
return insertRows(db, rows, nil)
})
}
@@ -52,7 +53,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
precision := q.Get("precision")
// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
db := q.Get("db")
- return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+ return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
return insertRows(db, rows, extraLabels)
})
}
diff --git a/app/vminsert/native/request_handler.go b/app/vminsert/native/request_handler.go
index 666848e4a8..f5e7cd714b 100644
--- a/app/vminsert/native/request_handler.go
+++ b/app/vminsert/native/request_handler.go
@@ -10,7 +10,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
- parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/metrics"
)
@@ -27,12 +27,12 @@ func InsertHandler(req *http.Request) error {
return err
}
isGzip := req.Header.Get("Content-Encoding") == "gzip"
- return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
+ return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
return insertRows(block, extraLabels)
})
}
-func insertRows(block *parser.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(block *stream.Block, extraLabels []prompbmarshal.Label) error {
ctx := getPushCtx()
defer putPushCtx(ctx)
diff --git a/app/vminsert/opentsdb/request_handler.go b/app/vminsert/opentsdb/request_handler.go
index 44bf3eb986..9f3f0fbdff 100644
--- a/app/vminsert/opentsdb/request_handler.go
+++ b/app/vminsert/opentsdb/request_handler.go
@@ -6,6 +6,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
"github.com/VictoriaMetrics/metrics"
)
@@ -18,7 +19,7 @@ var (
//
// See http://opentsdb.net/docs/build/html/api_telnet/put.html
func InsertHandler(r io.Reader) error {
- return parser.ParseStream(r, insertRows)
+ return stream.Parse(r, insertRows)
}
func insertRows(rows []parser.Row) error {
diff --git a/app/vminsert/opentsdbhttp/request_handler.go b/app/vminsert/opentsdbhttp/request_handler.go
index 3dafe29f92..2213e7dee9 100644
--- a/app/vminsert/opentsdbhttp/request_handler.go
+++ b/app/vminsert/opentsdbhttp/request_handler.go
@@ -9,6 +9,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
"github.com/VictoriaMetrics/metrics"
)
@@ -27,7 +28,7 @@ func InsertHandler(req *http.Request) error {
if err != nil {
return err
}
- return parser.ParseStream(req, func(rows []parser.Row) error {
+ return stream.Parse(req, func(rows []parser.Row) error {
return insertRows(rows, extraLabels)
})
default:
diff --git a/app/vminsert/prometheusimport/request_handler.go b/app/vminsert/prometheusimport/request_handler.go
index 1ef5abb084..09eefc7c40 100644
--- a/app/vminsert/prometheusimport/request_handler.go
+++ b/app/vminsert/prometheusimport/request_handler.go
@@ -9,6 +9,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
"github.com/VictoriaMetrics/metrics"
)
@@ -28,7 +29,7 @@ func InsertHandler(req *http.Request) error {
return err
}
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
- return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+ return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
return insertRows(rows, extraLabels)
}, func(s string) {
httpserver.LogError(req, s)
diff --git a/app/vminsert/prompush/push.go b/app/vminsert/prompush/push.go
index a51e77dc09..fdbbb347ba 100644
--- a/app/vminsert/prompush/push.go
+++ b/app/vminsert/prompush/push.go
@@ -21,7 +21,7 @@ func Push(wr *prompbmarshal.WriteRequest) {
tss := wr.Timeseries
for len(tss) > 0 {
- // Process big tss in smaller blocks in order to reduce maxmimum memory usage
+ // Process big tss in smaller blocks in order to reduce maximum memory usage
samplesCount := 0
i := 0
for i < len(tss) {
diff --git a/app/vminsert/promremotewrite/request_handler.go b/app/vminsert/promremotewrite/request_handler.go
index 8d94e3235a..9cda3c4d3e 100644
--- a/app/vminsert/promremotewrite/request_handler.go
+++ b/app/vminsert/promremotewrite/request_handler.go
@@ -8,7 +8,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
- parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
"github.com/VictoriaMetrics/metrics"
)
@@ -23,7 +23,7 @@ func InsertHandler(req *http.Request) error {
if err != nil {
return err
}
- return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
+ return stream.Parse(req.Body, func(tss []prompb.TimeSeries) error {
return insertRows(tss, extraLabels)
})
}
diff --git a/app/vminsert/vmimport/request_handler.go b/app/vminsert/vmimport/request_handler.go
index 2a3a450393..f5357cff8d 100644
--- a/app/vminsert/vmimport/request_handler.go
+++ b/app/vminsert/vmimport/request_handler.go
@@ -11,6 +11,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/metrics"
)
@@ -29,7 +30,7 @@ func InsertHandler(req *http.Request) error {
return err
}
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
- return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
+ return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
return insertRows(rows, extraLabels)
})
}
diff --git a/app/vmselect/graphiteql/parser.go b/app/vmselect/graphiteql/parser.go
index f24a2e8bf0..b827c0f0e1 100644
--- a/app/vmselect/graphiteql/parser.go
+++ b/app/vmselect/graphiteql/parser.go
@@ -164,7 +164,7 @@ func (p *parser) parseString() (*StringExpr, error) {
return se, nil
}
-// StringExpr represents string contant.
+// StringExpr represents string constant.
type StringExpr struct {
// S contains unquoted string contents.
S string
diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go
index a468526bef..2622e5cce2 100644
--- a/app/vmselect/netstorage/netstorage.go
+++ b/app/vmselect/netstorage/netstorage.go
@@ -194,7 +194,7 @@ func getTmpResult() *result {
func putTmpResult(r *result) {
currentTime := fasttime.UnixTimestamp()
if cap(r.rs.Values) > 1024*1024 && 4*len(r.rs.Values) < cap(r.rs.Values) && currentTime-r.lastResetTime > 10 {
- // Reset r.rs in order to preseve memory usage after processing big time series with millions of rows.
+ // Reset r.rs in order to preserve memory usage after processing big time series with millions of rows.
r.rs = Result{}
r.lastResetTime = currentTime
}
@@ -1015,7 +1015,7 @@ func TagValueSuffixes(qt *querytracer.Tracer, tr storage.TimeRange, tagKey, tagV
// TSDBStatus returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
//
-// It accepts aribtrary filters on time series in sq.
+// It accepts arbitrary filters on time series in sq.
func TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) {
qt = qt.NewChild("get tsdb stats: %s, focusLabel=%q, topN=%d", sq, focusLabel, topN)
defer qt.Done()
diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go
index 84c459915b..d432e58068 100644
--- a/app/vmselect/promql/aggr.go
+++ b/app/vmselect/promql/aggr.go
@@ -901,7 +901,7 @@ func quantileSorted(phi float64, values []float64) float64 {
func aggrFuncMAD(tss []*timeseries) []*timeseries {
// Calculate medians for each point across tss.
medians := getPerPointMedians(tss)
- // Calculate MAD values multipled by tolerance for each point across tss.
+ // Calculate MAD values multiplied by tolerance for each point across tss.
// See https://en.wikipedia.org/wiki/Median_absolute_deviation
mads := getPerPointMADs(tss, medians)
tss[0].Values = append(tss[0].Values[:0], mads...)
@@ -920,7 +920,7 @@ func aggrFuncOutliersMAD(afa *aggrFuncArg) ([]*timeseries, error) {
afe := func(tss []*timeseries, modifier *metricsql.ModifierExpr) []*timeseries {
// Calculate medians for each point across tss.
medians := getPerPointMedians(tss)
- // Calculate MAD values multipled by tolerance for each point across tss.
+ // Calculate MAD values multiplied by tolerance for each point across tss.
// See https://en.wikipedia.org/wiki/Median_absolute_deviation
mads := getPerPointMADs(tss, medians)
for n := range mads {
diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go
index ac364adbe5..37bdaed025 100644
--- a/app/vmselect/promql/eval.go
+++ b/app/vmselect/promql/eval.go
@@ -466,7 +466,7 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
// 1) execute the exprFirst
// 2) get common label filters for series returned at step 1
// 3) push down the found common label filters to exprSecond. This filters out unneeded series
- // during exprSecond exection instead of spending compute resources on extracting and processing these series
+ // during exprSecond execution instead of spending compute resources on extracting and processing these series
// before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
// 4) execute the exprSecond with possible additional filters found at step 3
//
diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go
index 993c247614..2b6dfe6b49 100644
--- a/app/vmselect/promql/rollup.go
+++ b/app/vmselect/promql/rollup.go
@@ -385,7 +385,7 @@ func getRollupFunc(funcName string) newRollupFunc {
}
type rollupFuncArg struct {
- // The value preceeding values if it fits staleness interval.
+ // The value preceding values if it fits staleness interval.
prevValue float64
// The timestamp for prevValue.
@@ -397,7 +397,7 @@ type rollupFuncArg struct {
// Timestamps for values.
timestamps []int64
- // Real value preceeding values without restrictions on staleness interval.
+ // Real value preceding values without restrictions on staleness interval.
realPrevValue float64
// Real value which goes after values.
@@ -587,7 +587,7 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
if window <= 0 {
window = rc.Step
if rc.MayAdjustWindow && window < maxPrevInterval {
- // Adjust lookbehind window only if it isn't set explicilty, e.g. rate(foo).
+ // Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
// when the window doesn't cover at least two raw samples (this is what most users expect).
//
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index c9eed3b854..4ed3678ffe 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -25,6 +25,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): show `median` instead of `avg` in graph tooltip and line legend, since `median` is more tolerant against spikes. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3706).
* BUGFIX: prevent from possible data ingestion slowdown and query performance slowdown during [background merges of big parts](https://docs.victoriametrics.com/#storage) on systems with small number of CPU cores (1 or 2 CPU cores). The issue has been introduced in [v1.85.0](https://docs.victoriametrics.com/CHANGELOG.html#v1850) when implementing [this feature](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337). See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3790).
+* BUGFIX: properly parse timestamps in milliseconds when [ingesting data via OpenTSDB telnet put protocol](https://docs.victoriametrics.com/#sending-data-via-telnet-put-protocol). Previously timestamps in milliseconds were mistakenly multiplied by 1000. Thanks to @Droxenator for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3810).
## [v1.87.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.1)
diff --git a/docs/keyConcepts.md b/docs/keyConcepts.md
index 3673ef6798..d4d13aabc0 100644
--- a/docs/keyConcepts.md
+++ b/docs/keyConcepts.md
@@ -79,6 +79,30 @@ requests_total{path="/", code="200"} 123 4567890
- The `4567890` is an optional timestamp for the sample. If it is missing,
then the current timestamp is used when storing the sample in VictoriaMetrics.
+#### Time series resolution
+
+Resolution is the minimum interval between [raw samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples)
+of the [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series). Consider the following example:
+```
+----------------------------------------------------------------------
+|