github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-01 14:47:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

Browse source
This commit is contained in:
Aliaksandr Valialkin 2023-08-11 06:45:26 -07:00
commit 978594f50f
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
103 changed files with 2819 additions and 589 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/check-licenses.yml vendored
View file

@ -17,7 +17,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@main
with:
go-version: 1.20.6
go-version: 1.21.0
id: go
- name: Code checkout
uses: actions/checkout@master

2
.github/workflows/codeql-analysis.yml vendored
View file

@ -57,7 +57,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.20.6
go-version: 1.21.0
check-latest: true
cache: true
if: ${{ matrix.language == 'go' }}

6
.github/workflows/main.yml vendored
View file

@ -32,7 +32,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v4
with:
go-version: 1.20.6
go-version: 1.21.0
check-latest: true
cache: true
@ -56,7 +56,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v4
with:
go-version: 1.20.6
go-version: 1.21.0
check-latest: true
cache: true
@ -81,7 +81,7 @@ jobs:
id: go
uses: actions/setup-go@v4
with:
go-version: 1.20.6
go-version: 1.21.0
check-latest: true
cache: true

5
README.md
View file

@ -117,6 +117,7 @@ Case studies:
* [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x)
* [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode)
* [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind)
* [Naver](https://docs.victoriametrics.com/CaseStudies.html#naver)
* [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay)
* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona)
* [Roblox](https://docs.victoriametrics.com/CaseStudies.html#roblox)
@ -1360,7 +1361,7 @@ VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-sc
## Sending data via OpenTelemetry
VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentemetry/api/v1/push` path.
VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentelemetry/api/v1/push` path.
VictoriaMetrics expects `protobuf`-encoded requests at `/opentelemetry/api/v1/push`.
Set HTTP request header `Content-Encoding: gzip` when sending gzip-compressed data to `/opentelemetry/api/v1/push`.
@ -1781,7 +1782,7 @@ created by community.
Graphs on the dashboards contain useful hints - hover the `i` icon in the top left corner of each graph to read it.
We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml)
We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
via [vmalert](https://docs.victoriametrics.com/vmalert.html) or via Prometheus.
VictoriaMetrics exposes currently running queries and their execution times at `/api/v1/status/active_queries` page.

31
app/vlstorage/main.go
View file

@ -121,24 +121,25 @@ func initStorageMetrics(strg *logstorage.Storage) *metrics.Set {
return float64(m().FileMergesTotal)
})
ms.NewGauge(`vl_rows{type="inmemory"}`, func() float64 {
ms.NewGauge(`vl_storage_rows{type="inmemory"}`, func() float64 {
return float64(m().InmemoryRowsCount)
})
ms.NewGauge(`vl_rows{type="file"}`, func() float64 {
ms.NewGauge(`vl_storage_rows{type="file"}`, func() float64 {
return float64(m().FileRowsCount)
})
ms.NewGauge(`vl_parts{type="inmemory"}`, func() float64 {
ms.NewGauge(`vl_storage_parts{type="inmemory"}`, func() float64 {
return float64(m().InmemoryParts)
})
ms.NewGauge(`vl_parts{type="file"}`, func() float64 {
ms.NewGauge(`vl_storage_parts{type="file"}`, func() float64 {
return float64(m().FileParts)
})
ms.NewGauge(`vl_blocks{type="inmemory"}`, func() float64 {
ms.NewGauge(`vl_storage_blocks{type="inmemory"}`, func() float64 {
return float64(m().InmemoryBlocks)
})
ms.NewGauge(`vl_blocks{type="file"}`, func() float64 {
ms.NewGauge(`vl_storage_blocks{type="file"}`, func() float64 {
return float64(m().FileBlocks)
})
ms.NewGauge(`vl_partitions`, func() float64 {
return float64(m().PartitionsCount)
})
@ -146,6 +147,24 @@ func initStorageMetrics(strg *logstorage.Storage) *metrics.Set {
return float64(m().StreamsCreatedTotal)
})
ms.NewGauge(`vl_indexdb_rows`, func() float64 {
return float64(m().IndexdbItemsCount)
})
ms.NewGauge(`vl_indexdb_parts`, func() float64 {
return float64(m().IndexdbPartsCount)
})
ms.NewGauge(`vl_indexdb_blocks`, func() float64 {
return float64(m().IndexdbBlocksCount)
})
ms.NewGauge(`vl_data_size_bytes{type="indexdb"}`, func() float64 {
return float64(m().IndexdbSizeBytes)
})
ms.NewGauge(`vl_data_size_bytes{type="storage"}`, func() float64 {
dm := m()
return float64(dm.CompressedInmemorySize + dm.CompressedFileSize)
})
ms.NewGauge(`vl_compressed_data_size_bytes{type="inmemory"}`, func() float64 {
return float64(m().CompressedInmemorySize)
})

2
app/vmagent/README.md
View file

@ -1571,7 +1571,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
Supports array of values separated by comma or specified via multiple flags.
-remoteWrite.sendTimeout array
Timeout for sending a single block of data to the corresponding -remoteWrite.url
Timeout for sending a single block of data to the corresponding -remoteWrite.url (default 1m)
Supports array of values separated by comma or specified via multiple flags.
-remoteWrite.shardByURL
Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages

9
app/vmagent/remotewrite/client.go
View file

@ -29,8 +29,9 @@ var (
rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
"is sent after temporary unavailability of the remote storage")
sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url (default "+
defaultSendTimeout.String()+")")
proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
@ -72,6 +73,8 @@ var (
awsSecretKey = flagutil.NewArrayString("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
)
const defaultSendTimeout = time.Minute
type client struct {
sanitizedURL string
remoteWriteURL string
@ -134,7 +137,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
}
hc := &http.Client{
Transport: tr,
Timeout: sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute),
Timeout: sendTimeout.GetOptionalArgOrDefault(argIdx, defaultSendTimeout),
}
c := &client{
sanitizedURL: sanitizedURL,

13
app/vmagent/remotewrite/remotewrite.go
View file

@ -10,8 +10,6 @@ import (
"sync/atomic"
"time"
"github.com/cespare/xxhash/v2"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@ -28,6 +26,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
"github.com/VictoriaMetrics/metrics"
"github.com/cespare/xxhash/v2"
)
var (
@ -705,11 +704,13 @@ var matchIdxsPool bytesutil.ByteBufferPool
func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, dropInput bool) []prompbmarshal.TimeSeries {
dst := src[:0]
for i, match := range matchIdxs {
if match == 0 {
continue
if !dropInput {
for i, match := range matchIdxs {
if match == 1 {
continue
}
dst = append(dst, src[i])
}
dst = append(dst, src[i])
}
tail := src[len(dst):]
_ = prompbmarshal.ResetTimeSeries(tail)

14
app/vmalert/README.md
View file

@ -117,11 +117,11 @@ name: <string>
[ limit: <int> | default = 0 ]
# How many rules execute at once within a group. Increasing concurrency may speed
# up round execution speed.
# up group's evaluation duration (exposed via `vmalert_iteration_duration_seconds` metric).
[ concurrency: <integer> | default = 1 ]
# Optional type for expressions inside the rules. Supported values: "graphite" and "prometheus".
# By default "prometheus" type is used.
# By default, "prometheus" type is used.
[ type: <string> ]
# Optional list of HTTP URL parameters
@ -361,9 +361,9 @@ For recording rules to work `-remoteWrite.url` must be specified.
### Alerts state on restarts
`vmalert` is stateless, it holds alerts state in the process memory. Restarting of `vmalert` process
will reset alerts state in memory. To prevent `vmalert` from losing alerts state it should be configured
to persist the state to the remote destination via the following flags:
`vmalert` holds alerts state in the memory. Restart of the `vmalert` process will reset the state of all active alerts
in the memory. To prevent `vmalert` from losing the state on restarts configure it to persist the state
to the remote database via the following flags:
* `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state
to the configured address in the form of [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series)
@ -519,7 +519,7 @@ Alertmanagers.
To avoid recording rules results and alerts state duplication in VictoriaMetrics server
don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
The recommended value for `-dedup.minScrapeInterval` must be greater or equal to vmalert `evaluation_interval`.
The recommended value for `-dedup.minScrapeInterval` must be multiple of vmalert's `evaluation_interval`.
If you observe inconsistent or "jumping" values in series produced by vmalert, try disabling `-datasource.queryTimeAlignment`
command line flag. Because of alignment, two or more vmalert HA pairs will produce results with the same timestamps.
But due of backfilling (data delivered to the datasource with some delay) values of such results may differ,
@ -1243,8 +1243,6 @@ The shortlist of configuration flags is the following:
See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage
Supports an array of values separated by comma or specified via multiple flags.
-rule.configCheckInterval duration
Interval for checking for changes in '-rule' files. By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead
-rule.maxResolveDuration duration
Limits the maximum duration for automatic alert expiration, which by default is 4 times evaluationInterval of the parent group.
-rule.resendDelay duration

62
app/vmalert/main.go
View file

@ -55,9 +55,6 @@ absolute path to all .tpl files in root.
-rule.templates="dir/**/*.tpl". Includes all the .tpl files in "dir" subfolders recursively.
`)
rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+
"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead")
configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
@ -311,10 +308,6 @@ See the docs at https://docs.victoriametrics.com/vmalert.html .
func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sighupCh <-chan os.Signal) {
var configCheckCh <-chan time.Time
checkInterval := *configCheckInterval
if checkInterval == 0 && *rulesCheckInterval > 0 {
logger.Warnf("flag `rule.configCheckInterval` is deprecated - use `configCheckInterval` instead")
checkInterval = *rulesCheckInterval
}
if checkInterval > 0 {
ticker := time.NewTicker(checkInterval)
configCheckCh = ticker.C
@ -326,8 +319,9 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
validateTplFn = notifier.ValidateTemplates
}
// init reload metrics with positive values to improve alerting conditions
setConfigSuccess(fasttime.UnixTimestamp())
// init metrics for config state with positive values to improve alerting conditions
setConfigSuccessAt(fasttime.UnixTimestamp())
parseFn := config.Parse
for {
select {
@ -365,11 +359,12 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
}
if configsEqual(newGroupsCfg, groupsCfg) {
templates.Reload()
// set success to 1 since previous reload
// could have been unsuccessful
// set success to 1 since previous reload could have been unsuccessful
// do not update configTimestamp as config version remains old.
configSuccess.Set(1)
setConfigError(nil)
// config didn't change - skip it
// reset the last config error since the config change was rolled back
setLastConfigErr(nil)
// config didn't change - skip iteration
continue
}
if err := m.update(ctx, newGroupsCfg, false); err != nil {
@ -379,7 +374,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
}
templates.Reload()
groupsCfg = newGroupsCfg
setConfigSuccess(fasttime.UnixTimestamp())
setConfigSuccessAt(fasttime.UnixTimestamp())
logger.Infof("Rules reloaded successfully from %q", *rulePath)
}
}
@ -396,39 +391,36 @@ func configsEqual(a, b []config.Group) bool {
return true
}
// setConfigSuccess sets config reload status to 1.
func setConfigSuccess(at uint64) {
// setConfigSuccessAt updates config related metrics as successful.
func setConfigSuccessAt(at uint64) {
configSuccess.Set(1)
configTimestamp.Set(at)
// reset the error if any
setConfigErr(nil)
// reset the lastConfigErr
setLastConfigErr(nil)
}
// setConfigError sets config reload status to 0.
// setConfigError updates config related metrics according to the error.
func setConfigError(err error) {
configReloadErrors.Inc()
configSuccess.Set(0)
setConfigErr(err)
setLastConfigErr(err)
}
var (
configErrMu sync.RWMutex
// configErr represent the error message from the last
// config reload.
configErr error
lastConfigErrMu sync.RWMutex
// lastConfigErr represent the error message from the last config reload.
// The message is used in web UI as notification
lastConfigErr error
)
func setConfigErr(err error) {
configErrMu.Lock()
configErr = err
configErrMu.Unlock()
func setLastConfigErr(err error) {
lastConfigErrMu.Lock()
lastConfigErr = err
lastConfigErrMu.Unlock()
}
func configError() error {
configErrMu.RLock()
defer configErrMu.RUnlock()
if configErr != nil {
return configErr
}
return nil
func getLastConfigError() error {
lastConfigErrMu.RLock()
defer lastConfigErrMu.RUnlock()
return lastConfigErr
}

36
app/vmalert/main_test.go
View file

@ -90,9 +90,10 @@ groups:
if err != nil {
t.Fatal(err)
}
defer func() { _ = os.Remove(f.Name()) }()
writeToFile(t, f.Name(), rules1)
*rulesCheckInterval = 200 * time.Millisecond
*configCheckInterval = 200 * time.Millisecond
*rulePath = []string{f.Name()}
ctx, cancel := context.WithCancel(context.Background())
@ -117,14 +118,37 @@ groups:
return len(m.groups)
}
time.Sleep(*rulesCheckInterval * 2)
checkCfg := func(err error) {
cErr := getLastConfigError()
cfgSuc := configSuccess.Get()
if err != nil {
if cErr == nil {
t.Fatalf("expected to have config error %s; got nil instead", cErr)
}
if cfgSuc != 0 {
t.Fatalf("expected to have metric configSuccess to be set to 0; got %d instead", cfgSuc)
}
return
}
if cErr != nil {
t.Fatalf("unexpected config error: %s", cErr)
}
if cfgSuc != 1 {
t.Fatalf("expected to have metric configSuccess to be set to 1; got %d instead", cfgSuc)
}
}
time.Sleep(*configCheckInterval * 2)
checkCfg(nil)
groupsLen := lenLocked(m)
if groupsLen != 1 {
t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
}
writeToFile(t, f.Name(), rules2)
time.Sleep(*rulesCheckInterval * 2)
time.Sleep(*configCheckInterval * 2)
checkCfg(nil)
groupsLen = lenLocked(m)
if groupsLen != 2 {
fmt.Println(m.groups)
@ -133,7 +157,8 @@ groups:
writeToFile(t, f.Name(), rules1)
procutil.SelfSIGHUP()
time.Sleep(*rulesCheckInterval / 2)
time.Sleep(*configCheckInterval / 2)
checkCfg(nil)
groupsLen = lenLocked(m)
if groupsLen != 1 {
t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
@ -141,7 +166,8 @@ groups:
writeToFile(t, f.Name(), `corrupted`)
procutil.SelfSIGHUP()
time.Sleep(*rulesCheckInterval / 2)
time.Sleep(*configCheckInterval / 2)
checkCfg(fmt.Errorf("config error"))
groupsLen = lenLocked(m)
if groupsLen != 1 { // should remain unchanged
t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)

4
app/vmalert/remotewrite/remotewrite.go
View file

@ -216,7 +216,9 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
retryInterval = maxRetryInterval
}
timeStart := time.Now()
defer sendDuration.Add(time.Since(timeStart).Seconds())
defer func() {
sendDuration.Add(time.Since(timeStart).Seconds())
}()
L:
for attempts := 0; ; attempts++ {
err := c.send(ctx, b)

58
app/vmalert/web.go
View file

@ -138,27 +138,8 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
return true
default:
// Support of deprecated links:
// * /api/v1/<groupID>/<alertID>/status
// * <groupID>/<alertID>/status
// TODO: to remove in next versions
if !strings.HasSuffix(r.URL.Path, "/status") {
httpserver.Errorf(w, r, "unsupported path requested: %q ", r.URL.Path)
return false
}
alert, err := rh.alertByPath(strings.TrimPrefix(r.URL.Path, "/api/v1/"))
if err != nil {
httpserver.Errorf(w, r, "%s", err)
return true
}
redirectURL := alert.WebLink()
if strings.HasPrefix(r.URL.Path, "/api/v1/") {
redirectURL = alert.APILink()
}
httpserver.Redirect(w, "/"+redirectURL)
return true
httpserver.Errorf(w, r, "unsupported path requested: %q ", r.URL.Path)
return false
}
}
@ -302,41 +283,6 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
return b, nil
}
func (rh *requestHandler) alertByPath(path string) (*APIAlert, error) {
if strings.HasPrefix(path, "/vmalert") {
path = strings.TrimLeft(path, "/vmalert")
}
parts := strings.SplitN(strings.TrimLeft(path, "/"), "/", -1)
if len(parts) != 3 {
return nil, &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/groupID/alertID/status"`, path),
StatusCode: http.StatusBadRequest,
}
}
groupID, err := uint64FromPath(parts[0])
if err != nil {
return nil, badRequest(fmt.Errorf(`cannot parse groupID: %w`, err))
}
alertID, err := uint64FromPath(parts[1])
if err != nil {
return nil, badRequest(fmt.Errorf(`cannot parse alertID: %w`, err))
}
resp, err := rh.m.AlertAPI(groupID, alertID)
if err != nil {
return nil, errResponse(err, http.StatusNotFound)
}
return resp, nil
}
func uint64FromPath(path string) (uint64, error) {
s := strings.TrimRight(path, "/")
return strconv.ParseUint(s, 10, 0)
}
func badRequest(err error) *httpserver.ErrorWithStatusCode {
return errResponse(err, http.StatusBadRequest)
}
func errResponse(err error, sc int) *httpserver.ErrorWithStatusCode {
return &httpserver.ErrorWithStatusCode{
Err: err,

12
app/vmalert/web.qtpl
View file

@ -12,7 +12,7 @@
{% func Welcome(r *http.Request) %}
{%= tpl.Header(r, navItems, "vmalert", configError()) %}
{%= tpl.Header(r, navItems, "vmalert", getLastConfigError()) %}
<p>
API:<br>
{% for _, p := range apiLinks %}
@ -40,7 +40,7 @@ btn-primary
{% func ListGroups(r *http.Request, originGroups []APIGroup) %}
{%code prefix := utils.Prefix(r.URL.Path) %}
{%= tpl.Header(r, navItems, "Groups", configError()) %}
{%= tpl.Header(r, navItems, "Groups", getLastConfigError()) %}
{%code
filter := r.URL.Query().Get("filter")
rOk := make(map[string]int)
@ -168,7 +168,7 @@ btn-primary
{% func ListAlerts(r *http.Request, groupAlerts []GroupAlerts) %}
{%code prefix := utils.Prefix(r.URL.Path) %}
{%= tpl.Header(r, navItems, "Alerts", configError()) %}
{%= tpl.Header(r, navItems, "Alerts", getLastConfigError()) %}
{% if len(groupAlerts) > 0 %}
<a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
<a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
@ -255,7 +255,7 @@ btn-primary
{% endfunc %}
{% func ListTargets(r *http.Request, targets map[notifier.TargetType][]notifier.Target) %}
{%= tpl.Header(r, navItems, "Notifiers", configError()) %}
{%= tpl.Header(r, navItems, "Notifiers", getLastConfigError()) %}
{% if len(targets) > 0 %}
<a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
<a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
@ -312,7 +312,7 @@ btn-primary
{% func Alert(r *http.Request, alert *APIAlert) %}
{%code prefix := utils.Prefix(r.URL.Path) %}
{%= tpl.Header(r, navItems, "", configError()) %}
{%= tpl.Header(r, navItems, "", getLastConfigError()) %}
{%code
var labelKeys []string
for k := range alert.Labels {
@ -399,7 +399,7 @@ btn-primary
{% func RuleDetails(r *http.Request, rule APIRule) %}
{%code prefix := utils.Prefix(r.URL.Path) %}
{%= tpl.Header(r, navItems, "", configError()) %}
{%= tpl.Header(r, navItems, "", getLastConfigError()) %}
{%code
var labelKeys []string
for k := range rule.Labels {

12
app/vmalert/web.qtpl.go
View file

@ -34,7 +34,7 @@ func StreamWelcome(qw422016 *qt422016.Writer, r *http.Request) {
qw422016.N().S(`
`)
//line app/vmalert/web.qtpl:15
tpl.StreamHeader(qw422016, r, navItems, "vmalert", configError())
tpl.StreamHeader(qw422016, r, navItems, "vmalert", getLastConfigError())
//line app/vmalert/web.qtpl:15
qw422016.N().S(`
<p>
@ -207,7 +207,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, originGroups [
qw422016.N().S(`
`)
//line app/vmalert/web.qtpl:43
tpl.StreamHeader(qw422016, r, navItems, "Groups", configError())
tpl.StreamHeader(qw422016, r, navItems, "Groups", getLastConfigError())
//line app/vmalert/web.qtpl:43
qw422016.N().S(`
`)
@ -647,7 +647,7 @@ func StreamListAlerts(qw422016 *qt422016.Writer, r *http.Request, groupAlerts []
qw422016.N().S(`
`)
//line app/vmalert/web.qtpl:171
tpl.StreamHeader(qw422016, r, navItems, "Alerts", configError())
tpl.StreamHeader(qw422016, r, navItems, "Alerts", getLastConfigError())
//line app/vmalert/web.qtpl:171
qw422016.N().S(`
`)
@ -922,7 +922,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n
qw422016.N().S(`
`)
//line app/vmalert/web.qtpl:258
tpl.StreamHeader(qw422016, r, navItems, "Notifiers", configError())
tpl.StreamHeader(qw422016, r, navItems, "Notifiers", getLastConfigError())
//line app/vmalert/web.qtpl:258
qw422016.N().S(`
`)
@ -1102,7 +1102,7 @@ func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) {
qw422016.N().S(`
`)
//line app/vmalert/web.qtpl:315
tpl.StreamHeader(qw422016, r, navItems, "", configError())
tpl.StreamHeader(qw422016, r, navItems, "", getLastConfigError())
//line app/vmalert/web.qtpl:315
qw422016.N().S(`
`)
@ -1311,7 +1311,7 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule)
qw422016.N().S(`
`)
//line app/vmalert/web.qtpl:402
tpl.StreamHeader(qw422016, r, navItems, "", configError())
tpl.StreamHeader(qw422016, r, navItems, "", getLastConfigError())
//line app/vmalert/web.qtpl:402
qw422016.N().S(`
`)

17
app/vmalert/web_test.go
View file

@ -145,23 +145,6 @@ func TestHandler(t *testing.T) {
t.Errorf("expected 1 group got %d", length)
}
})
// check deprecated links support
// TODO: remove as soon as deprecated links removed
t.Run("/api/v1/0/0/status", func(t *testing.T) {
alert := &APIAlert{}
getResp(ts.URL+"/api/v1/0/0/status", alert, 200)
expAlert := ar.newAlertAPI(*ar.alerts[0])
if !reflect.DeepEqual(alert, expAlert) {
t.Errorf("expected %v is equal to %v", alert, expAlert)
}
})
t.Run("/api/v1/0/1/status", func(t *testing.T) {
getResp(ts.URL+"/api/v1/0/1/status", nil, 404)
})
t.Run("/api/v1/1/0/status", func(t *testing.T) {
getResp(ts.URL+"/api/v1/1/0/status", nil, 404)
})
}
func TestEmptyResponse(t *testing.T) {

2
app/vmauth/README.md
View file

@ -351,6 +351,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
Prefix for environment variables if -envflag.enable is set
-eula
By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
-failTimeout duration
Sets a delay period for load balancing to skip a malfunctioning backend. (defaults 3s)
-flagsAuthKey string
Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
-fs.disableMmap

2
app/vmauth/auth_config.go
View file

@ -134,7 +134,7 @@ func (bu *backendURL) isBroken() bool {
}
func (bu *backendURL) setBroken() {
deadline := fasttime.UnixTimestamp() + 3
deadline := fasttime.UnixTimestamp() + uint64((*failTimeout).Seconds())
atomic.StoreUint64(&bu.brokenDeadline, deadline)
}

1
app/vmauth/main.go
View file

@ -41,6 +41,7 @@ var (
reloadAuthKey = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend.")
)
func main() {

19
app/vmctl/flags.go
View file

@ -325,8 +325,9 @@ const (
vmNativeFilterTimeEnd = "vm-native-filter-time-end"
vmNativeStepInterval = "vm-native-step-interval"
vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive"
vmNativeDisableRetries = "vm-native-disable-retries"
vmNativeDisableBinaryProtocol = "vm-native-disable-binary-protocol"
vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive"
vmNativeDisableRetries = "vm-native-disable-retries"
vmNativeSrcAddr = "vm-native-src-addr"
vmNativeSrcUser = "vm-native-src-user"
@ -450,6 +451,14 @@ var (
Usage: "Defines whether to disable retries with backoff policy for migration process",
Value: false,
},
&cli.BoolFlag{
Name: vmNativeDisableBinaryProtocol,
Usage: "Whether to use https://docs.victoriametrics.com/#how-to-export-data-in-json-line-format" +
"instead of https://docs.victoriametrics.com/#how-to-export-data-in-native-format API." +
"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks." +
"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
Value: false,
},
}
)
@ -468,6 +477,7 @@ const (
remoteReadHTTPTimeout = "remote-read-http-timeout"
remoteReadHeaders = "remote-read-headers"
remoteReadInsecureSkipVerify = "remote-read-insecure-skip-verify"
remoteReadDisablePathAppend = "remote-read-disable-path-append"
)
var (
@ -544,6 +554,11 @@ var (
Usage: "Whether to skip TLS certificate verification when connecting to the remote read address",
Value: false,
},
&cli.BoolFlag{
Name: remoteReadDisablePathAppend,
Usage: "Whether to disable automatic appending of the /api/v1/read suffix to --remote-read-src-addr",
Value: false,
},
}
)

2
app/vmctl/main.go
View file

@ -133,6 +133,7 @@ func main() {
LabelName: c.String(remoteReadFilterLabel),
LabelValue: c.String(remoteReadFilterLabelValue),
InsecureSkipVerify: c.Bool(remoteReadInsecureSkipVerify),
DisablePathAppend: c.Bool(remoteReadDisablePathAppend),
})
if err != nil {
return fmt.Errorf("error create remote read client: %s", err)
@ -254,6 +255,7 @@ func main() {
cc: c.Int(vmConcurrency),
disableRetries: c.Bool(vmNativeDisableRetries),
isSilent: c.Bool(globalSilent),
isNative: !c.Bool(vmNativeDisableBinaryProtocol),
}
return p.run(ctx)
},

62
app/vmctl/remoteread/remoteread.go
View file

@ -7,6 +7,7 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
@ -23,7 +24,6 @@ import (
const (
defaultReadTimeout = 5 * time.Minute
remoteReadPath = "/api/v1/read"
healthPath = "/-/healthy"
)
// StreamCallback is a callback function for processing time series
@ -32,19 +32,22 @@ type StreamCallback func(series *vm.TimeSeries) error
// Client is an HTTP client for reading
// time series via remote read protocol.
type Client struct {
addr string
c *http.Client
user string
password string
useStream bool
headers []keyValue
matchers []*prompb.LabelMatcher
addr string
disablePathAppend bool
c *http.Client
user string
password string
useStream bool
headers []keyValue
matchers []*prompb.LabelMatcher
}
// Config is config for remote read.
type Config struct {
// Addr of remote storage
Addr string
// DisablePathAppend disable automatic appending of the remote read path
DisablePathAppend bool
// Timeout defines timeout for HTTP requests
// made by remote read client
Timeout time.Duration
@ -105,13 +108,15 @@ func NewClient(cfg Config) (*Client, error) {
Timeout: cfg.Timeout,
Transport: utils.Transport(cfg.Addr, cfg.InsecureSkipVerify),
},
addr: strings.TrimSuffix(cfg.Addr, "/"),
user: cfg.Username,
password: cfg.Password,
useStream: cfg.UseStream,
headers: headers,
matchers: []*prompb.LabelMatcher{m},
addr: strings.TrimSuffix(cfg.Addr, "/"),
disablePathAppend: cfg.DisablePathAppend,
user: cfg.Username,
password: cfg.Password,
useStream: cfg.UseStream,
headers: headers,
matchers: []*prompb.LabelMatcher{m},
}
return c, nil
}
@ -154,27 +159,18 @@ func (c *Client) do(req *http.Request) (*http.Response, error) {
return c.c.Do(req)
}
// Ping checks the health of the read source
func (c *Client) Ping() error {
url := c.addr + healthPath
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return fmt.Errorf("cannot create request to %q: %s", url, err)
}
resp, err := c.do(req)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("bad status code: %d", resp.StatusCode)
}
return nil
}
func (c *Client) fetch(ctx context.Context, data []byte, streamCb StreamCallback) error {
r := bytes.NewReader(data)
url := c.addr + remoteReadPath
req, err := http.NewRequest(http.MethodPost, url, r)
// by default, we are using a common remote read path
u, err := url.JoinPath(c.addr, remoteReadPath)
if err != nil {
return fmt.Errorf("error create url from addr %s and default remote read path %s", c.addr, remoteReadPath)
}
// we should use full address from the remote-read-src-addr flag
if c.disablePathAppend {
u = c.addr
}
req, err := http.NewRequest(http.MethodPost, u, r)
if err != nil {
return fmt.Errorf("failed to create new HTTP request: %w", err)
}

6
app/vmctl/stepper/split.go
View file

@ -10,6 +10,8 @@ const (
StepMonth string = "month"
// StepDay represents a one day interval
StepDay string = "day"
// StepWeek represents a one week interval
StepWeek string = "week"
// StepHour represents a one hour interval
StepHour string = "hour"
// StepMinute represents a one minute interval
@ -40,6 +42,10 @@ func SplitDateRange(start, end time.Time, step string) ([][]time.Time, error) {
nextStep = func(t time.Time) (time.Time, time.Time) {
return t, t.AddDate(0, 0, 1)
}
case StepWeek:
nextStep = func(t time.Time) (time.Time, time.Time) {
return t, t.Add(7 * 24 * time.Hour)
}
case StepHour:
nextStep = func(t time.Time) (time.Time, time.Time) {
return t, t.Add(time.Hour * 1)

76
app/vmctl/stepper/split_test.go
View file

@ -170,6 +170,82 @@ func Test_splitDateRange(t *testing.T) {
},
wantErr: false,
},
{
name: "week chunking with not full week",
args: args{
start: "2023-07-30T00:00:00Z",
end: "2023-08-05T23:59:59.999999999Z",
granularity: StepWeek,
},
want: []testTimeRange{
{
"2023-07-30T00:00:00Z",
"2023-08-05T23:59:59.999999999Z",
},
},
},
{
name: "week chunking with start of the week and end of the week",
args: args{
start: "2023-07-30T00:00:00Z",
end: "2023-08-06T00:00:00Z",
granularity: StepWeek,
},
want: []testTimeRange{
{
"2023-07-30T00:00:00Z",
"2023-08-06T00:00:00Z",
},
},
},
{
name: "week chunking with next one day week",
args: args{
start: "2023-07-30T00:00:00Z",
end: "2023-08-07T01:12:00Z",
granularity: StepWeek,
},
want: []testTimeRange{
{
"2023-07-30T00:00:00Z",
"2023-08-06T00:00:00Z",
},
{
"2023-08-06T00:00:00Z",
"2023-08-07T01:12:00Z",
},
},
},
{
name: "week chunking with month and not full week representation",
args: args{
start: "2023-07-30T00:00:00Z",
end: "2023-09-01T01:12:00Z",
granularity: StepWeek,
},
want: []testTimeRange{
{
"2023-07-30T00:00:00Z",
"2023-08-06T00:00:00Z",
},
{
"2023-08-06T00:00:00Z",
"2023-08-13T00:00:00Z",
},
{
"2023-08-13T00:00:00Z",
"2023-08-20T00:00:00Z",
},
{
"2023-08-20T00:00:00Z",
"2023-08-27T00:00:00Z",
},
{
"2023-08-27T00:00:00Z",
"2023-09-01T01:12:00Z",
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

12
app/vmctl/vm_native.go
View file

@ -34,11 +34,12 @@ type vmNativeProcessor struct {
cc int
disableRetries bool
isSilent bool
isNative bool
}
const (
nativeExportAddr = "api/v1/export/native"
nativeImportAddr = "api/v1/import/native"
nativeExportAddr = "api/v1/export"
nativeImportAddr = "api/v1/import"
nativeWithBackoffTpl = `{{ blue "%s:" }} {{ counters . }} {{ bar . "[" "█" (cycle . "█") "▒" "]" }} {{ percent . }}`
nativeSingleProcessTpl = `Total: {{counters . }} {{ cycle . "↖" "↗" "↘" "↙" }} Speed: {{speed . }} {{string . "suffix"}}`
)
@ -159,9 +160,14 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU
func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, ranges [][]time.Time, silent bool) error {
exportAddr := nativeExportAddr
importAddr := nativeImportAddr
if p.isNative {
exportAddr += "/native"
importAddr += "/native"
}
srcURL := fmt.Sprintf("%s/%s", p.src.Addr, exportAddr)
importAddr, err := vm.AddExtraLabelsToImportPath(nativeImportAddr, p.dst.ExtraLabels)
importAddr, err := vm.AddExtraLabelsToImportPath(importAddr, p.dst.ExtraLabels)
if err != nil {
return fmt.Errorf("failed to add labels to import path: %s", err)
}

1
app/vmctl/vm_native_test.go
View file

@ -228,6 +228,7 @@ func Test_vmNativeProcessor_run(t *testing.T) {
interCluster: tt.fields.interCluster,
cc: tt.fields.cc,
isSilent: tt.args.silent,
isNative: true,
}
if err := p.run(tt.args.ctx); (err != nil) != tt.wantErr {

2
app/vminsert/common/insert_ctx.go
View file

@ -171,7 +171,7 @@ func (ctx *InsertCtx) dropAggregatedRows(matchIdxs []byte) {
src := ctx.mrs
if !*streamAggrDropInput {
for idx, match := range matchIdxs {
if match != 0 {
if match == 1 {
continue
}
dst = append(dst, src[idx])

11
app/vmselect/promql/exec_test.go
View file

@ -5826,6 +5826,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`share_eq_over_time`, func(t *testing.T) {
t.Parallel()
q := `share_eq_over_time(round(5*rand(0))[200s:10s], 1)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{0.1, 0.2, 0.25, 0.1, 0.3, 0.3},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`count_gt_over_time`, func(t *testing.T) {
t.Parallel()
q := `count_gt_over_time(rand(0)[200s:10s], 0.7)`

5
app/vmselect/promql/rollup.go
View file

@ -78,6 +78,7 @@ var rollupFuncs = map[string]newRollupFunc{
"scrape_interval": newRollupFuncOneArg(rollupScrapeInterval),
"share_gt_over_time": newRollupShareGT,
"share_le_over_time": newRollupShareLE,
"share_eq_over_time": newRollupShareEQ,
"stale_samples_over_time": newRollupFuncOneArg(rollupStaleSamples),
"stddev_over_time": newRollupFuncOneArg(rollupStddev),
"stdvar_over_time": newRollupFuncOneArg(rollupStdvar),
@ -1106,6 +1107,10 @@ func countFilterGT(values []float64, gt float64) int {
return n
}
func newRollupShareEQ(args []interface{}) (rollupFunc, error) {
return newRollupShareFilter(args, countFilterEQ)
}
func countFilterEQ(values []float64, eq float64) int {
n := 0
for _, v := range values {

19
app/vmselect/promql/rollup_test.go
View file

@ -261,6 +261,25 @@ func TestRollupShareGTOverTime(t *testing.T) {
f(1000, 0)
}
func TestRollupShareEQOverTime(t *testing.T) {
f := func(eq, vExpected float64) {
t.Helper()
eqs := []*timeseries{{
Values: []float64{eq},
Timestamps: []int64{123},
}}
var me metricsql.MetricExpr
args := []interface{}{&metricsql.RollupExpr{Expr: &me}, eqs}
testRollupFunc(t, "share_eq_over_time", args, &me, vExpected)
}
f(-123, 0)
f(34, 0.3333333333333333)
f(44, 0.16666666666666666)
f(123, 0.08333333333333333)
f(1000, 0)
}
func TestRollupCountLEOverTime(t *testing.T) {
f := func(le, vExpected float64) {
t.Helper()

2
app/vmui/Dockerfile-web
View file

@ -1,4 +1,4 @@
FROM golang:1.20.6 as build-web-stage
FROM golang:1.21.0 as build-web-stage
COPY build /build
WORKDIR /build

5
app/vmui/packages/vmui/src/App.tsx
View file

@ -13,7 +13,6 @@ import ExploreMetrics from "./pages/ExploreMetrics";
import PreviewIcons from "./components/Main/Icons/PreviewIcons";
import WithTemplate from "./pages/WithTemplate";
import Relabel from "./pages/Relabel";
import ExploreLogs from "./pages/ExploreLogs/ExploreLogs";
import ActiveQueries from "./pages/ActiveQueries";
const App: FC = () => {
@ -70,10 +69,6 @@ const App: FC = () => {
path={router.icons}
element={<PreviewIcons/>}
/>
<Route
path={router.logs}
element={<ExploreLogs/>}
/>
</Route>
</Routes>
)}

1
app/vmui/packages/vmui/src/api/types.ts
View file

@ -24,6 +24,7 @@ export interface TracingData {
export interface QueryStats {
seriesFetched?: string;
resultLength?: number;
isPartial?: boolean;
}
export interface Logs {

24
app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx
View file

@ -7,6 +7,7 @@ import "./style.scss";
import { QueryStats } from "../../../api/types";
import Tooltip from "../../Main/Tooltip/Tooltip";
import { WarningIcon } from "../../Main/Icons";
import { partialWarning, seriesFetchedWarning } from "./warningText";
export interface QueryEditorProps {
onChange: (query: string) => void;
@ -39,7 +40,17 @@ const QueryEditor: FC<QueryEditorProps> = ({
const [openAutocomplete, setOpenAutocomplete] = useState(false);
const autocompleteAnchorEl = useRef<HTMLDivElement>(null);
const showSeriesFetchedWarning = stats?.seriesFetched === "0" && !stats.resultLength;
const warnings = [
{
show: stats?.seriesFetched === "0" && !stats.resultLength,
text: seriesFetchedWarning
},
{
show: stats?.isPartial,
text: partialWarning
}
].filter((warning) => warning.show);
const handleSelect = (val: string) => {
onChange(val);
@ -108,17 +119,14 @@ const QueryEditor: FC<QueryEditorProps> = ({
onFoundOptions={handleChangeFoundOptions}
/>
)}
{showSeriesFetchedWarning && (
{!!warnings.length && (
<div className="vm-query-editor-warning">
<Tooltip
placement="bottom-right"
title={(
<span className="vm-query-editor-warning__tooltip">
{`No match!
This query hasn't selected any time series from database.
Either the requested metrics are missing in the database,
or there is a typo in series selector.`}
</span>
<div className="vm-query-editor-warning__tooltip">
{warnings.map((warning, index) => <p key={index}>{warning.text}</p>)}
</div>
)}
>
<WarningIcon/>

8
app/vmui/packages/vmui/src/components/Configurators/QueryEditor/style.scss
View file

@ -22,6 +22,14 @@
&__tooltip {
white-space: pre-line;
p {
margin-bottom: $padding-small;
&:last-child {
margin-bottom: 0;
}
}
}
}
}

7
app/vmui/packages/vmui/src/components/Configurators/QueryEditor/warningText.ts Normal file
View file

@ -0,0 +1,7 @@
export const seriesFetchedWarning = `No match!
This query hasn't selected any time series from database.
Either the requested metrics are missing in the database,
or there is a typo in series selector.`;
export const partialWarning = `The shown results are marked as PARTIAL.
The result is marked as partial if one or more vmstorage nodes failed to respond to the query.`;

2
app/vmui/packages/vmui/src/components/Main/DatePicker/DateTimeInput/DateTimeInput.tsx
View file

@ -36,7 +36,7 @@ const DateTimeInput: FC<DateTimeInputProps> = ({
const [maskedValue, setMaskedValue] = useState(formatStringDate(value));
const [focusToTime, setFocusToTime] = useState(false);
const [awaitChangeForEnter, setAwaitChangeForEnter] = useState(false);
const error = dayjs(maskedValue).isValid() ? "" : "Expected format: YYYY-MM-DD HH:mm:ss";
const error = dayjs(maskedValue).isValid() ? "" : "Invalid date format";
const handleMaskedChange = (e: ChangeEvent<HTMLInputElement>) => {
setMaskedValue(e.currentTarget.value);

8
app/vmui/packages/vmui/src/components/Main/TextField/TextField.tsx
View file

@ -3,6 +3,7 @@ import classNames from "classnames";
import { useMemo } from "preact/compat";
import { useAppState } from "../../../state/common/StateContext";
import useDeviceDetect from "../../../hooks/useDeviceDetect";
import TextFieldError from "./TextFieldError";
import "./style.scss";
interface TextFieldProps {
@ -132,12 +133,7 @@ const TextField: FC<TextFieldProps> = ({
)
}
{label && <span className="vm-text-field__label">{label}</span>}
<span
className="vm-text-field__error"
data-show={!!error}
>
{error}
</span>
<TextFieldError error={error}/>
{helperText && !error && (
<span className="vm-text-field__helper-text">
{helperText}

56
app/vmui/packages/vmui/src/components/Main/TextField/TextFieldError.tsx Normal file
View file

@ -0,0 +1,56 @@
import React, { FC, useEffect, useRef, useState } from "react";
import useEventListener from "../../../hooks/useEventListener";
import classNames from "classnames";
import "./style.scss";
interface TextFieldErrorProps {
error: string;
}
const TextFieldError: FC<TextFieldErrorProps> = ({ error }) => {
const errorRef = useRef<HTMLSpanElement>(null);
const [isErrorTruncated, setIsErrorTruncated] = useState(false);
const [showFull, setShowFull] = useState(false);
const checkIfTextTruncated = () => {
const el = errorRef.current;
if (el) {
const { offsetWidth, scrollWidth, offsetHeight, scrollHeight } = el;
// The "+1" is for the scrollbar in Firefox
const overflowed = (offsetWidth + 1) < scrollWidth || (offsetHeight + 1) < scrollHeight;
setIsErrorTruncated(overflowed);
} else {
setIsErrorTruncated(false);
}
};
const handleClickError = () => {
if (!isErrorTruncated) return;
setShowFull(true);
setIsErrorTruncated(false);
};
useEffect(() => {
setShowFull(false);
checkIfTextTruncated();
}, [errorRef, error]);
useEventListener("resize", checkIfTextTruncated);
return (
<span
className={classNames({
"vm-text-field__error": true,
"vm-text-field__error_overflowed": isErrorTruncated,
"vm-text-field__error_full": showFull,
})}
data-show={!!error}
ref={errorRef}
onClick={handleClickError}
>
{error}
</span>
);
};
export default TextFieldError;

27
app/vmui/packages/vmui/src/components/Main/TextField/style.scss
View file

@ -40,14 +40,9 @@
overflow: hidden;
text-overflow: ellipsis;
display: -webkit-box;
-webkit-line-clamp: 2; /* number of lines to show */
line-clamp: 2;
-webkit-line-clamp: 1; /* number of lines to show */
line-clamp: 1;
-webkit-box-orient: vertical;
@media (max-width: 500px) {
-webkit-line-clamp: 1; /* number of lines to show */
line-clamp: 1;
}
}
&__label {
@ -56,10 +51,22 @@
}
&__error {
top: calc((100% - ($font-size-small/2)) - 2px);
position: relative;
top: calc($font-size-small/-2);
width: fit-content;
overflow-wrap: anywhere;
color: $color-error;
pointer-events: auto;
user-select: text;
&_full {
display: block;
overflow: visible;
}
&_overflowed {
cursor: pointer;
}
}
&__helper-text {
@ -117,9 +124,9 @@
align-items: center;
justify-content: center;
max-width: 15px;
top: auto;
top: 0;
left: $padding-small;
height: 100%;
height: 40px;
position: absolute;
color: $color-text-secondary;
}

2
app/vmui/packages/vmui/src/components/Main/Tooltip/Tooltip.tsx
View file

@ -41,7 +41,7 @@ const Tooltip: FC<TooltipProps> = ({
return () => {
window.removeEventListener("scroll", onScrollWindow);
};
}, [isOpen]);
}, [isOpen, title]);
const popperStyle = useMemo(() => {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment

4
app/vmui/packages/vmui/src/constants/navigation.ts
View file

@ -38,10 +38,6 @@ export const defaultNavigation: NavigationItem[] = [
label: routerOptions[router.activeQueries].title,
value: router.activeQueries,
},
{
label: routerOptions[router.logs].title,
value: router.logs,
},
]
},
{

1
app/vmui/packages/vmui/src/hooks/useFetchQuery.ts
View file

@ -103,6 +103,7 @@ export const useFetchQuery = ({
if (response.ok) {
setQueryStats(prev => [...prev, {
...resp?.stats,
isPartial: resp?.isPartial,
resultLength: resp.data.result.length,
}]);
setQueryErrors(prev => [...prev, ""]);

16
app/vmui/packages/vmui/src/styles/core.scss
View file

@ -52,6 +52,7 @@ input[type=number]::-webkit-outer-spin-button {
left: $padding-global;
bottom: $padding-global;
z-index: 999;
animation: vm-slide-snackbar 150ms cubic-bezier(0.280, 0.840, 0.420, 1.1);
&-content {
display: grid;
@ -71,15 +72,14 @@ input[type=number]::-webkit-outer-spin-button {
bottom: 0;
left: 0;
right: 0;
animation: vm-slide-snackbar 150ms cubic-bezier(0.280, 0.840, 0.420, 1.1);
}
@keyframes vm-slide-snackbar {
0% {
transform: translateY(100%);
}
100% {
transform: translateY(0);
}
@keyframes vm-slide-snackbar {
0% {
transform: translateY(100%);
}
100% {
transform: translateY(0);
}
}
}

4
dashboards/victorialogs.json
View file

@ -98,7 +98,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(vl_rows)",
"expr": "sum(vl_storage_rows)",
"format": "time_series",
"instant": true,
"interval": "",
@ -776,4 +776,4 @@
"uid": "OqPIZTX4z",
"version": 4,
"weekStart": ""
}
}

2
dashboards/victoriametrics-cluster.json
View file

@ -624,7 +624,7 @@
"uid": "$ds"
},
"exemplar": true,
"expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"}) / sum(vm_rows{job=~\"$job_storage\"})",
"expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"}) / sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})",
"format": "time_series",
"instant": true,
"interval": "",

2
dashboards/victoriametrics.json
View file

@ -4434,7 +4434,7 @@
"uid": "$ds"
},
"editorMode": "code",
"expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) \n/ sum(vm_rows{job=~\"$job\", instance=~\"$instance\"})",
"expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) \n/ sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"})",
"format": "time_series",
"interval": "",
"intervalFactor": 1,

209
dashboards/vmagent.json
View file

@ -6,7 +6,7 @@
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "9.2.6"
"version": "9.2.7"
},
{
"type": "datasource",
@ -182,7 +182,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -249,7 +249,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -310,7 +310,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -379,7 +379,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -451,7 +451,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -515,7 +515,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -606,7 +606,7 @@
},
"showHeader": true
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -2372,8 +2372,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2475,8 +2474,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2581,8 +2579,7 @@
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
"color": "transparent"
},
{
"color": "red",
@ -2687,8 +2684,7 @@
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
"color": "transparent"
},
{
"color": "red",
@ -2792,8 +2788,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2898,8 +2893,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3009,8 +3003,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3113,8 +3106,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3188,8 +3180,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4054,7 +4045,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4070,7 +4062,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 46
"y": 38
},
"id": 73,
"links": [],
@ -4170,7 +4162,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4186,7 +4179,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 46
"y": 38
},
"id": 77,
"links": [],
@ -4233,6 +4226,123 @@
],
"title": "Error rate ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "Shows how many concurrent inserts are taking place.\n\nIf the number of concurrent inserts hitting the `limit` or is close to the `limit` constantly - it might be a sign of a resource shortage.\n\n If vmagent's CPU usage and remote write connection saturation are at normal level, it might be that `-maxConcurrentInserts` cmd-line flag need to be increased.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 46
},
"id": 130,
"links": [],
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": true,
"expr": "max_over_time(vm_concurrent_insert_current{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
"interval": "",
"legendFormat": "{{instance}} ({{job}})",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": true,
"expr": "min(vm_concurrent_insert_capacity{job=~\"$job\", instance=~\"$instance\"}) by(job)",
"interval": "",
"legendFormat": "limit ({{job}})",
"range": true,
"refId": "B"
}
],
"title": "Concurrent inserts ($instance)",
"type": "timeseries"
}
],
"targets": [
@ -4310,7 +4420,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4326,7 +4437,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 47
"y": 55
},
"id": 60,
"options": {
@ -4412,7 +4523,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4428,7 +4540,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 47
"y": 55
},
"id": 66,
"options": {
@ -4514,7 +4626,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4530,7 +4643,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 55
"y": 63
},
"id": 61,
"options": {
@ -4616,7 +4729,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4632,7 +4746,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 55
"y": 63
},
"id": 65,
"options": {
@ -4717,7 +4831,8 @@
"mode": "absolute",
"steps": [
{
"color": "transparent"
"color": "transparent",
"value": null
},
{
"color": "red",
@ -4733,7 +4848,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 63
"y": 71
},
"id": 88,
"options": {
@ -4815,7 +4930,8 @@
"mode": "absolute",
"steps": [
{
"color": "transparent"
"color": "transparent",
"value": null
},
{
"color": "red",
@ -4831,7 +4947,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 63
"y": 71
},
"id": 84,
"options": {
@ -4916,7 +5032,8 @@
"mode": "absolute",
"steps": [
{
"color": "transparent"
"color": "transparent",
"value": null
},
{
"color": "red",
@ -4932,7 +5049,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 71
"y": 79
},
"id": 90,
"options": {
@ -5605,8 +5722,8 @@
{
"current": {
"selected": true,
"text": "VictoriaMetrics",
"value": "VictoriaMetrics"
"text": "VictoriaMetrics - cluster",
"value": "VictoriaMetrics - cluster"
},
"hide": 0,
"includeAll": false,

318
dashboards/vmalert.json
View file

@ -6,7 +6,7 @@
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "9.2.6"
"version": "9.2.7"
},
{
"type": "datasource",
@ -204,7 +204,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -264,7 +264,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -324,7 +324,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -388,7 +388,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -452,7 +452,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -546,7 +546,7 @@
},
"showHeader": true
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -1182,7 +1182,7 @@
}
]
},
"pluginVersion": "9.2.6",
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@ -1243,6 +1243,230 @@
},
"id": 43,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "The precentage of used RSS memory\n\nIf you think that usage is abnormal or unexpected, please file an issue and attach memory profile if possible.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 33
},
"id": 37,
"links": [
{
"targetBlank": true,
"title": "Profiling",
"url": "https://docs.victoriametrics.com/vmagent.html#profiling"
}
],
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true,
"sortBy": "Last *",
"sortDesc": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Memory usage % ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "Amount of used RSS memory\n\nIf you think that usage is abnormal or unexpected, please file an issue and attach memory profile if possible.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 33
},
"id": 57,
"links": [
{
"targetBlank": true,
"title": "Profiling",
"url": "https://docs.victoriametrics.com/vmagent.html#profiling"
}
],
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true,
"sortBy": "Last *",
"sortDesc": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n) by(job)",
"interval": "",
"legendFormat": "{{job}}",
"range": true,
"refId": "A"
}
],
"title": "Memory usage ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
@ -1308,7 +1532,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 35
"y": 41
},
"id": 35,
"links": [
@ -1362,7 +1586,7 @@
"type": "prometheus",
"uid": "$ds"
},
"description": "Amount of used memory\n\nResident memory shows share which can be freed by OS when needed.\n\nAnonymous shows share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.\n\nIf you think that usage is abnormal or unexpected, please file an issue and attach memory profile if possible.",
"description": "Shows the max number of CPU cores used by a `job` and the corresponding limit.",
"fieldConfig": {
"defaults": {
"color": {
@ -1414,7 +1638,7 @@
}
]
},
"unit": "percentunit"
"unit": "short"
},
"overrides": []
},
@ -1422,9 +1646,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 35
"y": 41
},
"id": 37,
"id": 56,
"links": [
{
"targetBlank": true,
@ -1447,7 +1671,7 @@
},
"tooltip": {
"mode": "multi",
"sort": "none"
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
@ -1459,14 +1683,32 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
"expr": "max(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)",
"format": "time_series",
"interval": "",
"legendFormat": "__auto",
"intervalFactor": 1,
"legendFormat": "{{job}}",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": false,
"expr": "min(process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}) by(job)",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "limit ({{job}})",
"range": true,
"refId": "B"
}
],
"title": "Memory usage % ($instance)",
"title": "CPU usage ($instance)",
"type": "timeseries"
},
{
@ -1535,7 +1777,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 43
"y": 49
},
"id": 39,
"links": [],
@ -1641,7 +1883,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 43
"y": 49
},
"id": 41,
"links": [],
@ -1754,8 +1996,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -1857,8 +2098,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -1960,8 +2200,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2064,8 +2303,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2164,8 +2402,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2292,8 +2529,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2395,8 +2631,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2497,8 +2732,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2620,8 +2854,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2713,8 +2946,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -2776,9 +3008,9 @@
"list": [
{
"current": {
"selected": true,
"text": "VictoriaMetrics",
"value": "VictoriaMetrics"
"selected": false,
"text": "VictoriaMetrics - cluster",
"value": "VictoriaMetrics - cluster"
},
"hide": 0,
"includeAll": false,
@ -2862,7 +3094,7 @@
},
{
"current": {
"selected": true,
"selected": false,
"text": "5",
"value": "5"
},

18
deployment/docker/Makefile
View file

@ -9,22 +9,24 @@ ROOT_IMAGE ?= alpine:3.18.2
# TODO: sync it with ROOT_IMAGE when it will be fixed in the new alpine releases
CERTS_IMAGE := alpine:3.17.3
GO_BUILDER_IMAGE := golang:1.20.6-alpine
GO_BUILDER_IMAGE := golang:1.21.0-alpine
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
DOCKER_BUILD ?= docker build
DOCKER_COMPOSE ?= docker compose
DOCKER_IMAGE_LS ?= docker image ls --format '{{.Repository}}:{{.Tag}}'
package-base:
(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(BASE_IMAGE)$$') \
|| docker build \
($(DOCKER_IMAGE_LS) | grep -q '$(BASE_IMAGE)$$') \
|| $(DOCKER_BUILD) \
--build-arg root_image=$(ROOT_IMAGE) \
--build-arg certs_image=$(CERTS_IMAGE) \
--tag $(BASE_IMAGE) \
deployment/docker/base
package-builder:
(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(BUILDER_IMAGE)$$') \
|| docker build \
($(DOCKER_IMAGE_LS) | grep -q '$(BUILDER_IMAGE)$$') \
|| $(DOCKER_BUILD) \
--build-arg go_builder_image=$(GO_BUILDER_IMAGE) \
--tag $(BUILDER_IMAGE) \
deployment/docker/builder
@ -60,9 +62,9 @@ app-via-docker-windows: package-builder
-o bin/$(APP_NAME)-windows$(APP_SUFFIX)-prod.exe $(PKG_PREFIX)/app/$(APP_NAME)
package-via-docker: package-base
(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\
($(DOCKER_IMAGE_LS) | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\
$(MAKE) app-via-docker && \
docker build \
$(DOCKER_BUILD) \
--build-arg src_binary=$(APP_NAME)$(APP_SUFFIX)-prod \
--build-arg base_image=$(BASE_IMAGE) \
--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) \
@ -170,7 +172,7 @@ package-via-docker-386:
GOARCH=386 $(MAKE) package-via-docker-goarch-nocgo
remove-docker-images:
docker image ls --format '{{.Repository}}\t{{.ID}}' | awk '{print $$2}' | xargs docker image rm -f
docker image ls --format '{{.ID}}' | xargs docker image rm -f
docker-single-up:
$(DOCKER_COMPOSE) -f deployment/docker/docker-compose.yml up -d

19
deployment/docker/README.md
View file

@ -109,3 +109,22 @@ Grafana is provisioned by default with following entities:
* `VictoriaMetrics - vmalert` dashboard
Remember to pick `VictoriaMetrics - cluster` datasource when viewing `VictoriaMetrics - cluster` dashboard.
## Alerts
See below a list of recommended alerting rules for various VictoriaMetrics components for running in production.
Some of the alerting rules thresholds are just recommendations and could require an adjustment. The list
of alerting rules is the following:
* [alerts-health.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml):
alerting rules related to all VictoriaMetrics components for tracking their "health" state;
* [alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml):
alerting rules related to [single-server VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) installation;
* [alerts-cluster.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml):
alerting rules related to [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html);
* [alerts-vmagent.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmagent.yml):
alerting rules related to [vmagent](https://docs.victoriametrics.com/vmagent.html) component;
* [alerts-vmalert.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmalert.yml):
alerting rules related to [vmalert](https://docs.victoriametrics.com/vmalert.html) component;
Please, also see [how to monitor](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#monitoring)
VictoriaMetrics installations.

12
deployment/docker/alerts-cluster.yml
View file

@ -80,18 +80,6 @@ groups:
description: "RPC errors are interconnection errors between cluster components.\n
Possible reasons for errors are misconfiguration, overload, network blips or unreachable components."
- alert: ConcurrentFlushesHitTheLimit
expr: avg_over_time(vm_concurrent_insert_current[1m]) >= vm_concurrent_insert_capacity
for: 15m
labels:
severity: warning
show_at: dashboard
annotations:
dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=133&var-instance={{ $labels.instance }}"
summary: "vmstorage on instance {{ $labels.instance }} is constantly hitting concurrent flushes limit"
description: "The limit of concurrent flushes on instance {{ $labels.instance }} is equal to number of CPUs.\n
When vmstorage constantly hits the limit it means that storage is overloaded and requires more CPU."
- alert: RowsRejectedOnIngestion
expr: sum(rate(vm_rows_ignored_total[5m])) by (instance, reason) > 0
for: 15m

16
deployment/docker/alerts-health.yml
View file

@ -1,4 +1,5 @@
# File contains default list of alerts for VM components.
# File contains default list of alerts for various VM components.
# The following alerts are recommended for use for any VM installation.
# The alerts below are just recommendations and may require some updates
# and threshold calibration according to every specific setup.
groups:
@ -73,3 +74,16 @@ groups:
description: "The rate of TSID misses during query lookups is too high for \"{{ $labels.job }}\" ({{ $labels.instance }}).\n
Make sure you're running VictoriaMetrics of v1.85.3 or higher.\n
Related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502"
- alert: ConcurrentInsertsHitTheLimit
expr: avg_over_time(vm_concurrent_insert_current[1m]) >= vm_concurrent_insert_capacity
for: 15m
labels:
severity: warning
annotations:
summary: "{{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent inserts limit"
description: "The limit of concurrent inserts on instance {{ $labels.instance }} depends on the number of CPUs.\n
Usually, when component constantly hits the limit it is likely the component is overloaded and requires more CPU.
In some cases for components like vmagent or vminsert the alert might trigger if there are too many clients
making write attempts. If vmagent's or vminsert's CPU usage and network saturation are at normal level, then
it might be worth adjusting `-maxConcurrentInserts` cmd-line flag."

12
deployment/docker/alerts.yml
View file

@ -60,18 +60,6 @@ groups:
description: "Requests to path {{ $labels.path }} are receiving errors.
Please verify if clients are sending correct requests."
- alert: ConcurrentFlushesHitTheLimit
expr: avg_over_time(vm_concurrent_insert_current[1m]) >= vm_concurrent_insert_capacity
for: 15m
labels:
severity: warning
show_at: dashboard
annotations:
dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=59&var-instance={{ $labels.instance }}"
summary: "VictoriaMetrics on instance {{ $labels.instance }} is constantly hitting concurrent flushes limit"
description: "The limit of concurrent flushes on instance {{ $labels.instance }} is equal to number of CPUs.\n
When VictoriaMetrics constantly hits the limit it means that storage is overloaded and requires more CPU."
- alert: RowsRejectedOnIngestion
expr: sum(rate(vm_rows_ignored_total[5m])) by (instance, reason) > 0
for: 15m

12
deployment/docker/docker-compose-cluster.yml
View file

@ -2,7 +2,7 @@ version: '3.5'
services:
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.92.0
image: victoriametrics/vmagent:v1.92.1
depends_on:
- "vminsert"
ports:
@ -32,7 +32,7 @@ services:
vmstorage-1:
container_name: vmstorage-1
image: victoriametrics/vmstorage:v1.92.0-cluster
image: victoriametrics/vmstorage:v1.92.1-cluster
ports:
- 8482
- 8400
@ -44,7 +44,7 @@ services:
restart: always
vmstorage-2:
container_name: vmstorage-2
image: victoriametrics/vmstorage:v1.92.0-cluster
image: victoriametrics/vmstorage:v1.92.1-cluster
ports:
- 8482
- 8400
@ -56,7 +56,7 @@ services:
restart: always
vminsert:
container_name: vminsert
image: victoriametrics/vminsert:v1.92.0-cluster
image: victoriametrics/vminsert:v1.92.1-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@ -68,7 +68,7 @@ services:
restart: always
vmselect:
container_name: vmselect
image: victoriametrics/vmselect:v1.92.0-cluster
image: victoriametrics/vmselect:v1.92.1-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@ -82,7 +82,7 @@ services:
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.92.0
image: victoriametrics/vmalert:v1.92.1
depends_on:
- "vmselect"
ports:

6
deployment/docker/docker-compose.yml
View file

@ -2,7 +2,7 @@ version: "3.5"
services:
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.92.0
image: victoriametrics/vmagent:v1.92.1
depends_on:
- "victoriametrics"
ports:
@ -18,7 +18,7 @@ services:
restart: always
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.92.0
image: victoriametrics/victoria-metrics:v1.92.1
ports:
- 8428:8428
- 8089:8089
@ -56,7 +56,7 @@ services:
restart: always
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.92.0
image: victoriametrics/vmalert:v1.92.1
depends_on:
- "victoriametrics"
- "alertmanager"

5
deployment/docker/victorialogs/vector-docker/docker-compose.yml
View file

@ -20,11 +20,6 @@ services:
condition: service_healthy
victoriametrics:
condition: service_healthy
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8686/health"]
interval: 1s
timeout: 1s
retries: 10
# Run `make package-victoria-logs` to build victoria-logs image
victorialogs:

2
deployment/logs-benchmark/docker-compose.yml
View file

@ -105,7 +105,7 @@ services:
- '--config=/config.yml'
vmsingle:
image: victoriametrics/victoria-metrics:v1.92.0
image: victoriametrics/victoria-metrics:v1.92.1
ports:
- '8428:8428'
command:

2
deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md
View file

@ -8,7 +8,7 @@
4. Set variables `DIGITALOCEAN_API_TOKEN` with `VM_VERSION` for `packer` environment and run make from example below:
```console
make release-victoria-metrics-digitalocean-oneclick-droplet DIGITALOCEAN_API_TOKEN="dop_v23_2e46f4759ceeeba0d0248" VM_VERSION="1.92.0"
make release-victoria-metrics-digitalocean-oneclick-droplet DIGITALOCEAN_API_TOKEN="dop_v23_2e46f4759ceeeba0d0248" VM_VERSION="1.92.1"
```

4
deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click
View file

@ -19,8 +19,8 @@ On the server:
* VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface.
********************************************************************************
# This image includes 1.92.0 version of VictoriaMetrics.
# See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0
# This image includes 1.92.1 version of VictoriaMetrics.
# See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1
# Welcome to VictoriaMetrics droplet!

1
docs/Articles.md
View file

@ -73,6 +73,7 @@ See also [case studies](https://docs.victoriametrics.com/CaseStudies.html).
* [VictoriaMetrics static scraper](https://blog.differentpla.net/blog/2022/10/16/victoria-metrics-static-scraper/)
* [VictoriaMetrics and Open Cosmos boldly takes edge computing to the edge of space](https://www.iot-now.com/2022/07/19/122423-victoriametrics-and-open-cosmos-boldly-takes-edge-computing-to-the-edge-of-space/)
* [Evaluating Backend Options For Prometheus Metrics](https://www.techetio.com/2022/08/21/evaluating-backend-options-for-prometheus-metrics/)
* [Time Series in the Multiverse of Madness (in Korean)](https://www.youtube.com/watch?v=OUyXPgVcdw4), plus [these slides](https://deview.kr/data/deview/session/attach/%5B2B4%5DVictoriaMetrics_%E1%84%89%E1%85%B5%E1%84%80%E1%85%A8%E1%84%8B%E1%85%A7%E1%86%AF_%E1%84%83%E1%85%A6%E1%84%8B%E1%85%B5%E1%84%90%E1%85%A5_%E1%84%83%E1%85%A2%E1%84%92%E1%85%A9%E1%86%AB%E1%84%83%E1%85%A9%E1%86%AB%E1%84%8B%E1%85%B4_%E1%84%86%E1%85%A5%E1%86%AF%E1%84%90%E1%85%B5%E1%84%87%E1%85%A5%E1%84%89%E1%85%B3_Kor+Eng.pdf)
## Our articles

31
docs/CHANGELOG.md
View file

@ -24,6 +24,33 @@ The following `tip` changes can be tested by building VictoriaMetrics components
## tip
**Update note**: starting from this release, [vmagent](https://docs.victoriametrics.com/vmagent.html) ignores timestamps provided by scrape targets by default - it associates scraped metrics with local timestamps instead. Set `honor_timestamps: true` in [scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) if timestamps provided by scrape targets must be used instead. This change helps removing gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) such as `container_memory_usage_bytes`. This also improves data compression and query performance over metrics collected from `cadvisor`. See more details [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697).
* SECURITY: upgrade Go builder from Go1.20.6 to Go1.21.0 in order to fix [this issue](https://github.com/golang/go/issues/61460).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725).
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): allow configuring deadline for a backend to be excluded from the rotation on errors via `-failTimeout` cmd-line flag. This feature could be useful when it is expected for backends to be not available for significant periods of time. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4415) for details. Thanks to @SunKyu for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4416).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove deprecated in [v1.61.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.0) `-rule.configCheckInterval` command-line flag. Use `-configCheckInterval` command-line flag instead.
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove support of deprecated web links of `/api/v1/<groupID>/<alertID>/status` form in favour of `/api/v1/alerts?group_id=<>&alert_id=<>` links. Links of `/api/v1/<groupID>/<alertID>/status` form were deprecated in v1.79.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825) for details.
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow disabling binary export API protocol via `-vm-native-disable-binary-protocol` cmd-line flag when [migrating data from VictoriaMetrics](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need [to be configured](https://docs.victoriametrics.com/#deduplication) at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side.
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738).
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow specifying custom full url at `--remote-read-src-addr` command-line flag if `--remote-read-disable-path-append` command-line flag is set. This allows importing data from urls, which do not end with `/api/v1/read`. For example, from Promscale. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655).
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add warning in query field of vmui for partial data responses. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4721).
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): allow displaying the full error message on click for trimmed error messages in vmui. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4719).
* FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add `Concurrent inserts` panel to vmagent's dasbhoard. The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit.
* FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add panels for absolute Mem and CPU usage by vmalert. See related issue [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4627).
* FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): correctly calculate `Bytes per point` value for single-server and cluster VM dashboards. Before, the calculation mistakenly accounted for the number of entries in indexdb in denominator, which could have shown lower values than expected.
* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml) as it could be related to many VictoriaMetrics components.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly apply `if` filters during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements). Previously the `if` filter could improperly work. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4806) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4816).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly set `vmalert_config_last_reload_successful` value on configuration updates or rollbacks. The bug was introduced in [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0) in [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4543).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix `vmalert_remotewrite_send_duration_seconds_total` value, before it didn't count in the real time spending on remote write requests. See [this pr](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4801) for details.
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): keep unmatched series at [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) when `-remoteWrite.streamAggr.dropInput` is set to `false` to match intended behaviour introduced at [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4804).
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly handle client address with `X-Forwarded-For` part at the [Active queries](https://docs.victoriametrics.com/#active-queries) page. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4676#issuecomment-1663203424).
## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1)
Released at 2023-07-28
@ -46,7 +73,7 @@ The previous behavior can be restored in the following ways:
- by passing `-streamAggr.dropInput` command-line flag to single-node VictoriaMetrics;
- by passing `-remoteWrite.streamAggr.dropInput` command-line flag per each configured `-remoteWrite.streamAggr.config` at `vmagent`.
### CHANGES
---
* SECURITY: upgrade base docker image (alpine) from 3.18.0 to 3.18.2. See [alpine 3.18.2 release notes](https://alpinelinux.org/posts/Alpine-3.15.9-3.16.6-3.17.4-3.18.2-released.html).
* SECURITY: upgrade Go builder from Go1.20.5 to Go1.20.6. See [the list of issues addressed in Go1.20.6](https://github.com/golang/go/issues?q=milestone%3AGo1.20.6+label%3ACherryPickApproved).
@ -2125,7 +2152,7 @@ in front of VictoriaMetrics. [Contact us](mailto:sales@victoriametrics.com) if y
Released at 2021-01-13
* FEATURE: provide a sample list of alerting rules for VictoriaMetrics components. It is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml).
* FEATURE: provide a sample list of alerting rules for VictoriaMetrics components. It is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts).
* FEATURE: disable final merge for data for the previous month at the beginning of new month, since it may result in high disk IO and CPU usage. Final merge can be enabled by setting `-finalMergeDelay` command-line flag to positive duration.
* FEATURE: add `tfirst_over_time(m[d])` and `tlast_over_time(m[d])` functions to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) for returning timestamps for the first and the last data point in `m` over `d` duration.
* FEATURE: add ability to pass multiple labels to `sort_by_label()` and `sort_by_label_desc()` functions. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/992> .

8
docs/CaseStudies.md
View file

@ -30,6 +30,7 @@ where you can chat with VictoriaMetrics users to get additional references, revi
- [Groove X](#groove-x)
- [Idealo.de](#idealode)
- [MHI Vestas Offshore Wind](#mhi-vestas-offshore-wind)
- [Naver](#naver)
- [Percona](#percona)
- [Razorpay](#razorpay)
- [Roblox](#roblox)
@ -420,6 +421,13 @@ Numbers with current, limited roll out:
- Data size on disk: 800 GiB
- Retention period: 3 years
## Naver
[Naver](https://www.navercorp.com/en/) is a global tech platform that enables wide access to advanced technologies for SMEs, creators and partners,
fueling their greater growth around the world.
See [this video](https://www.youtube.com/watch?v=OUyXPgVcdw4) and [these slides](https://deview.kr/data/deview/session/attach/%5B2B4%5DVictoriaMetrics_%E1%84%89%E1%85%B5%E1%84%80%E1%85%A8%E1%84%8B%E1%85%A7%E1%86%AF_%E1%84%83%E1%85%A6%E1%84%8B%E1%85%B5%E1%84%90%E1%85%A5_%E1%84%83%E1%85%A2%E1%84%92%E1%85%A9%E1%86%AB%E1%84%83%E1%85%A9%E1%86%AB%E1%84%8B%E1%85%B4_%E1%84%86%E1%85%A5%E1%86%AF%E1%84%90%E1%85%B5%E1%84%87%E1%85%A5%E1%84%89%E1%85%B3_Kor+Eng.pdf) on why and how Naver uses VictoriaMetrics.
## Percona
[Percona](https://www.percona.com/) is a leader in providing best-of-breed enterprise-class support, consulting, managed services, training and software for MySQL®, MariaDB®, MongoDB®, PostgreSQL® and other open source databases in on-premises and cloud environments.

6
docs/Cluster-VictoriaMetrics.md
View file

@ -294,7 +294,7 @@ or Prometheus to scrape `/metrics` pages from all the cluster components, so the
with [the official Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11831). Graphs on these dashboards contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
It is recommended setting up alerts in [vmalert](https://docs.victoriametrics.com/vmalert.html) or in Prometheus from [this config](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/alerts.yml).
It is recommended setting up alerts in [vmalert](https://docs.victoriametrics.com/vmalert.html) or in Prometheus from [this list](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts).
See more details in the article [VictoriaMetrics Monitoring](https://victoriametrics.com/blog/victoriametrics-monitoring/).
## Cardinality limiter
@ -340,7 +340,7 @@ Check practical examples of VictoriaMetrics API [here](https://docs.victoriametr
- `prometheus/api/v1/import/native` - for importing data obtained via `api/v1/export/native` on `vmselect` (see below).
- `prometheus/api/v1/import/csv` - for importing arbitrary CSV data. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-csv-data) for details.
- `prometheus/api/v1/import/prometheus` - for importing data in [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format) and in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md). This endpoint also supports [Pushgateway protocol](https://github.com/prometheus/pushgateway#url). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) for details.
- `opentemetry/api/v1/push` - for ingesting data via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#sending-data-via-opentelemetry).
- `opentelemetry/api/v1/push` - for ingesting data via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#sending-data-via-opentelemetry).
- `datadog/api/v1/series` - for ingesting data with [DataDog submit metrics API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent) for details.
- `influx/write` and `influx/api/v2/write` - for ingesting data with [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for details.
- `opentsdb/api/put` - for accepting [OpenTSDB HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html). This handler is disabled by default. It is exposed on a distinct TCP address set via `-opentsdbHTTPListenAddr` command-line flag. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#sending-opentsdb-data-via-http-apiput-requests) for details.
@ -844,7 +844,7 @@ Below is the output for `/path/to/vminsert -help`:
-disableRerouting
Whether to disable re-routing when some of vmstorage nodes accept incoming data at slower speed compared to other storage nodes. Disabled re-routing limits the ingestion rate by the slowest vmstorage node. On the other side, disabled re-routing minimizes the number of active time series in the cluster during rolling restarts and during spikes in series churn rate. See also -dropSamplesOnOverload (default true)
-dropSamplesOnOverload
Whether to drop incoming samples if the destination vmstorage node is overloaded and/or unavailable. This prioritizes cluster availability over consistency, e.g. the cluster continues accepting all the ingested samples, but some of them may be dropped if vmstorage nodes are temporarily unavailable and/or overloaded
Whether to drop incoming samples if the destination vmstorage node is overloaded and/or unavailable. This prioritizes cluster availability over consistency, e.g. the cluster continues accepting all the ingested samples, but some of them may be dropped if vmstorage nodes are temporarily unavailable and/or overloaded. The drop of samples happens before the replication, so it's not recommended to use this flag with -replicationFactor enabled.
-enableTCP6
Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
-envflag.enable

8
docs/MetricsQL.md
View file

@ -744,6 +744,14 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
See also [share_gt_over_time](#share_gt_over_time).
#### share_eq_over_time
`share_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which returns share (in the range `[0...1]`) of raw samples
on the given lookbehind window `d`, which are equal to `eq`. It is calculated independently per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
#### stale_samples_over_time
`stale_samples_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number

3
docs/Quick-Start.md
View file

@ -145,8 +145,7 @@ VictoriaMetric team prepared a list of [Grafana dashboards](https://grafana.com/
for the main components. Each dashboard contains a lot of useful information and tips. It is recommended
to have these dashboards installed and up to date.
The list of alerts for [single](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml)
and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/alerts.yml)
Using the [recommended alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
versions would also help to identify and notify about issues with the system.
The rule of thumb is to have a separate installation of VictoriaMetrics or any other monitoring system

5
docs/README.md
View file

@ -120,6 +120,7 @@ Case studies:
* [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x)
* [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode)
* [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind)
* [Naver](https://docs.victoriametrics.com/CaseStudies.html#naver)
* [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay)
* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona)
* [Roblox](https://docs.victoriametrics.com/CaseStudies.html#roblox)
@ -1363,7 +1364,7 @@ VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-sc
## Sending data via OpenTelemetry
VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentemetry/api/v1/push` path.
VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentelemetry/api/v1/push` path.
VictoriaMetrics expects `protobuf`-encoded requests at `/opentelemetry/api/v1/push`.
Set HTTP request header `Content-Encoding: gzip` when sending gzip-compressed data to `/opentelemetry/api/v1/push`.
@ -1784,7 +1785,7 @@ created by community.
Graphs on the dashboards contain useful hints - hover the `i` icon in the top left corner of each graph to read it.
We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml)
We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
via [vmalert](https://docs.victoriametrics.com/vmalert.html) or via Prometheus.
VictoriaMetrics exposes currently running queries and their execution times at `/api/v1/status/active_queries` page.

15
docs/Release-Guide.md
View file

@ -48,18 +48,22 @@ Bumping the limits may significantly improve build speed.
## Release version and Docker images
1. Make sure that the release commits have no security issues.
1. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md) and update version if needed in [SECURITY.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/SECURITY.md)
1. Make sure all the changes are documented in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md).
Ideally, every change must be documented in the commit with the change. Alternatively, the change must be documented immediately
after the commit, which adds the change.
1. Make sure all the changes are synced between `master`, `cluster`, `enterprise-single-node` and `enteprise-cluster` branches.
Changes in these branches must be synced immediately after they are commited in at least a single branch.
1. Make sure that the release branches have no security issues.
1. Update release versions if needed in [SECURITY.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/SECURITY.md).
1. Add `(available starting from v1.xx.y)` line to feature docs introduced in the upcoming release.
1. Cut new version in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md)
and make it merged. See example in this [commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/b771152039d23b5ccd637a23ea748bc44a9511a7).
1. Cherry-pick all changes from `master` to `cluster` branch, and to ENT branches `enterprise-single-node`, `enterprise-cluster`.
and make it merged. See example in this [commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/b771152039d23b5ccd637a23ea748bc44a9511a7).
1. Cherry-pick bug fixes relevant for LTS releases.
1. Make sure you get all changes fetched `git fetch --all`.
1. Create the following release tags:
* `git tag -s v1.xx.y` in `master` branch
* `git tag -s v1.xx.y-cluster` in `cluster` branch
* `git tag -s v1.xx.y-enterprise` in `enterprise` branch
* `git tag -s v1.xx.y-enterprise` in `enterprise-single-node` branch
* `git tag -s v1.xx.y-enterprise-cluster` in `enterprise-cluster` branch
1. Run `TAG=v1.xx.y make publish-release`. This command performs the following tasks:
a) Build and package binaries in `*.tar.gz` release archives with the corresponding `_checksums.txt` files inside `bin` directory.
@ -98,6 +102,7 @@ and make it merged. See example in this [commit](https://github.com/VictoriaMetr
1. Bump version of the VictoriaMetrics cluster in the [sandbox environment](https://github.com/VictoriaMetrics/ops/blob/main/gcp-test/sandbox/manifests/benchmark-vm/vmcluster.yaml)
by [opening and merging PR](https://github.com/VictoriaMetrics/ops/pull/58).
1. Bump VictoriaMetrics version at `deployment/docker/docker-compose.yml` and at `deployment/docker/docker-compose-cluster.yml`.
1. Follow the instructions in [release follow-up](https://github.com/VictoriaMetrics/VictoriaMetrics-enterprise/blob/master/Release-Guide.md).
## Building snap package

5
docs/Single-server-VictoriaMetrics.md
View file

@ -128,6 +128,7 @@ Case studies:
* [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x)
* [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode)
* [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind)
* [Naver](https://docs.victoriametrics.com/CaseStudies.html#naver)
* [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay)
* [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona)
* [Roblox](https://docs.victoriametrics.com/CaseStudies.html#roblox)
@ -1371,7 +1372,7 @@ VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-sc
## Sending data via OpenTelemetry
VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentemetry/api/v1/push` path.
VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentelemetry/api/v1/push` path.
VictoriaMetrics expects `protobuf`-encoded requests at `/opentelemetry/api/v1/push`.
Set HTTP request header `Content-Encoding: gzip` when sending gzip-compressed data to `/opentelemetry/api/v1/push`.
@ -1792,7 +1793,7 @@ created by community.
Graphs on the dashboards contain useful hints - hover the `i` icon in the top left corner of each graph to read it.
We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml)
We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
via [vmalert](https://docs.victoriametrics.com/vmalert.html) or via Prometheus.
VictoriaMetrics exposes currently running queries and their execution times at `/api/v1/status/active_queries` page.

5
docs/Troubleshooting.md
View file

@ -414,9 +414,8 @@ would help identify and prevent most of the issues listed above.
[Grafana dashboards](https://grafana.com/orgs/victoriametrics/dashboards) contain panels reflecting the
health state, resource usage and other specific metrics for VictoriaMetrics components.
Alerting rules for [single-node](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml)
and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/alerts.yml) versions
of VictoriaMetrics will notify about issues with Victoriametrics components and provide recommendations for how to solve them.
The list of [recommended alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
for VictoriaMetrics components will notify about issues and provide recommendations for how to solve them.
Internally, we heavily rely both on dashboards and alerts, and constantly improve them.
It is important to stay up to date with such changes.

4
docs/VictoriaLogs/CHANGELOG.md
View file

@ -5,6 +5,10 @@ according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/QuickSta
## tip
* FEATURE: expose the following metrics at [/metrics](monitoring) page:
* `vl_data_size_bytes{type="storage"}` - on-disk size for data excluding [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes.
* `vl_data_size_bytes{type="indexdb"}` - on-disk size for [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes.
## [v0.3.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.3.0-victorialogs)
Released at 2023-07-20

14
docs/VictoriaLogs/data-ingestion/README.md
View file

@ -255,10 +255,10 @@ VictoriaLogs exposes various [metrics](https://docs.victoriametrics.com/Victoria
Here is the list of log collectors and their ingestion formats supported by VictoriaLogs:
| How to setup the collector | Format: Elasticsearch | Format: JSON Stream |
|------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------|---------------------------------------------------------------|
| [Filebeat](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Filebeat.html) | [Yes](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html) | No |
| [Fluentbit](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Fluentbit.html) | No | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/http) |
| [Logstash](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Logstash.html) | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) | No |
| [Vector](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Vector.html) | [Yes](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) | No |
| [Promtail](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Promtail.html) | No | No |
| How to setup the collector | Format: Elasticsearch | Format: JSON Stream | Format: Loki |
|------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------|---------------------------------------------------------------|-------------------------------------------------------------------------------------|
| [Filebeat](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Filebeat.html) | [Yes](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html) | No | No |
| [Fluentbit](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Fluentbit.html) | No | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/http) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/loki) |
| [Logstash](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Logstash.html) | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) | No | No |
| [Vector](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Vector.html) | [Yes](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) | No | [Yes](https://vector.dev/docs/reference/configuration/sinks/loki/) |
| [Promtail](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Promtail.html) | No | No | [Yes](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#clients) |

6
docs/guides/multi-regional-setup-dedicated-regions.md
View file

@ -75,10 +75,8 @@ You can set up vmalert in each Ground control region that evaluates recording an
For alert deduplication, please use [cluster mode in Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/#high-availability).
We also recommend adopting these alerts:
* VictoriaMetrics Single - [https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml)
* VictoriaMetrics Cluster - [https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/alerts.yml)
We also recommend adopting the list of [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
for VictoriaMetrics components.
### Monitoring

1450
docs/operator/CHANGELOG.md Normal file
View file

File diff suppressed because it is too large Load diff

103
docs/operator/api.md
View file

@ -63,6 +63,7 @@ This Document documents the types introduced by the VictoriaMetrics to be consum
* [EmbeddedPodDisruptionBudgetSpec](#embeddedpoddisruptionbudgetspec)
* [EmbeddedProbes](#embeddedprobes)
* [HTTPAuth](#httpauth)
* [KeyValue](#keyvalue)
* [ServiceSpec](#servicespec)
* [StorageSpec](#storagespec)
* [StreamAggrConfig](#streamaggrconfig)
@ -122,12 +123,14 @@ This Document documents the types introduced by the VictoriaMetrics to be consum
* [StaticRef](#staticref)
* [TargetRef](#targetref)
* [VMUser](#vmuser)
* [VMUserIPFilters](#vmuseripfilters)
* [VMUserList](#vmuserlist)
* [VMUserSpec](#vmuserspec)
* [EmbeddedIngress](#embeddedingress)
* [VMAuth](#vmauth)
* [VMAuthList](#vmauthlist)
* [VMAuthSpec](#vmauthspec)
* [VMAuthUnauthorizedPath](#vmauthunauthorizedpath)
* [TargetEndpoint](#targetendpoint)
* [VMStaticScrape](#vmstaticscrape)
* [VMStaticScrapeList](#vmstaticscrapelist)
@ -256,7 +259,7 @@ EmailConfig configures notifications via Email.
| auth_password | AuthPassword defines secret name and key at CRD namespace. | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false |
| auth_secret | AuthSecret defines secrent name and key at CRD namespace. It must contain the CRAM-MD5 secret. | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false |
| auth_identity | The identity to use for authentication. | string | false |
| headers | Further headers email header key/value pairs. Overrides any headers previously set by the notification implementation. | map[string]string | false |
| headers | Further headers email header key/value pairs. Overrides any headers previously set by the notification implementation. | EmailConfigHeaders | false |
| html | The HTML body of the email notification. | string | false |
| text | The text body of the email notification. | string | false |
| require_tls | The SMTP TLS requirement. Note that Go does not support unencrypted connections to remote SMTP endpoints. | *bool | false |
@ -504,7 +507,7 @@ SlackConfirmationField protect users from destructive actions or particularly di
## SlackField
See https://api.slack.com/docs/message-attachments#fields for more information.
SlackField configures a single Slack field that is sent with each notification. See https://api.slack.com/docs/message-attachments#fields for more information.
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
@ -807,7 +810,7 @@ VMAgentSpec defines the desired state of VMAgent
## VMAgentStatus
VmAgentStatus defines the observed state of VmAgent
VMAgentStatus defines the observed state of VMAgent
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
@ -838,7 +841,7 @@ BearerAuth defines auth with bearer token
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
| bearerTokenFilePath | | string | false |
| bearerTokenFile | Path to bearer token file | string | false |
| bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false |
[Back to TOC](#table-of-contents)
@ -932,14 +935,25 @@ HTTPAuth generic auth used with http protocols
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
| basicAuth | | *[BasicAuth](#basicauth) | false |
| OAuth2 | | *[OAuth2](#oauth2) | false |
| oauth2 | | *[OAuth2](#oauth2) | false |
| tlsConfig | | *[TLSConfig](#tlsconfig) | false |
| bearerTokenFilePath | | string | false |
| bearerTokenFile | Path to bearer token file | string | false |
| bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false |
| headers | Headers allow configuring custom http headers Must be in form of semicolon separated header with value e.g. headerName:headerValue vmalert supports it since 1.79.0 version | []string | false |
[Back to TOC](#table-of-contents)
## KeyValue
KeyValue defines a (key, value) tuple.
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
| key | Key of the tuple. | string | true |
| value | Value of the tuple. | string | true |
[Back to TOC](#table-of-contents)
## ServiceSpec
ServiceSpec defines additional service for CRD with user-defined params. by default, some of fields can be inherited from default service definition for the CRD: labels,selector, ports. if metadata.name is not defined, service will have format {{CRD_TYPE}}-{{CRD_NAME}}-additional-service.
@ -971,6 +985,7 @@ StreamAggrConfig defines the stream aggregation config
| ----- | ----------- | ------ | -------- |
| rules | Stream aggregation rules | [][StreamAggrRule](#streamaggrrule) | true |
| keepInput | Allows writing both raw and aggregate data | bool | false |
| dropInput | Allow drop all the input samples after the aggregation | bool | false |
| dedupInterval | Allows setting different de-duplication intervals per each configured remote storage | string | false |
[Back to TOC](#table-of-contents)
@ -981,8 +996,9 @@ StreamAggrRule defines the rule in stream aggregation config
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
| match | Match is a label selector for filtering time series for the given selector.\n\nIf the match isn&#39;t set, then all the input time series are processed. | string | false |
| match | Match is a label selector for filtering time series for the given selector.\n\nIf the match isn&#39;t set, then all the input time series are processed. | Match | false |
| interval | Interval is the interval between aggregations. | string | true |
| staleness_interval | StalenessInterval defines an interval after which the series state will be reset if no samples have been sent during it. | string | false |
| outputs | Outputs is a list of output aggregate functions to produce.\n\nThe following names are allowed:\n\n- total - aggregates input counters - increase - counts the increase over input counters - count_series - counts the input series - count_samples - counts the input samples - sum_samples - sums the input samples - last - the last biggest sample value - min - the minimum sample value - max - the maximum sample value - avg - the average value across all the samples - stddev - standard deviation across all the samples - stdvar - standard variance across all the samples - histogram_bucket - creates VictoriaMetrics histogram for input samples - quantiles(phi1, ..., phiN) - quantiles&#39; estimation for phi in the range [0..1]\n\nThe output time series will have the following names:\n\n input_name:aggr_&lt;interval&gt;_&lt;output&gt; | []string | true |
| by | By is an optional list of labels for grouping input series.\n\nSee also Without.\n\nIf neither By nor Without are set, then the Outputs are calculated individually per each input time series. | []string | false |
| without | Without is an optional list of labels, which must be excluded when grouping input series.\n\nSee also By.\n\nIf neither By nor Without are set, then the Outputs are calculated individually per each input time series. | []string | false |
@ -1005,15 +1021,15 @@ VMAlert executes a list of given alerting or recording rules against configured
## VMAlertDatasourceSpec
VMAgentRemoteReadSpec defines the remote storage configuration for VmAlert to read alerts from
VMAlertDatasourceSpec defines the remote storage configuration for VmAlert to read alerts from
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
| url | Victoria Metrics or VMSelect url. Required parameter. E.g. http://127.0.0.1:8428 | string | true |
| basicAuth | | *[BasicAuth](#basicauth) | false |
| OAuth2 | | *[OAuth2](#oauth2) | false |
| oauth2 | | *[OAuth2](#oauth2) | false |
| tlsConfig | | *[TLSConfig](#tlsconfig) | false |
| bearerTokenFilePath | | string | false |
| bearerTokenFile | Path to bearer token file | string | false |
| bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false |
| headers | Headers allow configuring custom http headers Must be in form of semicolon separated header with value e.g. headerName:headerValue vmalert supports it since 1.79.0 version | []string | false |
@ -1039,9 +1055,9 @@ VMAlertNotifierSpec defines the notifier url for sending information about alert
| url | AlertManager url. E.g. http://127.0.0.1:9093 | string | false |
| selector | Selector allows service discovery for alertmanager in this case all matched vmalertmanager replicas will be added into vmalert notifier.url as statefulset pod.fqdn | *[DiscoverySelector](#discoveryselector) | false |
| basicAuth | | *[BasicAuth](#basicauth) | false |
| OAuth2 | | *[OAuth2](#oauth2) | false |
| oauth2 | | *[OAuth2](#oauth2) | false |
| tlsConfig | | *[TLSConfig](#tlsconfig) | false |
| bearerTokenFilePath | | string | false |
| bearerTokenFile | Path to bearer token file | string | false |
| bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false |
| headers | Headers allow configuring custom http headers Must be in form of semicolon separated header with value e.g. headerName:headerValue vmalert supports it since 1.79.0 version | []string | false |
@ -1049,16 +1065,16 @@ VMAlertNotifierSpec defines the notifier url for sending information about alert
## VMAlertRemoteReadSpec
VMAgentRemoteReadSpec defines the remote storage configuration for VmAlert to read alerts from
VMAlertRemoteReadSpec defines the remote storage configuration for VmAlert to read alerts from
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
| url | URL of the endpoint to send samples to. | string | true |
| lookback | Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s) Applied only to RemoteReadSpec | *string | false |
| basicAuth | | *[BasicAuth](#basicauth) | false |
| OAuth2 | | *[OAuth2](#oauth2) | false |
| oauth2 | | *[OAuth2](#oauth2) | false |
| tlsConfig | | *[TLSConfig](#tlsconfig) | false |
| bearerTokenFilePath | | string | false |
| bearerTokenFile | Path to bearer token file | string | false |
| bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false |
| headers | Headers allow configuring custom http headers Must be in form of semicolon separated header with value e.g. headerName:headerValue vmalert supports it since 1.79.0 version | []string | false |
@ -1066,7 +1082,7 @@ VMAgentRemoteReadSpec defines the remote storage configuration for VmAlert to re
## VMAlertRemoteWriteSpec
VMAgentRemoteWriteSpec defines the remote storage configuration for VmAlert
VMAlertRemoteWriteSpec defines the remote storage configuration for VmAlert
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
@ -1076,9 +1092,9 @@ VMAgentRemoteWriteSpec defines the remote storage configuration for VmAlert
| maxBatchSize | Defines defines max number of timeseries to be flushed at once (default 1000) | *int32 | false |
| maxQueueSize | Defines the max number of pending datapoints to remote write endpoint (default 100000) | *int32 | false |
| basicAuth | | *[BasicAuth](#basicauth) | false |
| OAuth2 | | *[OAuth2](#oauth2) | false |
| oauth2 | | *[OAuth2](#oauth2) | false |
| tlsConfig | | *[TLSConfig](#tlsconfig) | false |
| bearerTokenFilePath | | string | false |
| bearerTokenFile | Path to bearer token file | string | false |
| bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false |
| headers | Headers allow configuring custom http headers Must be in form of semicolon separated header with value e.g. headerName:headerValue vmalert supports it since 1.79.0 version | []string | false |
@ -1114,7 +1130,7 @@ VMAlertSpec defines the desired state of VMAlert
| hostNetwork | HostNetwork controls whether the pod may use the node network namespace | bool | false |
| dnsPolicy | DNSPolicy sets DNS policy for the pod | [v1.DNSPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#pod-v1-core) | false |
| topologySpreadConstraints | TopologySpreadConstraints embedded kubernetes pod configuration option, controls how pods are spread across your cluster among failure-domains such as regions, zones, nodes, and other user-defined topology domains https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ | [][v1.TopologySpreadConstraint](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) | false |
| evaluationInterval | EvaluationInterval how often evalute rules by default | string | false |
| evaluationInterval | EvaluationInterval defines how often to evaluate rules by default | string | false |
| enforcedNamespaceLabel | EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created. | string | false |
| selectAllByDefault | SelectAllByDefault changes default behavior for empty CRD selectors, such RuleSelector. with selectAllByDefault: true and empty serviceScrapeSelector and RuleNamespaceSelector Operator selects all exist serviceScrapes with selectAllByDefault: false - selects nothing | bool | false |
| ruleSelector | RuleSelector selector to select which VMRules to mount for loading alerting rules from. Works in combination with NamespaceSelector. If both nil - behaviour controlled by selectAllByDefault NamespaceSelector nil - only objects at VMAlert namespace. | *[metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#labelselector-v1-meta) | false |
@ -1147,7 +1163,7 @@ VMAlertSpec defines the desired state of VMAlert
## VMAlertStatus
VmAlertStatus defines the observed state of VmAlert
VMAlertStatus defines the observed state of VMAlert
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
@ -1241,10 +1257,12 @@ VMSingleStatus defines the observed state of VMSingle
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
| replicas | ReplicaCount Total number of non-terminated pods targeted by this VMAlert cluster (their labels match the selector). | int32 | true |
| updatedReplicas | UpdatedReplicas Total number of non-terminated pods targeted by this VMAlert cluster that have the desired version spec. | int32 | true |
| availableReplicas | AvailableReplicas Total number of available pods (ready for at least minReadySeconds) targeted by this VMAlert cluster. | int32 | true |
| unavailableReplicas | UnavailableReplicas Total number of unavailable pods targeted by this VMAlert cluster. | int32 | true |
| replicas | ReplicaCount Total number of non-terminated pods targeted by this VMSingle. | int32 | true |
| updatedReplicas | UpdatedReplicas Total number of non-terminated pods targeted by this VMSingle. | int32 | true |
| availableReplicas | AvailableReplicas Total number of available pods (ready for at least minReadySeconds) targeted by this VMSingle. | int32 | true |
| unavailableReplicas | UnavailableReplicas Total number of unavailable pods targeted by this VMSingle. | int32 | true |
| singleStatus | | SingleStatus | true |
| reason | | string | false |
[Back to TOC](#table-of-contents)
@ -1259,8 +1277,10 @@ Rule describes an alerting or recording rule.
| expr | Expr is query, that will be evaluated at dataSource | string | true |
| debug | Debug enables logging for rule it useful for tracking | *bool | false |
| for | For evaluation interval in time.Duration format 30s, 1m, 1h or nanoseconds | string | false |
| keep_firing_for | KeepFiringFor will make alert continue firing for this long even when the alerting expression no longer has results. Use time.Duration format, 30s, 1m, 1h or nanoseconds | string | false |
| labels | Labels will be added to rule configuration | map[string]string | false |
| annotations | Annotations will be added to rule configuration | map[string]string | false |
| update_entries_limit | UpdateEntriesLimit defines max number of rule&#39;s state updates stored in memory. Overrides `-rule.updateEntriesLimit` in vmalert. | *int | false |
[Back to TOC](#table-of-contents)
@ -1281,6 +1301,7 @@ RuleGroup is a list of sequentially evaluated recording and alerting rules.
| params | Params optional HTTP URL parameters added to each rule request | url.Values | false |
| type | Type defines datasource type for enterprise version of vmalert possible values - prometheus,graphite | string | false |
| headers | Headers contains optional HTTP headers added to each rule request Must be in form `header-name: value` For example:\n headers:\n - \&#34;CustomHeader: foo\&#34;\n - \&#34;CustomHeader2: bar\&#34; | []string | false |
| notifier_headers | NotifierHeaders contains optional HTTP headers added to each alert request which will send to notifier Must be in form `header-name: value` For example:\n headers:\n - \&#34;CustomHeader: foo\&#34;\n - \&#34;CustomHeader2: bar\&#34; | []string | false |
[Back to TOC](#table-of-contents)
@ -1572,6 +1593,7 @@ PodMetricsEndpoint defines a scrapeable endpoint of a Kubernetes Pod serving Pro
| authorization | Authorization with http header Authorization | *[Authorization](#authorization) | false |
| vm_scrape_params | VMScrapeParams defines VictoriaMetrics specific scrape parametrs | *[VMScrapeParams](#vmscrapeparams) | false |
| attach_metadata | AttachMetadata configures metadata attaching from service discovery | [AttachMetadata](#attachmetadata) | false |
| filterRunning | FilterRunning applies filter with pod status == running it prevents from scrapping metrics at failed or succeed state pods. enabled by default | *bool | false |
[Back to TOC](#table-of-contents)
@ -1969,7 +1991,8 @@ StaticRef - user-defined routing host address.
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
| url | URL http url for given staticRef. | string | true |
| url | URL http url for given staticRef. | string | false |
| urls | URLs allows setting multiple urls for load-balancing at vmauth-side. | []string | false |
[Back to TOC](#table-of-contents)
@ -1984,6 +2007,7 @@ TargetRef describes target for user traffic forwarding. one of target types can
| paths | Paths - matched path to route. | []string | false |
| target_path_suffix | QueryParams []string `json:\&#34;queryParams,omitempty\&#34;` TargetPathSuffix allows to add some suffix to the target path It allows to hide tenant configuration from user with crd as ref. it also may contain any url encoded params. | string | false |
| headers | Headers represent additional http headers, that vmauth uses in form of [\&#34;header_key: header_value\&#34;] multiple values for header key: [\&#34;header_key: value1,value2\&#34;] it&#39;s available since 1.68.0 version of vmauth | []string | false |
| ip_filters | IPFilters defines per target src ip filters supported only with enterprise version of vmauth https://docs.victoriametrics.com/vmauth.html#ip-filters | [VMUserIPFilters](#vmuseripfilters) | false |
[Back to TOC](#table-of-contents)
@ -1999,6 +2023,17 @@ VMUser is the Schema for the vmusers API
[Back to TOC](#table-of-contents)
## VMUserIPFilters
VMUserIPFilters defines filters for IP addresses supported only with enterprise version of vmauth https://docs.victoriametrics.com/vmauth.html#ip-filters
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
| deny_list | | []string | false |
| allow_list | | []string | false |
[Back to TOC](#table-of-contents)
## VMUserList
VMUserList contains a list of VMUser
@ -2024,6 +2059,7 @@ VMUserSpec defines the desired state of VMUser
| generatePassword | GeneratePassword instructs operator to generate password for user if spec.password if empty. | bool | false |
| bearerToken | BearerToken Authorization header value for accessing protected endpoint. | *string | false |
| targetRefs | TargetRefs - reference to endpoints, which user may access. | [][TargetRef](#targetref) | true |
| default_url | DefaultURLs backend url for non-matching paths filter usually used for default backend with error message | []string | false |
[Back to TOC](#table-of-contents)
@ -2106,7 +2142,7 @@ VMAuthSpec defines the desired state of VMAuth
| userNamespaceSelector | UserNamespaceSelector Namespaces to be selected for VMAuth discovery. Works in combination with Selector. NamespaceSelector nil - only objects at VMAuth namespace. Selector nil - only objects at NamespaceSelector namespaces. If both nil - behaviour controlled by selectAllByDefault | *[metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#labelselector-v1-meta) | false |
| extraArgs | ExtraArgs that will be passed to VMAuth pod for example remoteWrite.tmpDataPath: /tmp | map[string]string | false |
| extraEnvs | ExtraEnvs that will be added to VMAuth pod | [][v1.EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#envvar-v1-core) | false |
| serviceSpec | ServiceSpec that will be added to vmauth service spec | *[ServiceSpec](#servicespec) | false |
| serviceSpec | ServiceSpec that will be added to vmsingle service spec | *[ServiceSpec](#servicespec) | false |
| serviceScrapeSpec | ServiceScrapeSpec that will be added to vmauth VMServiceScrape spec | *[VMServiceScrapeSpec](#vmservicescrapespec) | false |
| podDisruptionBudget | PodDisruptionBudget created by operator | *[EmbeddedPodDisruptionBudgetSpec](#embeddedpoddisruptionbudgetspec) | false |
| ingress | Ingress enables ingress configuration for VMAuth. | *[EmbeddedIngress](#embeddedingress) | false |
@ -2116,6 +2152,19 @@ VMAuthSpec defines the desired state of VMAuth
| nodeSelector | NodeSelector Define which Nodes the Pods are scheduled on. | map[string]string | false |
| terminationGracePeriodSeconds | TerminationGracePeriodSeconds period for container graceful termination | *int64 | false |
| readinessGates | ReadinessGates defines pod readiness gates | []v1.PodReadinessGate | false |
| unauthorizedAccessConfig | UnauthorizedAccessConfig configures access for un authorized users | [][VMAuthUnauthorizedPath](#vmauthunauthorizedpath) | false |
[Back to TOC](#table-of-contents)
## VMAuthUnauthorizedPath
VMAuthUnauthorizedPath defines url_map for unauthorized access
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |
| src_paths | Paths src request paths | []string | false |
| url_prefix | URLs defines url_prefix for dst routing | []string | false |
| ip_filters | IPFilters defines filter for src ip address enterprise only | [VMUserIPFilters](#vmuseripfilters) | false |
[Back to TOC](#table-of-contents)
@ -2200,7 +2249,7 @@ ProbeTargetIngress defines the set of Ingress objects considered for probing.
## VMProbe
\n VMProbe defines a probe for targets, that will be executed with prober,\n like blackbox exporter.\nIt helps to monitor reachability of target with various checks.
VMProbe defines a probe for targets, that will be executed with prober, like blackbox exporter. It helps to monitor reachability of target with various checks.
| Field | Description | Scheme | Required |
| ----- | ----------- | ------ | -------- |

24
docs/operator/vars.md
View file

@ -9,9 +9,8 @@ menu:
aliases:
- /operator/vars.html
---
# Auto Generated vars for package config
updated at Mon May 8 06:43:29 UTC 2023
# Auto Generated vars for package config
updated at Thu Aug 10 14:33:38 UTC 2023
| varible name | variable default value | variable required | variable description |
| --- | --- | --- | --- |
@ -20,7 +19,7 @@ updated at Mon May 8 06:43:29 UTC 2023
| VM_CUSTOMCONFIGRELOADERIMAGE | victoriametrics/operator:config-reloader-v0.32.0 | false | - |
| VM_PSPAUTOCREATEENABLED | true | false | - |
| VM_VMALERTDEFAULT_IMAGE | victoriametrics/vmalert | false | - |
| VM_VMALERTDEFAULT_VERSION | v1.89.1 | false | - |
| VM_VMALERTDEFAULT_VERSION | v1.91.3 | false | - |
| VM_VMALERTDEFAULT_PORT | 8080 | false | - |
| VM_VMALERTDEFAULT_USEDEFAULTRESOURCES | true | false | - |
| VM_VMALERTDEFAULT_RESOURCE_LIMIT_MEM | 500Mi | false | - |
@ -31,7 +30,7 @@ updated at Mon May 8 06:43:29 UTC 2023
| VM_VMALERTDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - |
| VM_VMALERTDEFAULT_CONFIGRELOADIMAGE | jimmidyson/configmap-reload:v0.3.0 | false | - |
| VM_VMAGENTDEFAULT_IMAGE | victoriametrics/vmagent | false | - |
| VM_VMAGENTDEFAULT_VERSION | v1.89.1 | false | - |
| VM_VMAGENTDEFAULT_VERSION | v1.91.3 | false | - |
| VM_VMAGENTDEFAULT_CONFIGRELOADIMAGE | quay.io/prometheus-operator/prometheus-config-reloader:v0.58.0 | false | - |
| VM_VMAGENTDEFAULT_PORT | 8429 | false | - |
| VM_VMAGENTDEFAULT_USEDEFAULTRESOURCES | true | false | - |
@ -42,7 +41,7 @@ updated at Mon May 8 06:43:29 UTC 2023
| VM_VMAGENTDEFAULT_CONFIGRELOADERCPU | 100m | false | - |
| VM_VMAGENTDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - |
| VM_VMSINGLEDEFAULT_IMAGE | victoriametrics/victoria-metrics | false | - |
| VM_VMSINGLEDEFAULT_VERSION | v1.89.1 | false | - |
| VM_VMSINGLEDEFAULT_VERSION | v1.91.3 | false | - |
| VM_VMSINGLEDEFAULT_PORT | 8429 | false | - |
| VM_VMSINGLEDEFAULT_USEDEFAULTRESOURCES | true | false | - |
| VM_VMSINGLEDEFAULT_RESOURCE_LIMIT_MEM | 1500Mi | false | - |
@ -53,14 +52,14 @@ updated at Mon May 8 06:43:29 UTC 2023
| VM_VMSINGLEDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - |
| VM_VMCLUSTERDEFAULT_USEDEFAULTRESOURCES | true | false | - |
| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_IMAGE | victoriametrics/vmselect | false | - |
| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_VERSION | v1.89.1-cluster | false | - |
| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_VERSION | v1.91.3-cluster | false | - |
| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_PORT | 8481 | false | - |
| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_LIMIT_MEM | 1000Mi | false | - |
| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_LIMIT_CPU | 500m | false | - |
| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_REQUEST_MEM | 500Mi | false | - |
| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_REQUEST_CPU | 100m | false | - |
| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_IMAGE | victoriametrics/vmstorage | false | - |
| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VERSION | v1.89.1-cluster | false | - |
| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VERSION | v1.91.3-cluster | false | - |
| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VMINSERTPORT | 8400 | false | - |
| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VMSELECTPORT | 8401 | false | - |
| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_PORT | 8482 | false | - |
@ -69,7 +68,7 @@ updated at Mon May 8 06:43:29 UTC 2023
| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_RESOURCE_REQUEST_MEM | 500Mi | false | - |
| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_RESOURCE_REQUEST_CPU | 250m | false | - |
| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_IMAGE | victoriametrics/vminsert | false | - |
| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_VERSION | v1.89.1-cluster | false | - |
| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_VERSION | v1.91.3-cluster | false | - |
| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_PORT | 8480 | false | - |
| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_RESOURCE_LIMIT_MEM | 500Mi | false | - |
| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_RESOURCE_LIMIT_CPU | 500m | false | - |
@ -88,7 +87,7 @@ updated at Mon May 8 06:43:29 UTC 2023
| VM_VMALERTMANAGER_RESOURCE_REQUEST_CPU | 30m | false | - |
| VM_DISABLESELFSERVICESCRAPECREATION | false | false | - |
| VM_VMBACKUP_IMAGE | victoriametrics/vmbackupmanager | false | - |
| VM_VMBACKUP_VERSION | v1.89.1-enterprise | false | - |
| VM_VMBACKUP_VERSION | v1.91.3-enterprise | false | - |
| VM_VMBACKUP_PORT | 8300 | false | - |
| VM_VMBACKUP_USEDEFAULTRESOURCES | true | false | - |
| VM_VMBACKUP_RESOURCE_LIMIT_MEM | 500Mi | false | - |
@ -97,7 +96,7 @@ updated at Mon May 8 06:43:29 UTC 2023
| VM_VMBACKUP_RESOURCE_REQUEST_CPU | 150m | false | - |
| VM_VMBACKUP_LOGLEVEL | INFO | false | - |
| VM_VMAUTHDEFAULT_IMAGE | victoriametrics/vmauth | false | - |
| VM_VMAUTHDEFAULT_VERSION | v1.89.1 | false | - |
| VM_VMAUTHDEFAULT_VERSION | v1.91.3 | false | - |
| VM_VMAUTHDEFAULT_CONFIGRELOADIMAGE | quay.io/prometheus-operator/prometheus-config-reloader:v0.48.1 | false | - |
| VM_VMAUTHDEFAULT_PORT | 8427 | false | - |
| VM_VMAUTHDEFAULT_USEDEFAULTRESOURCES | true | false | - |
@ -126,4 +125,5 @@ updated at Mon May 8 06:43:29 UTC 2023
| VM_PODWAITREADYTIMEOUT | 80s | false | - |
| VM_PODWAITREADYINTERVALCHECK | 5s | false | - |
| VM_PODWAITREADYINITDELAY | 10s | false | - |
| VM_FORCERESYNCINTERVAL | 60s | false | configures force resync interval for VMAgent, VMAlert and VMAlertmanager |
| VM_FORCERESYNCINTERVAL | 60s | false | configures force resync interval for VMAgent, VMAlert, VMAlertmanager and VMAuth. |
| VM_ENABLESTRICTSECURITY | true | false | EnableStrictSecurity will add default `securityContext` to pods and containers created by operatorDefault PodSecurityContext include:1. RunAsNonRoot: true2. RunAsUser/RunAsGroup/FSGroup: 65534'65534' refers to 'nobody' in all the used default images like alpine, busybox.If you're using customize image, please make sure '65534' is a valid uid in there or specify SecurityContext.3. FSGroupChangePolicy: &onRootMismatchIf KubeVersion>=1.20, use `FSGroupChangePolicy="onRootMismatch"` to skip the recursive permission changewhen the root of the volume already has the correct permissions4. SeccompProfile:type: RuntimeDefaultUse `RuntimeDefault` seccomp profile by default, which is defined by the container runtime,instead of using the Unconfined (seccomp disabled) mode.Default container SecurityContext include:1. AllowPrivilegeEscalation: false2. ReadOnlyRootFilesystem: true3. Capabilities:drop:- all |

3
docs/sd_configs.md
View file

@ -1445,7 +1445,8 @@ scrape_configs:
# If honor_timestamps is set to "false", the timestamps of the metrics exposed
# by the target will be ignored.
#
# By default, honor_timestamps is set to true.
# By default, honor_timestamps is set to false.
# See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1656540535 for details.
# honor_timestamps: <boolean>
# scheme configures the protocol scheme used for requests.

2
docs/stream-aggregation.md
View file

@ -83,7 +83,7 @@ Sometimes [alerting queries](https://docs.victoriametrics.com/vmalert.html#alert
disk IO and network bandwidth at metrics storage side. For example, if `http_request_duration_seconds` histogram is generated by thousands
of application instances, then the alerting query `histogram_quantile(0.99, sum(increase(http_request_duration_seconds_bucket[5m])) without (instance)) > 0.5`
can become slow, since it needs to scan too big number of unique [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series)
with `http_request_duration_seconds_bucket` name. This alerting query can be sped up by pre-calculating
with `http_request_duration_seconds_bucket` name. This alerting query can be speed up by pre-calculating
the `sum(increase(http_request_duration_seconds_bucket[5m])) without (instance)` via [recording rule](https://docs.victoriametrics.com/vmalert.html#recording-rules).
But this recording rule may take too much time to execute too. In this case the slow recording rule can be substituted
with the following [stream aggregation config](#stream-aggregation-config):

2
docs/vmagent.md
View file

@ -1582,7 +1582,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
Supports array of values separated by comma or specified via multiple flags.
-remoteWrite.sendTimeout array
Timeout for sending a single block of data to the corresponding -remoteWrite.url
Timeout for sending a single block of data to the corresponding -remoteWrite.url (default 1m)
Supports array of values separated by comma or specified via multiple flags.
-remoteWrite.shardByURL
Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages

14
docs/vmalert.md
View file

@ -128,11 +128,11 @@ name: <string>
[ limit: <int> | default = 0 ]
# How many rules execute at once within a group. Increasing concurrency may speed
# up round execution speed.
# up group's evaluation duration (exposed via `vmalert_iteration_duration_seconds` metric).
[ concurrency: <integer> | default = 1 ]
# Optional type for expressions inside the rules. Supported values: "graphite" and "prometheus".
# By default "prometheus" type is used.
# By default, "prometheus" type is used.
[ type: <string> ]
# Optional list of HTTP URL parameters
@ -372,9 +372,9 @@ For recording rules to work `-remoteWrite.url` must be specified.
### Alerts state on restarts
`vmalert` is stateless, it holds alerts state in the process memory. Restarting of `vmalert` process
will reset alerts state in memory. To prevent `vmalert` from losing alerts state it should be configured
to persist the state to the remote destination via the following flags:
`vmalert` holds alerts state in the memory. Restart of the `vmalert` process will reset the state of all active alerts
in the memory. To prevent `vmalert` from losing the state on restarts configure it to persist the state
to the remote database via the following flags:
* `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state
to the configured address in the form of [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series)
@ -530,7 +530,7 @@ Alertmanagers.
To avoid recording rules results and alerts state duplication in VictoriaMetrics server
don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
The recommended value for `-dedup.minScrapeInterval` must be greater or equal to vmalert `evaluation_interval`.
The recommended value for `-dedup.minScrapeInterval` must be multiple of vmalert's `evaluation_interval`.
If you observe inconsistent or "jumping" values in series produced by vmalert, try disabling `-datasource.queryTimeAlignment`
command line flag. Because of alignment, two or more vmalert HA pairs will produce results with the same timestamps.
But due of backfilling (data delivered to the datasource with some delay) values of such results may differ,
@ -1254,8 +1254,6 @@ The shortlist of configuration flags is the following:
See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage
Supports an array of values separated by comma or specified via multiple flags.
-rule.configCheckInterval duration
Interval for checking for changes in '-rule' files. By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead
-rule.maxResolveDuration duration
Limits the maximum duration for automatic alert expiration, which by default is 4 times evaluationInterval of the parent group.
-rule.resendDelay duration

2
docs/vmauth.md
View file

@ -362,6 +362,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
Prefix for environment variables if -envflag.enable is set
-eula
By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
-failTimeout duration
Sets a delay period for load balancing to skip a malfunctioning backend. (defaults 3s)
-flagsAuthKey string
Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
-fs.disableMmap

2
go.mod
View file

@ -12,7 +12,7 @@ require (
// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
github.com/VictoriaMetrics/fasthttp v1.2.0
github.com/VictoriaMetrics/metrics v1.24.0
github.com/VictoriaMetrics/metricsql v0.61.1
github.com/VictoriaMetrics/metricsql v0.62.0
github.com/aws/aws-sdk-go-v2 v1.19.0
github.com/aws/aws-sdk-go-v2/config v1.18.29
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.73

4
go.sum
View file

@ -70,8 +70,8 @@ github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2b
github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE=
github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw=
github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys=
github.com/VictoriaMetrics/metricsql v0.61.1 h1:vYkVDVa+ROvrDkhlrCzBLKB273tE6N681ftfZP+Lav8=
github.com/VictoriaMetrics/metricsql v0.61.1/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I=
github.com/VictoriaMetrics/metricsql v0.62.0 h1:g8Iv8HPuAtgZAJhIIHTHct3eby/ZTpW1zMlJBE3O01c=
github.com/VictoriaMetrics/metricsql v0.62.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=

10
lib/backup/fscommon/fscommon.go
View file

@ -52,7 +52,7 @@ func appendFilesInternal(dst []string, d *os.File) ([]string, error) {
// Process directory
dst, err = AppendFiles(dst, path)
if err != nil {
return nil, fmt.Errorf("cannot list %q: %w", path, err)
return nil, fmt.Errorf("cannot append files %q: %w", path, err)
}
continue
}
@ -124,9 +124,6 @@ func removeEmptyDirs(dir string) (bool, error) {
return false, err
}
ok, err := removeEmptyDirsInternal(d)
if err1 := d.Close(); err1 != nil {
err = err1
}
if err != nil {
return false, err
}
@ -157,7 +154,7 @@ func removeEmptyDirsInternal(d *os.File) (bool, error) {
// Process directory
ok, err := removeEmptyDirs(path)
if err != nil {
return false, fmt.Errorf("cannot list %q: %w", path, err)
return false, fmt.Errorf("cannot remove empty dirs %q: %w", path, err)
}
if !ok {
dirEntries++
@ -221,6 +218,9 @@ func removeEmptyDirsInternal(d *os.File) (bool, error) {
if dirEntries > 0 {
return false, nil
}
if err := d.Close(); err != nil {
return false, fmt.Errorf("cannot close %q: %w", dir, err)
}
// Use os.RemoveAll() instead of os.Remove(), since the dir may contain special files such as flock.lock and backupnames.RestoreInProgressFilename,
// which must be ignored.
if err := os.RemoveAll(dir); err != nil {

5
lib/backup/fsremote/fsremote.go
View file

@ -68,6 +68,7 @@ func (fs *FS) ListParts() ([]common.Part, error) {
return nil, fmt.Errorf("cannot stat file %q for part %q: %w", file, p.Path, err)
}
p.ActualSize = uint64(fi.Size())
p.Path = pathToCanonical(p.Path)
parts = append(parts, p)
}
return parts, nil
@ -75,6 +76,7 @@ func (fs *FS) ListParts() ([]common.Part, error) {
// DeletePart deletes the given part p from fs.
func (fs *FS) DeletePart(p common.Part) error {
p.Path = canonicalPathToLocal(p.Path)
path := fs.path(p)
if err := os.Remove(path); err != nil {
return fmt.Errorf("cannot remove %q: %w", path, err)
@ -95,6 +97,7 @@ func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
if !ok {
return fmt.Errorf("cannot perform server-side copying from %s to %s: both of them must be fsremote", srcFS, fs)
}
p.Path = canonicalPathToLocal(p.Path)
srcPath := src.path(p)
dstPath := fs.path(p)
if err := fs.mkdirAll(dstPath); err != nil {
@ -139,6 +142,7 @@ func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error {
// DownloadPart download part p from fs to w.
func (fs *FS) DownloadPart(p common.Part, w io.Writer) error {
p.Path = canonicalPathToLocal(p.Path)
path := fs.path(p)
r, err := os.Open(path)
if err != nil {
@ -201,6 +205,7 @@ func (fs *FS) path(p common.Part) string {
//
// The function does nothing if the filePath doesn't exist.
func (fs *FS) DeleteFile(filePath string) error {
filePath = canonicalPathToLocal(filePath)
path := filepath.Join(fs.Dir, filePath)
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {

12
lib/backup/fsremote/normilize_path_bsd.go Normal file
View file

@ -0,0 +1,12 @@
//go:build freebsd || openbsd || dragonfly || netbsd
// +build freebsd openbsd dragonfly netbsd
package fsremote
func pathToCanonical(path string) string {
return path
}
func canonicalPathToLocal(path string) string {
return path
}

9
lib/backup/fsremote/normilize_path_darwin.go Normal file
View file

@ -0,0 +1,9 @@
package fsremote
func pathToCanonical(path string) string {
return path
}
func canonicalPathToLocal(path string) string {
return path
}

9
lib/backup/fsremote/normilize_path_linux.go Normal file
View file

@ -0,0 +1,9 @@
package fsremote
func pathToCanonical(path string) string {
return path
}
func canonicalPathToLocal(path string) string {
return path
}

9
lib/backup/fsremote/normilize_path_solaris.go Normal file
View file

@ -0,0 +1,9 @@
package fsremote
func pathToCanonical(path string) string {
return path
}
func canonicalPathToLocal(path string) string {
return path
}

11
lib/backup/fsremote/normilize_path_windows.go Normal file
View file

@ -0,0 +1,11 @@
package fsremote
import "strings"
func pathToCanonical(path string) string {
return strings.ReplaceAll(path, "\\", "/")
}
func canonicalPathToLocal(path string) string {
return strings.ReplaceAll(path, "/", "\\")
}

7
lib/httpserver/httpserver.go
View file

@ -436,11 +436,12 @@ var (
// GetQuotedRemoteAddr returns quoted remote address.
func GetQuotedRemoteAddr(r *http.Request) string {
remoteAddr := strconv.Quote(r.RemoteAddr) // quote remoteAddr and X-Forwarded-For, since they may contain untrusted input
remoteAddr := r.RemoteAddr
if addr := r.Header.Get("X-Forwarded-For"); addr != "" {
remoteAddr += ", X-Forwarded-For: " + strconv.Quote(addr)
remoteAddr += ", X-Forwarded-For: " + addr
}
return remoteAddr
// quote remoteAddr and X-Forwarded-For, since they may contain untrusted input
return strconv.Quote(remoteAddr)
}
// Errorf writes formatted error message to w and to logger.

36
lib/httpserver/httpserver_test.go Normal file
View file

@ -0,0 +1,36 @@
package httpserver
import (
"encoding/json"
"net/http"
"testing"
)
func TestGetQuotedRemoteAddr(t *testing.T) {
f := func(remoteAddr, xForwardedFor, expectedAddr string) {
t.Helper()
req := &http.Request{
RemoteAddr: remoteAddr,
}
if xForwardedFor != "" {
req.Header = map[string][]string{
"X-Forwarded-For": {xForwardedFor},
}
}
addr := GetQuotedRemoteAddr(req)
if addr != expectedAddr {
t.Fatalf("unexpected remote addr;\ngot\n%s\nwant\n%s", addr, expectedAddr)
}
// Verify that the addr can be unmarshaled as JSON string
var s string
if err := json.Unmarshal([]byte(addr), &s); err != nil {
t.Fatalf("cannot unmarshal addr: %s", err)
}
}
f("1.2.3.4", "", `"1.2.3.4"`)
f("1.2.3.4", "foo.bar", `"1.2.3.4, X-Forwarded-For: foo.bar"`)
f("1.2\n\"3.4", "foo\nb\"ar", `"1.2\n\"3.4, X-Forwarded-For: foo\nb\"ar"`)
}

20
lib/logstorage/indexdb.go
View file

@ -33,6 +33,18 @@ const (
type IndexdbStats struct {
// StreamsCreatedTotal is the number of log streams created since the indexdb initialization.
StreamsCreatedTotal uint64
// IndexdbSizeBytes is the size of data in indexdb.
IndexdbSizeBytes uint64
// IndexdbItemsCount is the number of items in indexdb.
IndexdbItemsCount uint64
// IndexdbBlocksCount is the number of blocks in indexdb.
IndexdbBlocksCount uint64
// IndexdbPartsCount is the number of parts in indexdb.
IndexdbPartsCount uint64
}
type indexdb struct {
@ -88,6 +100,14 @@ func (idb *indexdb) debugFlush() {
func (idb *indexdb) updateStats(d *IndexdbStats) {
d.StreamsCreatedTotal += atomic.LoadUint64(&idb.streamsCreatedTotal)
var tm mergeset.TableMetrics
idb.tb.UpdateMetrics(&tm)
d.IndexdbSizeBytes += tm.InmemorySizeBytes + tm.FileSizeBytes
d.IndexdbItemsCount += tm.InmemoryItemsCount + tm.FileItemsCount
d.IndexdbPartsCount += tm.InmemoryPartsCount + tm.FilePartsCount
d.IndexdbBlocksCount += tm.InmemoryBlocksCount + tm.FileBlocksCount
}
func (idb *indexdb) appendStreamTagsByStreamID(dst []byte, sid *streamID) []byte {

2
lib/promrelabel/relabel.go
View file

@ -158,7 +158,7 @@ func FinalizeLabels(dst, src []prompbmarshal.Label) []prompbmarshal.Label {
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset int) []prompbmarshal.Label {
src := labels[labelsOffset:]
if !prc.If.Match(labels) {
if !prc.If.Match(src) {
if prc.Action == "keep" {
// Drop the target on `if` mismatch for `action: keep`
return labels[:labelsOffset]

46
lib/promrelabel/relabel_test.go
View file

@ -915,3 +915,49 @@ func newTestRegexRelabelConfig(pattern string) *parsedRelabelConfig {
}
return prc
}
func TestParsedRelabelConfigsApplyForMultipleSeries(t *testing.T) {
f := func(config string, metrics []string, resultExpected []string) {
t.Helper()
pcs, err := ParseRelabelConfigsData([]byte(config))
if err != nil {
t.Fatalf("cannot parse %q: %s", config, err)
}
totalLabels := 0
var labels []prompbmarshal.Label
for _, metric := range metrics {
labels = append(labels, promutils.MustNewLabelsFromString(metric).GetLabels()...)
resultLabels := pcs.Apply(labels, totalLabels)
SortLabels(resultLabels)
totalLabels += len(resultLabels)
labels = resultLabels
}
var result []string
for i := range labels {
result = append(result, LabelsToString(labels[i:i+1]))
}
if len(result) != len(resultExpected) {
t.Fatalf("unexpected number of results; got\n%q\nwant\n%q", result, resultExpected)
}
for i := range result {
if result[i] != resultExpected[i] {
t.Fatalf("unexpected result[%d]; got\n%q\nwant\n%q", i, result[i], resultExpected[i])
}
}
}
t.Run("drops one of series", func(t *testing.T) {
f(`
- action: drop
if: '{__name__!~"smth"}'
`, []string{`smth`, `notthis`}, []string{`smth`})
f(`
- action: drop
if: '{__name__!~"smth"}'
`, []string{`notthis`, `smth`}, []string{`smth`})
})
}

22
lib/promscrape/config.go
View file

@ -239,12 +239,17 @@ type GlobalConfig struct {
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config
type ScrapeConfig struct {
JobName string `yaml:"job_name"`
ScrapeInterval *promutils.Duration `yaml:"scrape_interval,omitempty"`
ScrapeTimeout *promutils.Duration `yaml:"scrape_timeout,omitempty"`
MetricsPath string `yaml:"metrics_path,omitempty"`
HonorLabels bool `yaml:"honor_labels,omitempty"`
HonorTimestamps *bool `yaml:"honor_timestamps,omitempty"`
JobName string `yaml:"job_name"`
ScrapeInterval *promutils.Duration `yaml:"scrape_interval,omitempty"`
ScrapeTimeout *promutils.Duration `yaml:"scrape_timeout,omitempty"`
MetricsPath string `yaml:"metrics_path,omitempty"`
HonorLabels bool `yaml:"honor_labels,omitempty"`
// HonorTimestamps is set to false by default contrary to Prometheus, which sets it to true by default,
// because of the issue with gaps on graphs when scraping cadvisor or similar targets, which export invalid timestamps.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799 for details.
HonorTimestamps bool `yaml:"honor_timestamps,omitempty"`
Scheme string `yaml:"scheme,omitempty"`
Params map[string][]string `yaml:"params,omitempty"`
HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
@ -984,10 +989,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
scrapeTimeout = scrapeInterval
}
honorLabels := sc.HonorLabels
honorTimestamps := true
if sc.HonorTimestamps != nil {
honorTimestamps = *sc.HonorTimestamps
}
honorTimestamps := sc.HonorTimestamps
denyRedirects := false
if sc.HTTPClientConfig.FollowRedirects != nil {
denyRedirects = !*sc.HTTPClientConfig.FollowRedirects

164
lib/promscrape/config_test.go
View file

@ -89,7 +89,7 @@ scrape_configs:
scrape_configs:
- job_name: foo
honor_labels: true
honor_timestamps: false
honor_timestamps: true
scheme: https
params:
foo:
@ -243,10 +243,9 @@ scrape_configs:
resetNonEssentialFields(sws)
swsExpected := []*ScrapeWork{
{
ScrapeURL: "http://host1:80/metric/path1?x=y",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://host1:80/metric/path1?x=y",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "host1:80",
"job": "abc",
@ -256,10 +255,9 @@ scrape_configs:
jobNameOriginal: "abc",
},
{
ScrapeURL: "https://host2:443/metric/path2?x=y",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "https://host2:443/metric/path2?x=y",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "host2:443",
"job": "abc",
@ -269,10 +267,9 @@ scrape_configs:
jobNameOriginal: "abc",
},
{
ScrapeURL: "http://host3:1234/metric/path3?arg1=value1&x=y",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://host3:1234/metric/path3?arg1=value1&x=y",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "host3:1234",
"job": "abc",
@ -282,10 +279,9 @@ scrape_configs:
jobNameOriginal: "abc",
},
{
ScrapeURL: "https://host4:1234/foo/bar?x=y",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "https://host4:1234/foo/bar?x=y",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "host4:1234",
"job": "abc",
@ -330,10 +326,9 @@ scrape_configs:
sws := cfg.getStaticScrapeWork()
resetNonEssentialFields(sws)
swsExpected := []*ScrapeWork{{
ScrapeURL: "http://black:9115/probe?module=dns_udp_example&target=8.8.8.8",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://black:9115/probe?module=dns_udp_example&target=8.8.8.8",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "8.8.8.8",
"job": "blackbox",
@ -757,10 +752,9 @@ scrape_configs:
- files: ["testdata/file_sd.json", "testdata/file_sd*.yml"]
`, []*ScrapeWork{
{
ScrapeURL: "http://host1:80/abc/de",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://host1:80/abc/de",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "host1:80",
"job": "foo",
@ -771,10 +765,9 @@ scrape_configs:
jobNameOriginal: "foo",
},
{
ScrapeURL: "http://host2:80/abc/de",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://host2:80/abc/de",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "host2:80",
"job": "foo",
@ -785,10 +778,9 @@ scrape_configs:
jobNameOriginal: "foo",
},
{
ScrapeURL: "http://localhost:9090/abc/de",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://localhost:9090/abc/de",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "localhost:9090",
"job": "foo",
@ -821,10 +813,9 @@ scrape_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "foo.bar:1234",
"job": "foo",
@ -845,10 +836,9 @@ scrape_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "foo.bar:1234",
"job": "foo",
@ -873,7 +863,7 @@ scrape_configs:
metrics_path: /foo/bar
scheme: https
honor_labels: true
honor_timestamps: false
honor_timestamps: true
follow_redirects: false
params:
p: ["x&y", "="]
@ -899,7 +889,7 @@ scrape_configs:
ScrapeInterval: 54 * time.Second,
ScrapeTimeout: 5 * time.Second,
HonorLabels: true,
HonorTimestamps: false,
HonorTimestamps: true,
DenyRedirects: true,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "foo.bar:443",
@ -916,7 +906,7 @@ scrape_configs:
ScrapeInterval: 54 * time.Second,
ScrapeTimeout: 5 * time.Second,
HonorLabels: true,
HonorTimestamps: false,
HonorTimestamps: true,
DenyRedirects: true,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "aaa:443",
@ -929,10 +919,9 @@ scrape_configs:
jobNameOriginal: "foo",
},
{
ScrapeURL: "http://1.2.3.4:80/metrics",
ScrapeInterval: 8 * time.Second,
ScrapeTimeout: 8 * time.Second,
HonorTimestamps: true,
ScrapeURL: "http://1.2.3.4:80/metrics",
ScrapeInterval: 8 * time.Second,
ScrapeTimeout: 8 * time.Second,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "1.2.3.4:80",
"job": "qwer",
@ -945,10 +934,9 @@ scrape_configs:
jobNameOriginal: "qwer",
},
{
ScrapeURL: "http://foobar:80/metrics",
ScrapeInterval: 8 * time.Second,
ScrapeTimeout: 8 * time.Second,
HonorTimestamps: true,
ScrapeURL: "http://foobar:80/metrics",
ScrapeInterval: 8 * time.Second,
ScrapeTimeout: 8 * time.Second,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "foobar:80",
"job": "asdf",
@ -995,10 +983,9 @@ scrape_configs:
- targets: ["foo.bar:1234", "drop-this-target"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics?x=keep_me",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://foo.bar:1234/metrics?x=keep_me",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"hash": "82",
"instance": "foo.bar:1234",
@ -1039,10 +1026,9 @@ scrape_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "mailto://foo.bar:1234/abc.de?a=b",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "mailto://foo.bar:1234/abc.de?a=b",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "fake.addr",
"job": "https",
@ -1074,10 +1060,9 @@ scrape_configs:
- targets: ["foo.bar:1234", "xyz"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "foo.bar:1234",
"job": "3",
@ -1098,10 +1083,9 @@ scrape_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "foo.bar:1234",
"job": "foo",
@ -1118,10 +1102,9 @@ scrape_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "foo.bar:1234",
"job": "foo",
@ -1138,10 +1121,9 @@ scrape_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "foo.bar:1234",
"job": "foo",
@ -1172,10 +1154,9 @@ scrape_configs:
job: yyy
`, []*ScrapeWork{
{
ScrapeURL: "http://pp:80/metrics?a=c&a=xy",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://pp:80/metrics?a=c&a=xy",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"foo": "bar",
"instance": "pp:80",
@ -1239,10 +1220,9 @@ scrape_configs:
replacement: true
`, []*ScrapeWork{
{
ScrapeURL: "http://127.0.0.1:9116/snmp?module=if_mib&target=192.168.1.2",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://127.0.0.1:9116/snmp?module=if_mib&target=192.168.1.2",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "192.168.1.2",
"job": "snmp",
@ -1269,10 +1249,9 @@ scrape_configs:
target_label: __metrics_path__
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metricspath",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
ScrapeURL: "http://foo.bar:1234/metricspath",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "foo.bar:1234",
"job": "path wo slash",
@ -1300,7 +1279,6 @@ scrape_configs:
ScrapeTimeout: time.Hour * 24,
ScrapeAlignInterval: time.Hour * 24,
ScrapeOffset: time.Hour * 24 * 2,
HonorTimestamps: true,
NoStaleMarkers: true,
Labels: promutils.NewLabelsFromMap(map[string]string{
"instance": "foo.bar:1234",
@ -1340,16 +1318,14 @@ func TestScrapeConfigClone(t *testing.T) {
f(&ScrapeConfig{})
bFalse := false
var ie promrelabel.IfExpression
if err := ie.Parse(`{foo=~"bar",baz!="z"}`); err != nil {
t.Fatalf("unexpected error: %s", err)
}
f(&ScrapeConfig{
JobName: "foo",
ScrapeInterval: promutils.NewDuration(time.Second * 47),
HonorLabels: true,
HonorTimestamps: &bFalse,
JobName: "foo",
ScrapeInterval: promutils.NewDuration(time.Second * 47),
HonorLabels: true,
Params: map[string][]string{
"foo": {"bar", "baz"},
},

2
lib/storage/index_db_test.go
View file

@ -1479,7 +1479,7 @@ func TestIndexDBRepopulateAfterRotation(t *testing.T) {
prevGeneration := db.generation
// force index rotation
s.mustRotateIndexDB()
s.mustRotateIndexDB(time.Now())
// check tsidCache wasn't reset after the rotation
var cs2 fastcache.Stats

9
lib/storage/storage.go
View file

@ -692,8 +692,8 @@ func (s *Storage) retentionWatcher() {
select {
case <-s.stop:
return
case <-time.After(time.Second * time.Duration(d)):
s.mustRotateIndexDB()
case currentTime := <-time.After(time.Second * time.Duration(d)):
s.mustRotateIndexDB(currentTime)
}
}
}
@ -750,14 +750,15 @@ func (s *Storage) nextDayMetricIDsUpdater() {
}
}
func (s *Storage) mustRotateIndexDB() {
func (s *Storage) mustRotateIndexDB(currentTime time.Time) {
// Create new indexdb table, which will be used as idbNext
newTableName := nextIndexDBTableName()
idbNewPath := filepath.Join(s.path, indexdbDirname, newTableName)
idbNew := mustOpenIndexDB(idbNewPath, s, &s.isReadOnly)
// Update nextRotationTimestamp
atomic.AddInt64(&s.nextRotationTimestamp, s.retentionMsecs/1000)
nextRotationTimestamp := currentTime.UnixMilli() + s.retentionMsecs/1000
atomic.StoreInt64(&s.nextRotationTimestamp, nextRotationTimestamp)
// Set idbNext to idbNew
idbNext := s.idbNext.Load()

2
lib/storage/storage_test.go
View file

@ -1094,7 +1094,7 @@ func TestStorageRotateIndexDB(t *testing.T) {
return
default:
time.Sleep(time.Millisecond)
s.mustRotateIndexDB()
s.mustRotateIndexDB(time.Now())
}
}
}()

Some files were not shown because too many files have changed in this diff Show more