mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-01 14:47:38 +00:00
Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files
This commit is contained in:
Aliaksandr Valialkin
commit
0158237875
22 changed files with 256 additions and 57 deletions
5
.github/workflows/nightly-build.yml
vendored
5
.github/workflows/nightly-build.yml
vendored
|
|
@ -17,6 +17,11 @@ jobs:
|
|||
with:
|
||||
go-version: 1.19.4
|
||||
id: go
|
||||
- name: Setup docker scan
|
||||
run: |
|
||||
mkdir -p ~/.docker/cli-plugins && \
|
||||
curl https://github.com/docker/scan-cli-plugin/releases/latest/download/docker-scan_linux_amd64 -L -s -S -o ~/.docker/cli-plugins/docker-scan &&\
|
||||
chmod +x ~/.docker/cli-plugins/docker-scan
|
||||
- name: Code checkout
|
||||
uses: actions/checkout@master
|
||||
- name: Publish
|
||||
|
|
|
|||
10
Makefile
10
Makefile
|
|
@ -167,10 +167,10 @@ vmutils-crossbuild: \
|
|||
vmutils-windows-amd64
|
||||
|
||||
publish-release:
|
||||
git checkout $(TAG) && $(MAKE) release publish && \
|
||||
git checkout $(TAG)-cluster && $(MAKE) release publish && \
|
||||
git checkout $(TAG)-enterprise && $(MAKE) release publish && \
|
||||
git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish
|
||||
git checkout $(TAG) && LATEST_TAG=stable $(MAKE) release publish && \
|
||||
git checkout $(TAG)-cluster && LATEST_TAG=cluster-stable $(MAKE) release publish && \
|
||||
git checkout $(TAG)-enterprise && LATEST_TAG=enterprise-stable $(MAKE) release publish && \
|
||||
git checkout $(TAG)-enterprise-cluster && LATEST_TAG=enterprise-cluster-stable $(MAKE) release publish
|
||||
|
||||
release: \
|
||||
release-victoria-metrics \
|
||||
|
|
@ -383,7 +383,7 @@ golangci-lint: install-golangci-lint
|
|||
golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
|
||||
|
||||
install-golangci-lint:
|
||||
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.48.0
|
||||
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.50.1
|
||||
|
||||
govulncheck: install-govulncheck
|
||||
govulncheck ./...
|
||||
|
|
|
|||
|
|
@ -363,7 +363,7 @@ DataDog agent allows configuring destinations for metrics sending via ENV variab
|
|||
or via [configuration file](https://docs.datadoghq.com/agent/guide/agent-configuration-files/) in section `dd_url`.
|
||||
|
||||
<p align="center">
|
||||
<img src="Single-server-VictoriaMetrics-sending_DD_metrics_to_VM.png" width="800">
|
||||
<img src="docs/Single-server-VictoriaMetrics-sending_DD_metrics_to_VM.png" width="800">
|
||||
</p>
|
||||
|
||||
To configure DataDog agent via ENV variable add the following prefix:
|
||||
|
|
@ -397,7 +397,7 @@ DataDog allows configuring [Dual Shipping](https://docs.datadoghq.com/agent/guid
|
|||
sending via ENV variable `DD_ADDITIONAL_ENDPOINTS` or via configuration file `additional_endpoints`.
|
||||
|
||||
<p align="center">
|
||||
<img src="Single-server-VictoriaMetrics-sending_DD_metrics_to_VM_and_DD.png" width="800">
|
||||
<img src="docs/Single-server-VictoriaMetrics-sending_DD_metrics_to_VM_and_DD.png" width="800">
|
||||
</p>
|
||||
|
||||
Run DataDog using the following ENV variable with VictoriaMetrics as additional metrics receiver:
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ func initLabelsGlobal() {
|
|||
}
|
||||
|
||||
func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
|
||||
if len(extraLabels) == 0 && pcs.Len() == 0 {
|
||||
if len(extraLabels) == 0 && pcs.Len() == 0 && !*usePromCompatibleNaming {
|
||||
// Nothing to change.
|
||||
return tss
|
||||
}
|
||||
|
|
|
|||
49
app/vmagent/remotewrite/relabel_test.go
Normal file
49
app/vmagent/remotewrite/relabel_test.go
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
package remotewrite
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
)
|
||||
|
||||
func TestApplyRelabeling(t *testing.T) {
|
||||
f := func(extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
|
||||
rctx := &relabelCtx{}
|
||||
tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
|
||||
gotTss := rctx.applyRelabeling(tss, extraLabels, pcs)
|
||||
if !reflect.DeepEqual(gotTss, expTss) {
|
||||
t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
|
||||
}
|
||||
}
|
||||
|
||||
f(nil, nil, "up", "up")
|
||||
f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, "up", `up{foo="bar"}`)
|
||||
f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, `up{foo="baz"}`, `up{foo="bar"}`)
|
||||
|
||||
pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
|
||||
- target_label: "foo"
|
||||
replacement: "aaa"
|
||||
- action: labeldrop
|
||||
regex: "env.*"
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
f(nil, pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
|
||||
|
||||
oldVal := *usePromCompatibleNaming
|
||||
*usePromCompatibleNaming = true
|
||||
f(nil, nil, `foo.bar`, `foo_bar`)
|
||||
*usePromCompatibleNaming = oldVal
|
||||
}
|
||||
|
||||
func parseSeries(data string) []prompbmarshal.TimeSeries {
|
||||
var tss []prompbmarshal.TimeSeries
|
||||
tss = append(tss, prompbmarshal.TimeSeries{
|
||||
Labels: promutils.MustNewLabelsFromString(data).GetLabels(),
|
||||
})
|
||||
return tss
|
||||
}
|
||||
|
|
@ -27,11 +27,11 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
textTpl "text/template"
|
||||
"time"
|
||||
|
||||
textTpl "text/template"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
)
|
||||
|
||||
|
|
@ -350,15 +350,7 @@ func templateFuncs() textTpl.FuncMap {
|
|||
if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
|
||||
return fmt.Sprintf("%.4g", v), nil
|
||||
}
|
||||
prefix := ""
|
||||
for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
|
||||
if math.Abs(v) < 1024 {
|
||||
break
|
||||
}
|
||||
prefix = p
|
||||
v /= 1024
|
||||
}
|
||||
return fmt.Sprintf("%.4g%s", v, prefix), nil
|
||||
return formatutil.HumanizeBytes(v), nil
|
||||
},
|
||||
|
||||
// humanizeDuration converts given seconds to a human-readable duration
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package templates
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
textTpl "text/template"
|
||||
|
|
@ -50,6 +51,31 @@ func TestTemplateFuncs(t *testing.T) {
|
|||
if !ok {
|
||||
t.Fatalf("unexpected mismatch")
|
||||
}
|
||||
|
||||
formatting := func(funcName string, p interface{}, resultExpected string) {
|
||||
t.Helper()
|
||||
v := funcs[funcName]
|
||||
fLocal := v.(func(s interface{}) (string, error))
|
||||
result, err := fLocal(p)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
|
||||
}
|
||||
if result != resultExpected {
|
||||
t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
|
||||
}
|
||||
}
|
||||
formatting("humanize1024", float64(0), "0")
|
||||
formatting("humanize1024", math.Inf(0), "+Inf")
|
||||
formatting("humanize1024", math.NaN(), "NaN")
|
||||
formatting("humanize1024", float64(127087), "124.1ki")
|
||||
formatting("humanize1024", float64(130137088), "124.1Mi")
|
||||
formatting("humanize1024", float64(133260378112), "124.1Gi")
|
||||
formatting("humanize1024", float64(136458627186688), "124.1Ti")
|
||||
formatting("humanize1024", float64(139733634239168512), "124.1Pi")
|
||||
formatting("humanize1024", float64(143087241460908556288), "124.1Ei")
|
||||
formatting("humanize1024", float64(146521335255970361638912), "124.1Zi")
|
||||
formatting("humanize1024", float64(150037847302113650318245888), "124.1Yi")
|
||||
formatting("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
|
||||
}
|
||||
|
||||
func mkTemplate(current, replacement interface{}) textTemplate {
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ The result on the GCS bucket. We see only 3 daily backups:
|
|||
* GET `/api/v1/backups` - returns list of backups in remote storage.
|
||||
Example output:
|
||||
```json
|
||||
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
|
||||
[{"name":"daily/2022-11-30","size_bytes":26664689,"size":"25.429Mi"},{"name":"daily/2022-12-01","size_bytes":40160965,"size":"38.300Mi"},{"name":"hourly/2022-11-30:12","size_bytes":5846529,"size":"5.576Mi"},{"name":"hourly/2022-11-30:13","size_bytes":17651847,"size":"16.834Mi"},{"name":"hourly/2022-11-30:13:22","size_bytes":8797831,"size":"8.390Mi"},{"name":"hourly/2022-11-30:14","size_bytes":10680454,"size":"10.186Mi"}]
|
||||
```
|
||||
|
||||
* POST `/api/v1/restore` - saves backup name to restore when [performing restore](#restore-commands).
|
||||
|
|
@ -211,7 +211,7 @@ It can be changed by using flag:
|
|||
`vmbackupmanager backup list` lists backups in remote storage:
|
||||
```console
|
||||
$ ./vmbackupmanager backup list
|
||||
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
|
||||
[{"name":"daily/2022-11-30","size_bytes":26664689,"size":"25.429Mi"},{"name":"daily/2022-12-01","size_bytes":40160965,"size":"38.300Mi"},{"name":"hourly/2022-11-30:12","size_bytes":5846529,"size":"5.576Mi"},{"name":"hourly/2022-11-30:13","size_bytes":17651847,"size":"16.834Mi"},{"name":"hourly/2022-11-30:13:22","size_bytes":8797831,"size":"8.390Mi"},{"name":"hourly/2022-11-30:14","size_bytes":10680454,"size":"10.186Mi"}]
|
||||
```
|
||||
|
||||
### Restore commands
|
||||
|
|
@ -270,7 +270,15 @@ If restore mark doesn't exist at `storageDataPath`(restore wasn't requested) `vm
|
|||
|
||||
### How to restore in Kubernetes
|
||||
|
||||
1. Enter container running `vmbackupmanager`
|
||||
1. Ensure there is an init container with `vmbackupmanager restore` in `vmstorage` or `vmsingle` pod.
|
||||
For [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/VictoriaMetrics-Operator.html) deployments it is required to add:
|
||||
```yaml
|
||||
vmbackup:
|
||||
restore:
|
||||
onStart: "true"
|
||||
```
|
||||
See operator `VMStorage` schema [here](https://docs.victoriametrics.com/operator/api.html#vmstorage) and `VMSingle` [here](https://docs.victoriametrics.com/operator/api.html#vmsinglespec).
|
||||
2. Enter container running `vmbackupmanager`
|
||||
2. Use `vmbackupmanager backup list` to get list of available backups:
|
||||
```console
|
||||
$ /vmbackupmanager-prod backup list
|
||||
|
|
@ -287,6 +295,33 @@ If restore mark doesn't exist at `storageDataPath`(restore wasn't requested) `vm
|
|||
```
|
||||
4. Restart pod
|
||||
|
||||
#### Restore cluster into another cluster
|
||||
|
||||
These steps are assuming that [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/VictoriaMetrics-Operator.html) is used to manage `VMCluster`.
|
||||
Clusters here are referred to as `source` and `destination`.
|
||||
|
||||
1. Create a new cluster with access to *source* cluster `vmbackupmanager` storage and same number of storage nodes.
|
||||
Add the following section in order to enable restore on start (operator `VMStorage` schema can be found [here](https://docs.victoriametrics.com/operator/api.html#vmstorage):
|
||||
```yaml
|
||||
vmbackup:
|
||||
restore:
|
||||
onStart: "true"
|
||||
```
|
||||
Note: it is safe to leave this section in the cluster configuration, since it will be ignored if restore mark doesn't exist.
|
||||
> Important! Use different `-dst` for *destination* cluster to avoid overwriting backup data of the *source* cluster.
|
||||
2. Enter container running `vmbackupmanager` in *source* cluster
|
||||
2. Use `vmbackupmanager backup list` to get list of available backups:
|
||||
```console
|
||||
$ /vmbackupmanager-prod backup list
|
||||
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
|
||||
```
|
||||
3. Use `vmbackupmanager restore create` to create restore mark at each pod of the *destination* cluster.
|
||||
Each pod in *destination* cluster should be restored from backup of respective pod in *source* cluster.
|
||||
For example: `vmstorage-source-0` in *source* cluster should be restored from `vmstorage-destination-0` in *destination* cluster.
|
||||
```console
|
||||
$ /vmbackupmanager-prod restore create s3://source_cluster/vmstorage-source-0/daily/2022-10-06
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Flags
|
||||
|
|
|
|||
|
|
@ -935,7 +935,7 @@ func TestExecSuccess(t *testing.T) {
|
|||
))`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{nan, nan, nan, 1, nan, nan},
|
||||
Values: []float64{nan, nan, 1, 1, nan, nan},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
|
|
@ -6692,7 +6692,7 @@ func TestExecSuccess(t *testing.T) {
|
|||
q := `rate((2000-time())[100s])`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{5.5, 4.5, 3.5, 2.5, 1.5, 0.5},
|
||||
Values: []float64{5, 4, 3, 2, 1, 0},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
|
|
@ -6703,7 +6703,7 @@ func TestExecSuccess(t *testing.T) {
|
|||
q := `rate((2000-time())[100s:])`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{5.5, 4.5, 3.5, 2.5, 1.5, 0.5},
|
||||
Values: []float64{5, 4, 3, 2, 1, 0},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
|
|
@ -6714,7 +6714,7 @@ func TestExecSuccess(t *testing.T) {
|
|||
q := `rate((2000-time())[100s:100s])`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0, 0, 6.5, 4.5, 2.5, 0.5},
|
||||
Values: []float64{0, 0, 6, 4, 2, 0},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
|
|
@ -6725,7 +6725,7 @@ func TestExecSuccess(t *testing.T) {
|
|||
q := `rate((2000-time())[100s:100s] offset 100s)`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0, 0, 3.5, 5.5, 3.5, 1.5},
|
||||
Values: []float64{0, 0, 7, 5, 3, 1},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
|
|
@ -6736,7 +6736,7 @@ func TestExecSuccess(t *testing.T) {
|
|||
q := `rate((2000-time())[100s:100s] offset 100s)[:] offset 100s`
|
||||
r := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0, 0, 0, 3.5, 5.5, 3.5},
|
||||
Values: []float64{0, 0, 0, 7, 5, 3},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
resultExpected := []netstorage.Result{r}
|
||||
|
|
|
|||
|
|
@ -145,11 +145,11 @@ var rollupAggrFuncs = map[string]rollupFunc{
|
|||
"zscore_over_time": rollupZScoreOverTime,
|
||||
}
|
||||
|
||||
// VictoriaMetrics can increase lookbehind window in square brackets for these functions
|
||||
// if the given window doesn't contain enough samples for calculations.
|
||||
// VictoriaMetrics can extends lookbehind window for these functions
|
||||
// in order to make sure it contains enough points for returning non-empty results.
|
||||
//
|
||||
// This is needed in order to return the expected non-empty graphs when zooming in the graph in Grafana,
|
||||
// which is built with `func_name(metric[$__interval])` query.
|
||||
// This is needed for returning the expected non-empty graphs when zooming in the graph in Grafana,
|
||||
// which is built with `func_name(metric)` query.
|
||||
var rollupFuncsCanAdjustWindow = map[string]bool{
|
||||
"default_rollup": true,
|
||||
"deriv": true,
|
||||
|
|
@ -584,15 +584,23 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
|
|||
window := rc.Window
|
||||
if window <= 0 {
|
||||
window = rc.Step
|
||||
if rc.MayAdjustWindow && window < maxPrevInterval {
|
||||
// Adjust lookbehind window only if it isn't set explicilty, e.g. rate(foo).
|
||||
// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
|
||||
// when the window doesn't cover at least two raw samples (this is what most users expect).
|
||||
//
|
||||
// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
|
||||
// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
|
||||
//
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
|
||||
window = maxPrevInterval
|
||||
}
|
||||
if rc.isDefaultRollup && rc.LookbackDelta > 0 && window > rc.LookbackDelta {
|
||||
// Implicit window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
|
||||
// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
|
||||
window = rc.LookbackDelta
|
||||
}
|
||||
}
|
||||
if rc.MayAdjustWindow && window < maxPrevInterval {
|
||||
window = maxPrevInterval
|
||||
}
|
||||
rfa := getRollupFuncArg()
|
||||
rfa.idx = 0
|
||||
rfa.window = window
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"files": {
|
||||
"main.css": "./static/css/main.fdc77f08.css",
|
||||
"main.js": "./static/js/main.2d332988.js",
|
||||
"main.js": "./static/js/main.ca04fac1.js",
|
||||
"static/js/27.c1ccfd29.chunk.js": "./static/js/27.c1ccfd29.chunk.js",
|
||||
"index.html": "./index.html"
|
||||
},
|
||||
"entrypoints": [
|
||||
"static/css/main.fdc77f08.css",
|
||||
"static/js/main.2d332988.js"
|
||||
"static/js/main.ca04fac1.js"
|
||||
]
|
||||
}
|
||||
|
|
@ -1 +1 @@
|
|||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono&family=Lato:wght@300;400;700&display=swap" rel="stylesheet"><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.2d332988.js"></script><link href="./static/css/main.fdc77f08.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono&family=Lato:wght@300;400;700&display=swap" rel="stylesheet"><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.ca04fac1.js"></script><link href="./static/css/main.fdc77f08.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -37,6 +37,7 @@ interface FetchDataParams {
|
|||
query: string[],
|
||||
stateSeriesLimits: SeriesLimits,
|
||||
showAllSeries?: boolean,
|
||||
hideQuery?: number[]
|
||||
}
|
||||
|
||||
export const useFetchQuery = ({
|
||||
|
|
@ -75,6 +76,7 @@ export const useFetchQuery = ({
|
|||
query,
|
||||
stateSeriesLimits,
|
||||
showAllSeries,
|
||||
hideQuery,
|
||||
}: FetchDataParams) => {
|
||||
const controller = new AbortController();
|
||||
setFetchQueue([...fetchQueue, controller]);
|
||||
|
|
@ -87,6 +89,13 @@ export const useFetchQuery = ({
|
|||
let totalLength = 0;
|
||||
|
||||
for await (const url of fetchUrl) {
|
||||
|
||||
const isHideQuery = hideQuery?.includes(counter - 1);
|
||||
if (isHideQuery) {
|
||||
counter++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const response = await fetch(url, { signal: controller.signal });
|
||||
const resp = await response.json();
|
||||
|
||||
|
|
@ -126,12 +135,6 @@ export const useFetchQuery = ({
|
|||
|
||||
const throttledFetchData = useCallback(debounce(fetchData, 800), []);
|
||||
|
||||
const filterExpr = (q: string, i: number) => {
|
||||
const byQuery = q.trim();
|
||||
const byHideQuery = hideQuery ? !hideQuery.includes(i) : true;
|
||||
return byQuery && byHideQuery;
|
||||
};
|
||||
|
||||
const fetchUrl = useMemo(() => {
|
||||
const expr = predefinedQuery ?? query;
|
||||
const displayChart = (display || displayType) === "chart";
|
||||
|
|
@ -143,7 +146,7 @@ export const useFetchQuery = ({
|
|||
} else if (isValidHttpUrl(serverUrl)) {
|
||||
const updatedPeriod = { ...period };
|
||||
updatedPeriod.step = customStep;
|
||||
return expr.filter(filterExpr).map(q => displayChart
|
||||
return expr.map(q => displayChart
|
||||
? getQueryRangeUrl(serverUrl, q, updatedPeriod, nocache, isTracingEnabled)
|
||||
: getQueryUrl(serverUrl, q, updatedPeriod, isTracingEnabled));
|
||||
} else {
|
||||
|
|
@ -163,6 +166,7 @@ export const useFetchQuery = ({
|
|||
query: expr,
|
||||
stateSeriesLimits,
|
||||
showAllSeries,
|
||||
hideQuery,
|
||||
});
|
||||
}, [fetchUrl, visible, stateSeriesLimits, showAllSeries]);
|
||||
|
||||
|
|
|
|||
|
|
@ -113,5 +113,6 @@ See also [case studies](https://docs.victoriametrics.com/CaseStudies.html).
|
|||
* [OSA Con 2022: Specifics of data analysis in Time Series Databases](https://www.youtube.com/watch?v=_zORxrgLtec)
|
||||
* [OSMC 2022. VictoriaMetrics: scaling to 100 million metrics per second](https://www.youtube.com/watch?v=xfed9_Q0_qU). See also [slides](https://www.slideshare.net/NETWAYS/osmc-2022-victoriametrics-scaling-to-100-million-metrics-per-second-by-aliaksandr-valialkin)
|
||||
* [CNCF Paris Meetup 2022-09-15 - VictoriaMetrics - The cost of scale in Prometheus ecosystem](https://www.youtube.com/watch?v=gcZYHpri2Hw). See also [slides](https://docs.google.com/presentation/d/1jhZuKnAXi15M-mdBP5a4ZAiyrMeHhYmzO8xcZ6pMyLc/edit?usp=sharing)
|
||||
* [VictoriaMetrics Meetup December 2022](https://www.youtube.com/watch?v=Mesc6JBFNhQ). See also [slides for "VictoriaMetrics 2022: new features" talk](https://docs.google.com/presentation/d/1jI7XZoodmuzLymdu4MToG9onAKQjzCNwMO2NDupyUkQ/edit?usp=sharing).
|
||||
* [Comparing Thanos to VictoriaMetrics cluster](https://faun.pub/comparing-thanos-to-victoriametrics-cluster-b193bea1683)
|
||||
* [Evaluation performance and correctness: VictoriaMetrics response](https://valyala.medium.com/evaluating-performance-and-correctness-victoriametrics-response-e27315627e87)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,19 @@ The following tip changes can be tested by building VictoriaMetrics components f
|
|||
## tip
|
||||
|
||||
|
||||
## [v1.85.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.3)
|
||||
|
||||
Released at 2022-12-20
|
||||
|
||||
**Update note 1:** This and newer releases of VictoriaMetrics may return gaps for `rate(m[d])` queries on short time ranges if `[d]` lookbehind window is set expliticly. For example, `rate(http_requests_total[$__interval])`. This reduces confusion level when the user expects the needed results from the query with explicitly set lookbehind window. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483). The previous gap filling behaviour can be restored by removing explicit lookbehind window `[d]` from the query, e.g. by substituting the `rate(m[d])` with `rate(m)`. See [these docs](https://docs.victoriametrics.com/MetricsQL.html#implicit-query-conversions) for details.
|
||||
|
||||
* BUGFIX: fix `error when searching for TSIDs by metricIDs in the previous indexdb: EOF` error, which can occur during queries after unclean shutdown of VictoriaMetrics (e.g. via hardware reset, out of memory crash or `kill -9`). The error has been introduced in [v1.85.2](https://docs.victoriametrics.com/CHANGELOG.html#v1852). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3515).
|
||||
* BUGFIX: [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html): expose proper values for `vm_downsampling_partitions_scheduled` and `vm_downsampling_partitions_scheduled_size_bytes` metrics, which were added at [v1.78.0](https://docs.victoriametrics.com/CHANGELOG.html#v1780). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2612).
|
||||
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): never extend explicitly set lookbehind window for [rate()](https://docs.victoriametrics.com/MetricsQL.html#rate) function. This reduces the level of confusion when the user expects the needed results after explicitly seting the lookbehind window `[d]` in the query `rate(m[d])`. Previously VictoriaMetrics could silently extend the lookbehind window, so it covers at least two raw samples. Now this behavior works only if the lookbehind window in square brackets isn't set explicitly, e.g. in the case of `rate(m)`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483) for details.
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): respect `-usePromCompatibleNaming` flag if no relabeling or extra labels were set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3511) for details.
|
||||
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the wrong legend when queries are hidden. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3512).
|
||||
|
||||
|
||||
## [v1.85.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.2)
|
||||
|
||||
Released at 2022-12-19
|
||||
|
|
@ -23,11 +36,13 @@ Released at 2022-12-19
|
|||
* FEATURE: support overriding of `-search.latencyOffset` value via URL param `latency_offset` when performing requests to [/api/v1/query](https://docs.victoriametrics.com/keyConcepts.html#instant-query) and [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3481).
|
||||
* FEATURE: allow changing field names in JSON logs if VictoriaMetrics components are started with `-loggerFormat=json` command-line flags. The field names can be changed with the `-loggerJSONFields` command-line flag. For example `-loggerJSONFields=ts:timestamp,msg:message` would rename `ts` and `msg` fields on the output JSON to `timestamp` and `message` fields. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2348). Thanks to @michal-kralik for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3488).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `__meta_consul_tag_<tagname>` and `__meta_consul_tagpresent_<tagname>` labels for targets discovered via [consul_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs). This simplifies converting [Consul service tags](https://developer.hashicorp.com/consul/docs/discovery/services#service-definition) to target labels with a simple [relabeling rule](https://docs.victoriametrics.com/vmagent.html#relabeling):
|
||||
```yml
|
||||
- action: labelmap
|
||||
regex: __meta_consul_tag_(.+)
|
||||
```
|
||||
This resolves [this StackOverflow question](https://stackoverflow.com/questions/44339461/relabeling-in-prometheus).
|
||||
|
||||
```yml
|
||||
- action: labelmap
|
||||
regex: __meta_consul_tag_(.+)
|
||||
```
|
||||
|
||||
This resolves [this StackOverflow question](https://stackoverflow.com/questions/44339461/relabeling-in-prometheus).
|
||||
|
||||
* BUGFIX: properly return query results for time series, which stop receiving new samples after the rotation of `indexdb`. Previously such time series could be missing in query results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502). The issue has been introduced in [v1.83.0](https://docs.victoriametrics.com/CHANGELOG.html#v1830).
|
||||
* BUGFIX: allow specifying values bigger than 2GiB to the following command-line flag values on 32-bit architectures (`386` and `arm`): `-storage.minFreeDiskSpaceBytes` and `-remoteWrite.maxDiskUsagePerURL`. Previously values bigger than 2GiB were incorrectly truncated on these architectures.
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ It has integration with VictoriaMetrics `vmbackupmanager` - advanced tools for m
|
|||
|
||||
For kubernetes-cluster administrators, it simplifies installation, configuration and management for `VictoriaMetrics` application. The main feature of operator is its ability to delegate the configuration of applications monitoring to the end-users.
|
||||
|
||||
For applications developers, its great possibility for managing observability of applications. You can define metrics scraping and alerting configuration for your application and manage it with an application deployment process. Just define app_deployment.yaml, app_vmpodscrape.yaml and app_vmrule.yaml. That's it, you can apply it to a kubernetes cluster. Check [quick-start](/Operator/quick-start.html) for an example.
|
||||
For applications developers, its great possibility for managing observability of applications. You can define metrics scraping and alerting configuration for your application and manage it with an application deployment process. Just define app_deployment.yaml, app_vmpodscrape.yaml and app_vmrule.yaml. That's it, you can apply it to a kubernetes cluster. Check [quick-start](/operator/quick-start.html) for an example.
|
||||
|
||||
## Operator vs helm-chart
|
||||
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ The result on the GCS bucket. We see only 3 daily backups:
|
|||
* GET `/api/v1/backups` - returns list of backups in remote storage.
|
||||
Example output:
|
||||
```json
|
||||
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
|
||||
[{"name":"daily/2022-11-30","size_bytes":26664689,"size":"25.429Mi"},{"name":"daily/2022-12-01","size_bytes":40160965,"size":"38.300Mi"},{"name":"hourly/2022-11-30:12","size_bytes":5846529,"size":"5.576Mi"},{"name":"hourly/2022-11-30:13","size_bytes":17651847,"size":"16.834Mi"},{"name":"hourly/2022-11-30:13:22","size_bytes":8797831,"size":"8.390Mi"},{"name":"hourly/2022-11-30:14","size_bytes":10680454,"size":"10.186Mi"}]
|
||||
```
|
||||
|
||||
* POST `/api/v1/restore` - saves backup name to restore when [performing restore](#restore-commands).
|
||||
|
|
@ -215,7 +215,7 @@ It can be changed by using flag:
|
|||
`vmbackupmanager backup list` lists backups in remote storage:
|
||||
```console
|
||||
$ ./vmbackupmanager backup list
|
||||
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
|
||||
[{"name":"daily/2022-11-30","size_bytes":26664689,"size":"25.429Mi"},{"name":"daily/2022-12-01","size_bytes":40160965,"size":"38.300Mi"},{"name":"hourly/2022-11-30:12","size_bytes":5846529,"size":"5.576Mi"},{"name":"hourly/2022-11-30:13","size_bytes":17651847,"size":"16.834Mi"},{"name":"hourly/2022-11-30:13:22","size_bytes":8797831,"size":"8.390Mi"},{"name":"hourly/2022-11-30:14","size_bytes":10680454,"size":"10.186Mi"}]
|
||||
```
|
||||
|
||||
### Restore commands
|
||||
|
|
@ -274,7 +274,15 @@ If restore mark doesn't exist at `storageDataPath`(restore wasn't requested) `vm
|
|||
|
||||
### How to restore in Kubernetes
|
||||
|
||||
1. Enter container running `vmbackupmanager`
|
||||
1. Ensure there is an init container with `vmbackupmanager restore` in `vmstorage` or `vmsingle` pod.
|
||||
For [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/VictoriaMetrics-Operator.html) deployments it is required to add:
|
||||
```yaml
|
||||
vmbackup:
|
||||
restore:
|
||||
onStart: "true"
|
||||
```
|
||||
See operator `VMStorage` schema [here](https://docs.victoriametrics.com/operator/api.html#vmstorage) and `VMSingle` [here](https://docs.victoriametrics.com/operator/api.html#vmsinglespec).
|
||||
2. Enter container running `vmbackupmanager`
|
||||
2. Use `vmbackupmanager backup list` to get list of available backups:
|
||||
```console
|
||||
$ /vmbackupmanager-prod backup list
|
||||
|
|
@ -291,6 +299,33 @@ If restore mark doesn't exist at `storageDataPath`(restore wasn't requested) `vm
|
|||
```
|
||||
4. Restart pod
|
||||
|
||||
#### Restore cluster into another cluster
|
||||
|
||||
These steps are assuming that [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/VictoriaMetrics-Operator.html) is used to manage `VMCluster`.
|
||||
Clusters here are referred to as `source` and `destination`.
|
||||
|
||||
1. Create a new cluster with access to *source* cluster `vmbackupmanager` storage and same number of storage nodes.
|
||||
Add the following section in order to enable restore on start (operator `VMStorage` schema can be found [here](https://docs.victoriametrics.com/operator/api.html#vmstorage):
|
||||
```yaml
|
||||
vmbackup:
|
||||
restore:
|
||||
onStart: "true"
|
||||
```
|
||||
Note: it is safe to leave this section in the cluster configuration, since it will be ignored if restore mark doesn't exist.
|
||||
> Important! Use different `-dst` for *destination* cluster to avoid overwriting backup data of the *source* cluster.
|
||||
2. Enter container running `vmbackupmanager` in *source* cluster
|
||||
2. Use `vmbackupmanager backup list` to get list of available backups:
|
||||
```console
|
||||
$ /vmbackupmanager-prod backup list
|
||||
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
|
||||
```
|
||||
3. Use `vmbackupmanager restore create` to create restore mark at each pod of the *destination* cluster.
|
||||
Each pod in *destination* cluster should be restored from backup of respective pod in *source* cluster.
|
||||
For example: `vmstorage-source-0` in *source* cluster should be restored from `vmstorage-destination-0` in *destination* cluster.
|
||||
```console
|
||||
$ /vmbackupmanager-prod restore create s3://source_cluster/vmstorage-source-0/daily/2022-10-06
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Flags
|
||||
|
|
|
|||
|
|
@ -11,6 +11,12 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsnil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
var (
|
||||
bytesUploadedTotal = uint64(0)
|
||||
bytesUploadedTotalMetric = metrics.NewCounter(`vm_backups_uploaded_bytes_total`)
|
||||
)
|
||||
|
||||
// Backup performs backup according to the provided settings.
|
||||
|
|
@ -163,6 +169,8 @@ func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, con
|
|||
n := atomic.LoadUint64(&bytesUploaded)
|
||||
logger.Infof("uploaded %d out of %d bytes from src %s to dst %s in %s", n, uploadSize, src, dst, elapsed)
|
||||
})
|
||||
atomic.AddUint64(&bytesUploadedTotal, bytesUploaded)
|
||||
bytesUploadedTotalMetric.Set(bytesUploadedTotal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
19
lib/formatutil/human.go
Normal file
19
lib/formatutil/human.go
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
package formatutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
)
|
||||
|
||||
// HumanizeBytes returns human-readable representation of size in bytes with 1024 base.
|
||||
func HumanizeBytes(size float64) string {
|
||||
prefix := ""
|
||||
for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
|
||||
if math.Abs(size) < 1024 {
|
||||
break
|
||||
}
|
||||
prefix = p
|
||||
size /= 1024
|
||||
}
|
||||
return fmt.Sprintf("%.4g%s", size, prefix)
|
||||
}
|
||||
|
|
@ -1905,11 +1905,13 @@ func (db *indexDB) getTSIDsFromMetricIDs(qt *querytracer.Tracer, metricIDs []uin
|
|||
// Cannot find TSID for the given metricID.
|
||||
// This may be the case on incomplete indexDB
|
||||
// due to snapshot or due to unflushed entries.
|
||||
// Just increment errors counter and skip it.
|
||||
// Just increment errors counter and skip it for now.
|
||||
atomic.AddUint64(&is.db.missingTSIDsForMetricID, 1)
|
||||
err = nil
|
||||
continue
|
||||
}
|
||||
err = fmt.Errorf("cannot find tsid for metricID=%d: %w", metricID, err)
|
||||
return
|
||||
}
|
||||
is.db.putToMetricIDCache(metricID, tsid)
|
||||
i++
|
||||
|
|
|
|||
Loading…
Reference in a new issue