Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

This commit is contained in:
Aliaksandr Valialkin 2022-12-20 14:52:33 -08:00
commit 0158237875
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
22 changed files with 256 additions and 57 deletions

View file

@ -17,6 +17,11 @@ jobs:
with: with:
go-version: 1.19.4 go-version: 1.19.4
id: go id: go
- name: Setup docker scan
run: |
mkdir -p ~/.docker/cli-plugins && \
curl https://github.com/docker/scan-cli-plugin/releases/latest/download/docker-scan_linux_amd64 -L -s -S -o ~/.docker/cli-plugins/docker-scan &&\
chmod +x ~/.docker/cli-plugins/docker-scan
- name: Code checkout - name: Code checkout
uses: actions/checkout@master uses: actions/checkout@master
- name: Publish - name: Publish

View file

@ -167,10 +167,10 @@ vmutils-crossbuild: \
vmutils-windows-amd64 vmutils-windows-amd64
publish-release: publish-release:
git checkout $(TAG) && $(MAKE) release publish && \ git checkout $(TAG) && LATEST_TAG=stable $(MAKE) release publish && \
git checkout $(TAG)-cluster && $(MAKE) release publish && \ git checkout $(TAG)-cluster && LATEST_TAG=cluster-stable $(MAKE) release publish && \
git checkout $(TAG)-enterprise && $(MAKE) release publish && \ git checkout $(TAG)-enterprise && LATEST_TAG=enterprise-stable $(MAKE) release publish && \
git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish git checkout $(TAG)-enterprise-cluster && LATEST_TAG=enterprise-cluster-stable $(MAKE) release publish
release: \ release: \
release-victoria-metrics \ release-victoria-metrics \
@ -383,7 +383,7 @@ golangci-lint: install-golangci-lint
golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
install-golangci-lint: install-golangci-lint:
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.48.0 which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.50.1
govulncheck: install-govulncheck govulncheck: install-govulncheck
govulncheck ./... govulncheck ./...

View file

@ -363,7 +363,7 @@ DataDog agent allows configuring destinations for metrics sending via ENV variab
or via [configuration file](https://docs.datadoghq.com/agent/guide/agent-configuration-files/) in section `dd_url`. or via [configuration file](https://docs.datadoghq.com/agent/guide/agent-configuration-files/) in section `dd_url`.
<p align="center"> <p align="center">
<img src="Single-server-VictoriaMetrics-sending_DD_metrics_to_VM.png" width="800"> <img src="docs/Single-server-VictoriaMetrics-sending_DD_metrics_to_VM.png" width="800">
</p> </p>
To configure DataDog agent via ENV variable add the following prefix: To configure DataDog agent via ENV variable add the following prefix:
@ -397,7 +397,7 @@ DataDog allows configuring [Dual Shipping](https://docs.datadoghq.com/agent/guid
sending via ENV variable `DD_ADDITIONAL_ENDPOINTS` or via configuration file `additional_endpoints`. sending via ENV variable `DD_ADDITIONAL_ENDPOINTS` or via configuration file `additional_endpoints`.
<p align="center"> <p align="center">
<img src="Single-server-VictoriaMetrics-sending_DD_metrics_to_VM_and_DD.png" width="800"> <img src="docs/Single-server-VictoriaMetrics-sending_DD_metrics_to_VM_and_DD.png" width="800">
</p> </p>
Run DataDog using the following ENV variable with VictoriaMetrics as additional metrics receiver: Run DataDog using the following ENV variable with VictoriaMetrics as additional metrics receiver:

View file

@ -86,7 +86,7 @@ func initLabelsGlobal() {
} }
func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries { func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
if len(extraLabels) == 0 && pcs.Len() == 0 { if len(extraLabels) == 0 && pcs.Len() == 0 && !*usePromCompatibleNaming {
// Nothing to change. // Nothing to change.
return tss return tss
} }

View file

@ -0,0 +1,49 @@
package remotewrite
import (
"reflect"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
)
func TestApplyRelabeling(t *testing.T) {
f := func(extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
rctx := &relabelCtx{}
tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
gotTss := rctx.applyRelabeling(tss, extraLabels, pcs)
if !reflect.DeepEqual(gotTss, expTss) {
t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
}
}
f(nil, nil, "up", "up")
f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, "up", `up{foo="bar"}`)
f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, `up{foo="baz"}`, `up{foo="bar"}`)
pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
- target_label: "foo"
replacement: "aaa"
- action: labeldrop
regex: "env.*"
`))
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
f(nil, pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
oldVal := *usePromCompatibleNaming
*usePromCompatibleNaming = true
f(nil, nil, `foo.bar`, `foo_bar`)
*usePromCompatibleNaming = oldVal
}
func parseSeries(data string) []prompbmarshal.TimeSeries {
var tss []prompbmarshal.TimeSeries
tss = append(tss, prompbmarshal.TimeSeries{
Labels: promutils.MustNewLabelsFromString(data).GetLabels(),
})
return tss
}

View file

@ -27,11 +27,11 @@ import (
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
textTpl "text/template"
"time" "time"
textTpl "text/template"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
) )
@ -350,15 +350,7 @@ func templateFuncs() textTpl.FuncMap {
if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) { if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
return fmt.Sprintf("%.4g", v), nil return fmt.Sprintf("%.4g", v), nil
} }
prefix := "" return formatutil.HumanizeBytes(v), nil
for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
if math.Abs(v) < 1024 {
break
}
prefix = p
v /= 1024
}
return fmt.Sprintf("%.4g%s", v, prefix), nil
}, },
// humanizeDuration converts given seconds to a human-readable duration // humanizeDuration converts given seconds to a human-readable duration

View file

@ -1,6 +1,7 @@
package templates package templates
import ( import (
"math"
"strings" "strings"
"testing" "testing"
textTpl "text/template" textTpl "text/template"
@ -50,6 +51,31 @@ func TestTemplateFuncs(t *testing.T) {
if !ok { if !ok {
t.Fatalf("unexpected mismatch") t.Fatalf("unexpected mismatch")
} }
formatting := func(funcName string, p interface{}, resultExpected string) {
t.Helper()
v := funcs[funcName]
fLocal := v.(func(s interface{}) (string, error))
result, err := fLocal(p)
if err != nil {
t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
}
if result != resultExpected {
t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
}
}
formatting("humanize1024", float64(0), "0")
formatting("humanize1024", math.Inf(0), "+Inf")
formatting("humanize1024", math.NaN(), "NaN")
formatting("humanize1024", float64(127087), "124.1ki")
formatting("humanize1024", float64(130137088), "124.1Mi")
formatting("humanize1024", float64(133260378112), "124.1Gi")
formatting("humanize1024", float64(136458627186688), "124.1Ti")
formatting("humanize1024", float64(139733634239168512), "124.1Pi")
formatting("humanize1024", float64(143087241460908556288), "124.1Ei")
formatting("humanize1024", float64(146521335255970361638912), "124.1Zi")
formatting("humanize1024", float64(150037847302113650318245888), "124.1Yi")
formatting("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
} }
func mkTemplate(current, replacement interface{}) textTemplate { func mkTemplate(current, replacement interface{}) textTemplate {

View file

@ -158,7 +158,7 @@ The result on the GCS bucket. We see only 3 daily backups:
* GET `/api/v1/backups` - returns list of backups in remote storage. * GET `/api/v1/backups` - returns list of backups in remote storage.
Example output: Example output:
```json ```json
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"] [{"name":"daily/2022-11-30","size_bytes":26664689,"size":"25.429Mi"},{"name":"daily/2022-12-01","size_bytes":40160965,"size":"38.300Mi"},{"name":"hourly/2022-11-30:12","size_bytes":5846529,"size":"5.576Mi"},{"name":"hourly/2022-11-30:13","size_bytes":17651847,"size":"16.834Mi"},{"name":"hourly/2022-11-30:13:22","size_bytes":8797831,"size":"8.390Mi"},{"name":"hourly/2022-11-30:14","size_bytes":10680454,"size":"10.186Mi"}]
``` ```
* POST `/api/v1/restore` - saves backup name to restore when [performing restore](#restore-commands). * POST `/api/v1/restore` - saves backup name to restore when [performing restore](#restore-commands).
@ -211,7 +211,7 @@ It can be changed by using flag:
`vmbackupmanager backup list` lists backups in remote storage: `vmbackupmanager backup list` lists backups in remote storage:
```console ```console
$ ./vmbackupmanager backup list $ ./vmbackupmanager backup list
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"] [{"name":"daily/2022-11-30","size_bytes":26664689,"size":"25.429Mi"},{"name":"daily/2022-12-01","size_bytes":40160965,"size":"38.300Mi"},{"name":"hourly/2022-11-30:12","size_bytes":5846529,"size":"5.576Mi"},{"name":"hourly/2022-11-30:13","size_bytes":17651847,"size":"16.834Mi"},{"name":"hourly/2022-11-30:13:22","size_bytes":8797831,"size":"8.390Mi"},{"name":"hourly/2022-11-30:14","size_bytes":10680454,"size":"10.186Mi"}]
``` ```
### Restore commands ### Restore commands
@ -270,7 +270,15 @@ If restore mark doesn't exist at `storageDataPath`(restore wasn't requested) `vm
### How to restore in Kubernetes ### How to restore in Kubernetes
1. Enter container running `vmbackupmanager` 1. Ensure there is an init container with `vmbackupmanager restore` in `vmstorage` or `vmsingle` pod.
For [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/VictoriaMetrics-Operator.html) deployments it is required to add:
```yaml
vmbackup:
restore:
onStart: "true"
```
See operator `VMStorage` schema [here](https://docs.victoriametrics.com/operator/api.html#vmstorage) and `VMSingle` [here](https://docs.victoriametrics.com/operator/api.html#vmsinglespec).
2. Enter container running `vmbackupmanager`
2. Use `vmbackupmanager backup list` to get list of available backups: 2. Use `vmbackupmanager backup list` to get list of available backups:
```console ```console
$ /vmbackupmanager-prod backup list $ /vmbackupmanager-prod backup list
@ -287,6 +295,33 @@ If restore mark doesn't exist at `storageDataPath`(restore wasn't requested) `vm
``` ```
4. Restart pod 4. Restart pod
#### Restore cluster into another cluster
These steps are assuming that [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/VictoriaMetrics-Operator.html) is used to manage `VMCluster`.
Clusters here are referred to as `source` and `destination`.
1. Create a new cluster with access to *source* cluster `vmbackupmanager` storage and same number of storage nodes.
Add the following section in order to enable restore on start (operator `VMStorage` schema can be found [here](https://docs.victoriametrics.com/operator/api.html#vmstorage):
```yaml
vmbackup:
restore:
onStart: "true"
```
Note: it is safe to leave this section in the cluster configuration, since it will be ignored if restore mark doesn't exist.
> Important! Use different `-dst` for *destination* cluster to avoid overwriting backup data of the *source* cluster.
2. Enter container running `vmbackupmanager` in *source* cluster
2. Use `vmbackupmanager backup list` to get list of available backups:
```console
$ /vmbackupmanager-prod backup list
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
```
3. Use `vmbackupmanager restore create` to create restore mark at each pod of the *destination* cluster.
Each pod in *destination* cluster should be restored from backup of respective pod in *source* cluster.
For example: `vmstorage-source-0` in *source* cluster should be restored from `vmstorage-destination-0` in *destination* cluster.
```console
$ /vmbackupmanager-prod restore create s3://source_cluster/vmstorage-source-0/daily/2022-10-06
```
## Configuration ## Configuration
### Flags ### Flags

View file

@ -935,7 +935,7 @@ func TestExecSuccess(t *testing.T) {
))` ))`
r := netstorage.Result{ r := netstorage.Result{
MetricName: metricNameExpected, MetricName: metricNameExpected,
Values: []float64{nan, nan, nan, 1, nan, nan}, Values: []float64{nan, nan, 1, 1, nan, nan},
Timestamps: timestampsExpected, Timestamps: timestampsExpected,
} }
resultExpected := []netstorage.Result{r} resultExpected := []netstorage.Result{r}
@ -6692,7 +6692,7 @@ func TestExecSuccess(t *testing.T) {
q := `rate((2000-time())[100s])` q := `rate((2000-time())[100s])`
r := netstorage.Result{ r := netstorage.Result{
MetricName: metricNameExpected, MetricName: metricNameExpected,
Values: []float64{5.5, 4.5, 3.5, 2.5, 1.5, 0.5}, Values: []float64{5, 4, 3, 2, 1, 0},
Timestamps: timestampsExpected, Timestamps: timestampsExpected,
} }
resultExpected := []netstorage.Result{r} resultExpected := []netstorage.Result{r}
@ -6703,7 +6703,7 @@ func TestExecSuccess(t *testing.T) {
q := `rate((2000-time())[100s:])` q := `rate((2000-time())[100s:])`
r := netstorage.Result{ r := netstorage.Result{
MetricName: metricNameExpected, MetricName: metricNameExpected,
Values: []float64{5.5, 4.5, 3.5, 2.5, 1.5, 0.5}, Values: []float64{5, 4, 3, 2, 1, 0},
Timestamps: timestampsExpected, Timestamps: timestampsExpected,
} }
resultExpected := []netstorage.Result{r} resultExpected := []netstorage.Result{r}
@ -6714,7 +6714,7 @@ func TestExecSuccess(t *testing.T) {
q := `rate((2000-time())[100s:100s])` q := `rate((2000-time())[100s:100s])`
r := netstorage.Result{ r := netstorage.Result{
MetricName: metricNameExpected, MetricName: metricNameExpected,
Values: []float64{0, 0, 6.5, 4.5, 2.5, 0.5}, Values: []float64{0, 0, 6, 4, 2, 0},
Timestamps: timestampsExpected, Timestamps: timestampsExpected,
} }
resultExpected := []netstorage.Result{r} resultExpected := []netstorage.Result{r}
@ -6725,7 +6725,7 @@ func TestExecSuccess(t *testing.T) {
q := `rate((2000-time())[100s:100s] offset 100s)` q := `rate((2000-time())[100s:100s] offset 100s)`
r := netstorage.Result{ r := netstorage.Result{
MetricName: metricNameExpected, MetricName: metricNameExpected,
Values: []float64{0, 0, 3.5, 5.5, 3.5, 1.5}, Values: []float64{0, 0, 7, 5, 3, 1},
Timestamps: timestampsExpected, Timestamps: timestampsExpected,
} }
resultExpected := []netstorage.Result{r} resultExpected := []netstorage.Result{r}
@ -6736,7 +6736,7 @@ func TestExecSuccess(t *testing.T) {
q := `rate((2000-time())[100s:100s] offset 100s)[:] offset 100s` q := `rate((2000-time())[100s:100s] offset 100s)[:] offset 100s`
r := netstorage.Result{ r := netstorage.Result{
MetricName: metricNameExpected, MetricName: metricNameExpected,
Values: []float64{0, 0, 0, 3.5, 5.5, 3.5}, Values: []float64{0, 0, 0, 7, 5, 3},
Timestamps: timestampsExpected, Timestamps: timestampsExpected,
} }
resultExpected := []netstorage.Result{r} resultExpected := []netstorage.Result{r}

View file

@ -145,11 +145,11 @@ var rollupAggrFuncs = map[string]rollupFunc{
"zscore_over_time": rollupZScoreOverTime, "zscore_over_time": rollupZScoreOverTime,
} }
// VictoriaMetrics can increase lookbehind window in square brackets for these functions // VictoriaMetrics can extends lookbehind window for these functions
// if the given window doesn't contain enough samples for calculations. // in order to make sure it contains enough points for returning non-empty results.
// //
// This is needed in order to return the expected non-empty graphs when zooming in the graph in Grafana, // This is needed for returning the expected non-empty graphs when zooming in the graph in Grafana,
// which is built with `func_name(metric[$__interval])` query. // which is built with `func_name(metric)` query.
var rollupFuncsCanAdjustWindow = map[string]bool{ var rollupFuncsCanAdjustWindow = map[string]bool{
"default_rollup": true, "default_rollup": true,
"deriv": true, "deriv": true,
@ -584,15 +584,23 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
window := rc.Window window := rc.Window
if window <= 0 { if window <= 0 {
window = rc.Step window = rc.Step
if rc.MayAdjustWindow && window < maxPrevInterval {
// Adjust lookbehind window only if it isn't set explicilty, e.g. rate(foo).
// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
// when the window doesn't cover at least two raw samples (this is what most users expect).
//
// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
//
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
window = maxPrevInterval
}
if rc.isDefaultRollup && rc.LookbackDelta > 0 && window > rc.LookbackDelta { if rc.isDefaultRollup && rc.LookbackDelta > 0 && window > rc.LookbackDelta {
// Implicit window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval // Implicit window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784 // according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
window = rc.LookbackDelta window = rc.LookbackDelta
} }
} }
if rc.MayAdjustWindow && window < maxPrevInterval {
window = maxPrevInterval
}
rfa := getRollupFuncArg() rfa := getRollupFuncArg()
rfa.idx = 0 rfa.idx = 0
rfa.window = window rfa.window = window

View file

@ -1,12 +1,12 @@
{ {
"files": { "files": {
"main.css": "./static/css/main.fdc77f08.css", "main.css": "./static/css/main.fdc77f08.css",
"main.js": "./static/js/main.2d332988.js", "main.js": "./static/js/main.ca04fac1.js",
"static/js/27.c1ccfd29.chunk.js": "./static/js/27.c1ccfd29.chunk.js", "static/js/27.c1ccfd29.chunk.js": "./static/js/27.c1ccfd29.chunk.js",
"index.html": "./index.html" "index.html": "./index.html"
}, },
"entrypoints": [ "entrypoints": [
"static/css/main.fdc77f08.css", "static/css/main.fdc77f08.css",
"static/js/main.2d332988.js" "static/js/main.ca04fac1.js"
] ]
} }

View file

@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono&family=Lato:wght@300;400;700&display=swap" rel="stylesheet"><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.2d332988.js"></script><link href="./static/css/main.fdc77f08.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html> <!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono&family=Lato:wght@300;400;700&display=swap" rel="stylesheet"><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.ca04fac1.js"></script><link href="./static/css/main.fdc77f08.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

View file

@ -37,6 +37,7 @@ interface FetchDataParams {
query: string[], query: string[],
stateSeriesLimits: SeriesLimits, stateSeriesLimits: SeriesLimits,
showAllSeries?: boolean, showAllSeries?: boolean,
hideQuery?: number[]
} }
export const useFetchQuery = ({ export const useFetchQuery = ({
@ -75,6 +76,7 @@ export const useFetchQuery = ({
query, query,
stateSeriesLimits, stateSeriesLimits,
showAllSeries, showAllSeries,
hideQuery,
}: FetchDataParams) => { }: FetchDataParams) => {
const controller = new AbortController(); const controller = new AbortController();
setFetchQueue([...fetchQueue, controller]); setFetchQueue([...fetchQueue, controller]);
@ -87,6 +89,13 @@ export const useFetchQuery = ({
let totalLength = 0; let totalLength = 0;
for await (const url of fetchUrl) { for await (const url of fetchUrl) {
const isHideQuery = hideQuery?.includes(counter - 1);
if (isHideQuery) {
counter++;
continue;
}
const response = await fetch(url, { signal: controller.signal }); const response = await fetch(url, { signal: controller.signal });
const resp = await response.json(); const resp = await response.json();
@ -126,12 +135,6 @@ export const useFetchQuery = ({
const throttledFetchData = useCallback(debounce(fetchData, 800), []); const throttledFetchData = useCallback(debounce(fetchData, 800), []);
const filterExpr = (q: string, i: number) => {
const byQuery = q.trim();
const byHideQuery = hideQuery ? !hideQuery.includes(i) : true;
return byQuery && byHideQuery;
};
const fetchUrl = useMemo(() => { const fetchUrl = useMemo(() => {
const expr = predefinedQuery ?? query; const expr = predefinedQuery ?? query;
const displayChart = (display || displayType) === "chart"; const displayChart = (display || displayType) === "chart";
@ -143,7 +146,7 @@ export const useFetchQuery = ({
} else if (isValidHttpUrl(serverUrl)) { } else if (isValidHttpUrl(serverUrl)) {
const updatedPeriod = { ...period }; const updatedPeriod = { ...period };
updatedPeriod.step = customStep; updatedPeriod.step = customStep;
return expr.filter(filterExpr).map(q => displayChart return expr.map(q => displayChart
? getQueryRangeUrl(serverUrl, q, updatedPeriod, nocache, isTracingEnabled) ? getQueryRangeUrl(serverUrl, q, updatedPeriod, nocache, isTracingEnabled)
: getQueryUrl(serverUrl, q, updatedPeriod, isTracingEnabled)); : getQueryUrl(serverUrl, q, updatedPeriod, isTracingEnabled));
} else { } else {
@ -163,6 +166,7 @@ export const useFetchQuery = ({
query: expr, query: expr,
stateSeriesLimits, stateSeriesLimits,
showAllSeries, showAllSeries,
hideQuery,
}); });
}, [fetchUrl, visible, stateSeriesLimits, showAllSeries]); }, [fetchUrl, visible, stateSeriesLimits, showAllSeries]);

View file

@ -113,5 +113,6 @@ See also [case studies](https://docs.victoriametrics.com/CaseStudies.html).
* [OSA Con 2022: Specifics of data analysis in Time Series Databases](https://www.youtube.com/watch?v=_zORxrgLtec) * [OSA Con 2022: Specifics of data analysis in Time Series Databases](https://www.youtube.com/watch?v=_zORxrgLtec)
* [OSMC 2022. VictoriaMetrics: scaling to 100 million metrics per second](https://www.youtube.com/watch?v=xfed9_Q0_qU). See also [slides](https://www.slideshare.net/NETWAYS/osmc-2022-victoriametrics-scaling-to-100-million-metrics-per-second-by-aliaksandr-valialkin) * [OSMC 2022. VictoriaMetrics: scaling to 100 million metrics per second](https://www.youtube.com/watch?v=xfed9_Q0_qU). See also [slides](https://www.slideshare.net/NETWAYS/osmc-2022-victoriametrics-scaling-to-100-million-metrics-per-second-by-aliaksandr-valialkin)
* [CNCF Paris Meetup 2022-09-15 - VictoriaMetrics - The cost of scale in Prometheus ecosystem](https://www.youtube.com/watch?v=gcZYHpri2Hw). See also [slides](https://docs.google.com/presentation/d/1jhZuKnAXi15M-mdBP5a4ZAiyrMeHhYmzO8xcZ6pMyLc/edit?usp=sharing) * [CNCF Paris Meetup 2022-09-15 - VictoriaMetrics - The cost of scale in Prometheus ecosystem](https://www.youtube.com/watch?v=gcZYHpri2Hw). See also [slides](https://docs.google.com/presentation/d/1jhZuKnAXi15M-mdBP5a4ZAiyrMeHhYmzO8xcZ6pMyLc/edit?usp=sharing)
* [VictoriaMetrics Meetup December 2022](https://www.youtube.com/watch?v=Mesc6JBFNhQ). See also [slides for "VictoriaMetrics 2022: new features" talk](https://docs.google.com/presentation/d/1jI7XZoodmuzLymdu4MToG9onAKQjzCNwMO2NDupyUkQ/edit?usp=sharing).
* [Comparing Thanos to VictoriaMetrics cluster](https://faun.pub/comparing-thanos-to-victoriametrics-cluster-b193bea1683) * [Comparing Thanos to VictoriaMetrics cluster](https://faun.pub/comparing-thanos-to-victoriametrics-cluster-b193bea1683)
* [Evaluation performance and correctness: VictoriaMetrics response](https://valyala.medium.com/evaluating-performance-and-correctness-victoriametrics-response-e27315627e87) * [Evaluation performance and correctness: VictoriaMetrics response](https://valyala.medium.com/evaluating-performance-and-correctness-victoriametrics-response-e27315627e87)

View file

@ -16,6 +16,19 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip ## tip
## [v1.85.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.3)
Released at 2022-12-20
**Update note 1:** This and newer releases of VictoriaMetrics may return gaps for `rate(m[d])` queries on short time ranges if `[d]` lookbehind window is set expliticly. For example, `rate(http_requests_total[$__interval])`. This reduces confusion level when the user expects the needed results from the query with explicitly set lookbehind window. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483). The previous gap filling behaviour can be restored by removing explicit lookbehind window `[d]` from the query, e.g. by substituting the `rate(m[d])` with `rate(m)`. See [these docs](https://docs.victoriametrics.com/MetricsQL.html#implicit-query-conversions) for details.
* BUGFIX: fix `error when searching for TSIDs by metricIDs in the previous indexdb: EOF` error, which can occur during queries after unclean shutdown of VictoriaMetrics (e.g. via hardware reset, out of memory crash or `kill -9`). The error has been introduced in [v1.85.2](https://docs.victoriametrics.com/CHANGELOG.html#v1852). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3515).
* BUGFIX: [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html): expose proper values for `vm_downsampling_partitions_scheduled` and `vm_downsampling_partitions_scheduled_size_bytes` metrics, which were added at [v1.78.0](https://docs.victoriametrics.com/CHANGELOG.html#v1780). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2612).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): never extend explicitly set lookbehind window for [rate()](https://docs.victoriametrics.com/MetricsQL.html#rate) function. This reduces the level of confusion when the user expects the needed results after explicitly seting the lookbehind window `[d]` in the query `rate(m[d])`. Previously VictoriaMetrics could silently extend the lookbehind window, so it covers at least two raw samples. Now this behavior works only if the lookbehind window in square brackets isn't set explicitly, e.g. in the case of `rate(m)`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483) for details.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): respect `-usePromCompatibleNaming` flag if no relabeling or extra labels were set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3511) for details.
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the wrong legend when queries are hidden. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3512).
## [v1.85.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.2) ## [v1.85.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.2)
Released at 2022-12-19 Released at 2022-12-19
@ -23,11 +36,13 @@ Released at 2022-12-19
* FEATURE: support overriding of `-search.latencyOffset` value via URL param `latency_offset` when performing requests to [/api/v1/query](https://docs.victoriametrics.com/keyConcepts.html#instant-query) and [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3481). * FEATURE: support overriding of `-search.latencyOffset` value via URL param `latency_offset` when performing requests to [/api/v1/query](https://docs.victoriametrics.com/keyConcepts.html#instant-query) and [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3481).
* FEATURE: allow changing field names in JSON logs if VictoriaMetrics components are started with `-loggerFormat=json` command-line flags. The field names can be changed with the `-loggerJSONFields` command-line flag. For example `-loggerJSONFields=ts:timestamp,msg:message` would rename `ts` and `msg` fields on the output JSON to `timestamp` and `message` fields. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2348). Thanks to @michal-kralik for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3488). * FEATURE: allow changing field names in JSON logs if VictoriaMetrics components are started with `-loggerFormat=json` command-line flags. The field names can be changed with the `-loggerJSONFields` command-line flag. For example `-loggerJSONFields=ts:timestamp,msg:message` would rename `ts` and `msg` fields on the output JSON to `timestamp` and `message` fields. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2348). Thanks to @michal-kralik for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3488).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `__meta_consul_tag_<tagname>` and `__meta_consul_tagpresent_<tagname>` labels for targets discovered via [consul_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs). This simplifies converting [Consul service tags](https://developer.hashicorp.com/consul/docs/discovery/services#service-definition) to target labels with a simple [relabeling rule](https://docs.victoriametrics.com/vmagent.html#relabeling): * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `__meta_consul_tag_<tagname>` and `__meta_consul_tagpresent_<tagname>` labels for targets discovered via [consul_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs). This simplifies converting [Consul service tags](https://developer.hashicorp.com/consul/docs/discovery/services#service-definition) to target labels with a simple [relabeling rule](https://docs.victoriametrics.com/vmagent.html#relabeling):
```yml
- action: labelmap ```yml
regex: __meta_consul_tag_(.+) - action: labelmap
``` regex: __meta_consul_tag_(.+)
This resolves [this StackOverflow question](https://stackoverflow.com/questions/44339461/relabeling-in-prometheus). ```
This resolves [this StackOverflow question](https://stackoverflow.com/questions/44339461/relabeling-in-prometheus).
* BUGFIX: properly return query results for time series, which stop receiving new samples after the rotation of `indexdb`. Previously such time series could be missing in query results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502). The issue has been introduced in [v1.83.0](https://docs.victoriametrics.com/CHANGELOG.html#v1830). * BUGFIX: properly return query results for time series, which stop receiving new samples after the rotation of `indexdb`. Previously such time series could be missing in query results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502). The issue has been introduced in [v1.83.0](https://docs.victoriametrics.com/CHANGELOG.html#v1830).
* BUGFIX: allow specifying values bigger than 2GiB to the following command-line flag values on 32-bit architectures (`386` and `arm`): `-storage.minFreeDiskSpaceBytes` and `-remoteWrite.maxDiskUsagePerURL`. Previously values bigger than 2GiB were incorrectly truncated on these architectures. * BUGFIX: allow specifying values bigger than 2GiB to the following command-line flag values on 32-bit architectures (`386` and `arm`): `-storage.minFreeDiskSpaceBytes` and `-remoteWrite.maxDiskUsagePerURL`. Previously values bigger than 2GiB were incorrectly truncated on these architectures.

View file

@ -24,7 +24,7 @@ It has integration with VictoriaMetrics `vmbackupmanager` - advanced tools for m
For kubernetes-cluster administrators, it simplifies installation, configuration and management for `VictoriaMetrics` application. The main feature of operator is its ability to delegate the configuration of applications monitoring to the end-users. For kubernetes-cluster administrators, it simplifies installation, configuration and management for `VictoriaMetrics` application. The main feature of operator is its ability to delegate the configuration of applications monitoring to the end-users.
For applications developers, its great possibility for managing observability of applications. You can define metrics scraping and alerting configuration for your application and manage it with an application deployment process. Just define app_deployment.yaml, app_vmpodscrape.yaml and app_vmrule.yaml. That's it, you can apply it to a kubernetes cluster. Check [quick-start](/Operator/quick-start.html) for an example. For applications developers, its great possibility for managing observability of applications. You can define metrics scraping and alerting configuration for your application and manage it with an application deployment process. Just define app_deployment.yaml, app_vmpodscrape.yaml and app_vmrule.yaml. That's it, you can apply it to a kubernetes cluster. Check [quick-start](/operator/quick-start.html) for an example.
## Operator vs helm-chart ## Operator vs helm-chart

View file

@ -162,7 +162,7 @@ The result on the GCS bucket. We see only 3 daily backups:
* GET `/api/v1/backups` - returns list of backups in remote storage. * GET `/api/v1/backups` - returns list of backups in remote storage.
Example output: Example output:
```json ```json
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"] [{"name":"daily/2022-11-30","size_bytes":26664689,"size":"25.429Mi"},{"name":"daily/2022-12-01","size_bytes":40160965,"size":"38.300Mi"},{"name":"hourly/2022-11-30:12","size_bytes":5846529,"size":"5.576Mi"},{"name":"hourly/2022-11-30:13","size_bytes":17651847,"size":"16.834Mi"},{"name":"hourly/2022-11-30:13:22","size_bytes":8797831,"size":"8.390Mi"},{"name":"hourly/2022-11-30:14","size_bytes":10680454,"size":"10.186Mi"}]
``` ```
* POST `/api/v1/restore` - saves backup name to restore when [performing restore](#restore-commands). * POST `/api/v1/restore` - saves backup name to restore when [performing restore](#restore-commands).
@ -215,7 +215,7 @@ It can be changed by using flag:
`vmbackupmanager backup list` lists backups in remote storage: `vmbackupmanager backup list` lists backups in remote storage:
```console ```console
$ ./vmbackupmanager backup list $ ./vmbackupmanager backup list
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"] [{"name":"daily/2022-11-30","size_bytes":26664689,"size":"25.429Mi"},{"name":"daily/2022-12-01","size_bytes":40160965,"size":"38.300Mi"},{"name":"hourly/2022-11-30:12","size_bytes":5846529,"size":"5.576Mi"},{"name":"hourly/2022-11-30:13","size_bytes":17651847,"size":"16.834Mi"},{"name":"hourly/2022-11-30:13:22","size_bytes":8797831,"size":"8.390Mi"},{"name":"hourly/2022-11-30:14","size_bytes":10680454,"size":"10.186Mi"}]
``` ```
### Restore commands ### Restore commands
@ -274,7 +274,15 @@ If restore mark doesn't exist at `storageDataPath`(restore wasn't requested) `vm
### How to restore in Kubernetes ### How to restore in Kubernetes
1. Enter container running `vmbackupmanager` 1. Ensure there is an init container with `vmbackupmanager restore` in `vmstorage` or `vmsingle` pod.
For [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/VictoriaMetrics-Operator.html) deployments it is required to add:
```yaml
vmbackup:
restore:
onStart: "true"
```
See operator `VMStorage` schema [here](https://docs.victoriametrics.com/operator/api.html#vmstorage) and `VMSingle` [here](https://docs.victoriametrics.com/operator/api.html#vmsinglespec).
2. Enter container running `vmbackupmanager`
2. Use `vmbackupmanager backup list` to get list of available backups: 2. Use `vmbackupmanager backup list` to get list of available backups:
```console ```console
$ /vmbackupmanager-prod backup list $ /vmbackupmanager-prod backup list
@ -291,6 +299,33 @@ If restore mark doesn't exist at `storageDataPath`(restore wasn't requested) `vm
``` ```
4. Restart pod 4. Restart pod
#### Restore cluster into another cluster
These steps are assuming that [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/VictoriaMetrics-Operator.html) is used to manage `VMCluster`.
Clusters here are referred to as `source` and `destination`.
1. Create a new cluster with access to *source* cluster `vmbackupmanager` storage and same number of storage nodes.
Add the following section in order to enable restore on start (operator `VMStorage` schema can be found [here](https://docs.victoriametrics.com/operator/api.html#vmstorage):
```yaml
vmbackup:
restore:
onStart: "true"
```
Note: it is safe to leave this section in the cluster configuration, since it will be ignored if restore mark doesn't exist.
> Important! Use different `-dst` for *destination* cluster to avoid overwriting backup data of the *source* cluster.
2. Enter container running `vmbackupmanager` in *source* cluster
2. Use `vmbackupmanager backup list` to get list of available backups:
```console
$ /vmbackupmanager-prod backup list
["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
```
3. Use `vmbackupmanager restore create` to create restore mark at each pod of the *destination* cluster.
Each pod in *destination* cluster should be restored from backup of respective pod in *source* cluster.
For example: `vmstorage-source-0` in *source* cluster should be restored from `vmstorage-destination-0` in *destination* cluster.
```console
$ /vmbackupmanager-prod restore create s3://source_cluster/vmstorage-source-0/daily/2022-10-06
```
## Configuration ## Configuration
### Flags ### Flags

View file

@ -11,6 +11,12 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsnil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsnil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/metrics"
)
var (
bytesUploadedTotal = uint64(0)
bytesUploadedTotalMetric = metrics.NewCounter(`vm_backups_uploaded_bytes_total`)
) )
// Backup performs backup according to the provided settings. // Backup performs backup according to the provided settings.
@ -163,6 +169,8 @@ func runBackup(src *fslocal.FS, dst common.RemoteFS, origin common.OriginFS, con
n := atomic.LoadUint64(&bytesUploaded) n := atomic.LoadUint64(&bytesUploaded)
logger.Infof("uploaded %d out of %d bytes from src %s to dst %s in %s", n, uploadSize, src, dst, elapsed) logger.Infof("uploaded %d out of %d bytes from src %s to dst %s in %s", n, uploadSize, src, dst, elapsed)
}) })
atomic.AddUint64(&bytesUploadedTotal, bytesUploaded)
bytesUploadedTotalMetric.Set(bytesUploadedTotal)
if err != nil { if err != nil {
return err return err
} }

19
lib/formatutil/human.go Normal file
View file

@ -0,0 +1,19 @@
package formatutil
import (
"fmt"
"math"
)
// HumanizeBytes returns human-readable representation of size in bytes with 1024 base.
func HumanizeBytes(size float64) string {
prefix := ""
for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
if math.Abs(size) < 1024 {
break
}
prefix = p
size /= 1024
}
return fmt.Sprintf("%.4g%s", size, prefix)
}

View file

@ -1905,11 +1905,13 @@ func (db *indexDB) getTSIDsFromMetricIDs(qt *querytracer.Tracer, metricIDs []uin
// Cannot find TSID for the given metricID. // Cannot find TSID for the given metricID.
// This may be the case on incomplete indexDB // This may be the case on incomplete indexDB
// due to snapshot or due to unflushed entries. // due to snapshot or due to unflushed entries.
// Just increment errors counter and skip it. // Just increment errors counter and skip it for now.
atomic.AddUint64(&is.db.missingTSIDsForMetricID, 1) atomic.AddUint64(&is.db.missingTSIDsForMetricID, 1)
err = nil
continue continue
} }
err = fmt.Errorf("cannot find tsid for metricID=%d: %w", metricID, err) err = fmt.Errorf("cannot find tsid for metricID=%d: %w", metricID, err)
return
} }
is.db.putToMetricIDCache(metricID, tsid) is.db.putToMetricIDCache(metricID, tsid)
i++ i++