diff --git a/app/vmagent/remotewrite/pendingseries_test.go b/app/vmagent/remotewrite/pendingseries_test.go index 487fdab49..6e949a94f 100644 --- a/app/vmagent/remotewrite/pendingseries_test.go +++ b/app/vmagent/remotewrite/pendingseries_test.go @@ -34,7 +34,7 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte return true } if !tryPushWriteRequest(wr, pushBlock, isVMRemoteWrite) { - t.Fatalf("cannot push data to to remote storage") + t.Fatalf("cannot push data to remote storage") } if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc { t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%", diff --git a/app/vmagent/remotewrite/streamaggr.go b/app/vmagent/remotewrite/streamaggr.go index 0c357d298..454d14bf8 100644 --- a/app/vmagent/remotewrite/streamaggr.go +++ b/app/vmagent/remotewrite/streamaggr.go @@ -143,7 +143,7 @@ func getStreamAggrOpts(idx int) (string, *streamaggr.Options) { if len(*streamAggrConfig) == 0 { return "", &opts } - return (*streamAggrConfig)[idx], &opts + return streamAggrConfig.GetOptionalArg(idx), &opts } func newStreamAggrConfigWithOpts(pushFunc streamaggr.PushFunc, path string, opts *streamaggr.Options) (*streamaggr.Aggregators, error) { diff --git a/app/vmctl/opentsdb/opentsdb.go b/app/vmctl/opentsdb/opentsdb.go index 4efc18c6c..b75e20b5a 100644 --- a/app/vmctl/opentsdb/opentsdb.go +++ b/app/vmctl/opentsdb/opentsdb.go @@ -26,7 +26,7 @@ type Retention struct { FirstOrder string SecondOrder string AggTime string - // The actual ranges will will attempt to query (as offsets from now) + // The actual ranges will attempt to query (as offsets from now) QueryRanges []TimeRange } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 9c44d1e5d..726afa0db 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -70,6 +70,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): reduce the default value for `-maxLabelValueLen` command-line flag from `16KiB` to `1KiB`. This should prevent from issues like [this one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6176) when time series with too long labels are ingested into VictoriaMetrics. * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly release memory used for metrics during config reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6247). * BUGFIX: [dashboards](https://grafana.com/orgs/victoriametrics): fix `AnnotationQueryRunner` error in Grafana when executing annotations query against Prometheus backend. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6309) for details. +* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): filter deleted label names and values from [`/api/v1/labels`](https://docs.victoriametrics.com/url-examples/#apiv1labels) and [`/api/v1/label/.../values`](https://docs.victoriametrics.com/url-examples/#apiv1labelvalues) responses when `match[]` filter matches small number of time series. The issue was introduced [v1.81.0](https://docs.victoriametrics.com/changelog_2022/#v1810). * DEPRECATION: [vmagent](https://docs.victoriametrics.com/vmagent/): removed deprecated `-remoteWrite.multitenantURL` flag from vmagent. This flag was deprecated since [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0). Use `-enableMultitenantHandlers` instead, as it is easier to use and combine with [multitenant URL at vminsert](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels). See these [docs for details](https://docs.victoriametrics.com/vmagent.html#multitenancy). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 3c6e82da3..158b0725f 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -328,7 +328,7 @@ See also [increases_over_time](#increases_over_time). `default_rollup(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples) value on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). -Compared to [last_over_time](last_over_time) it accounts for [staleness markers](https://docs.victoriametrics.com/vmagent/#prometheus-staleness-markers) to detect stale series. +Compared to [last_over_time](#last_over_time) it accounts for [staleness markers](https://docs.victoriametrics.com/vmagent/#prometheus-staleness-markers) to detect stale series. If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query), diff --git a/docs/anomaly-detection/FAQ.md b/docs/anomaly-detection/FAQ.md index bed5bbc18..67ddfc930 100644 --- a/docs/anomaly-detection/FAQ.md +++ b/docs/anomaly-detection/FAQ.md @@ -27,6 +27,17 @@ The decision to set the changepoint at `1.0` is made to ensure consistency acros > Note: `anomaly_score` is a metric itself, which preserves all labels found in input data and (optionally) appends [custom labels, specified in writer](/anomaly-detection/components/writer.html#metrics-formatting) - follow the link for detailed output example. +## How is anomaly score calculated? +For most of the [univariate models](/anomaly-detection/components/models/#univariate-models) that can generate `yhat`, `yhat_lower`, and `yhat_upper` time series in [their output](/anomaly-detection/components/models/#vmanomaly-output) (such as [Prophet](/anomaly-detection/components/models/#prophet) or [Z-score](/anomaly-detection/components/models/#z-score)), the anomaly score is calculated as follows: +- If `yhat` (expected series behavior) equals `y` (actual value observed), then the anomaly score is 0. +- If `y` (actual value observed) falls within the `[yhat_lower, yhat_upper]` confidence interval, the anomaly score will gradually approach 1, the closer `y` is to the boundary. +- If `y` (actual value observed) strictly exceeds the `[yhat_lower, yhat_upper]` interval, the anomaly score will be greater than 1, increasing as the margin between the actual value and the expected range grows. + +Please see example graph illustrating this logic below: + +anomaly-score-calculation-example + + ## How does vmanomaly work? `vmanomaly` applies built-in (or custom) [anomaly detection algorithms](/anomaly-detection/components/models.html), specified in a config file. Although a single config file supports one model, running multiple instances of `vmanomaly` with different configs is possible and encouraged for parallel processing or better support for your use case (i.e. simpler model for simple metrics, more sophisticated one for metrics with trends and seasonalities). @@ -45,7 +56,9 @@ Respective config is defined in a [`reader`](/anomaly-detection/components/reade `vmanomaly` models generate [metrics](/anomaly-detection/components/models.html#vmanomaly-output) like `anomaly_score`, `yhat`, `yhat_lower`, `yhat_upper`, and `y`. These metrics provide a comprehensive view of the detected anomalies. The service also produces [health check metrics](/anomaly-detection/components/monitoring.html#metrics-generated-by-vmanomaly) for monitoring its performance. ## Choosing the right model for vmanomaly -Selecting the best model for `vmanomaly` depends on the data's nature and the types of anomalies to detect. For instance, [Z-score](anomaly-detection/components/models.html#z-score) is suitable for data without trends or seasonality, while more complex patterns might require models like [Prophet](anomaly-detection/components/models.html#prophet). +Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect. For instance, [Z-score](anomaly-detection/components/models.html#z-score) is suitable for data without trends or seasonality, while more complex patterns might require models like [Prophet](anomaly-detection/components/models.html#prophet). + +Also, starting from [v1.12.0](/anomaly-detection/changelog/#v1120) it's possible to auto-tune the most important params of selected model class, find [the details here](https://docs.victoriametrics.com/anomaly-detection/components/models/#autotuned). Please refer to [respective blogpost on anomaly types and alerting heuristics](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/) for more details. @@ -57,8 +70,86 @@ While `vmanomaly` detects anomalies and produces scores, it *does not directly g ## Preventing alert fatigue Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](/vmalert.html) can use [MetricsQL](https://docs.victoriametrics.com/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive). +## How to backtest particular configuration on historical data? +Starting from [v1.7.2](/anomaly-detection/changelog/#v172) you can produce (and write back to VictoriaMetrics TSDB) anomaly scores for historical (backtesting) period, using `BacktestingScheduler` [component](/anomaly-detection/components/scheduler/#backtesting-scheduler) to imitate consecutive "production runs" of `PeriodicScheduler` [component](/anomaly-detection/components/scheduler/#periodic-scheduler). Please find an example config below: + +```yaml +schedulers: + scheduler_alias: + class: "scheduler.backtesting.BacktestingScheduler" + # define historical period to backtest on + # should be bigger than at least (fit_window + fit_every) time range + from_iso: '2024-01-01T00:00:00Z' + to_iso: '2024-01-15T00:00:00Z' + # copy these from your PeriodicScheduler args + fit_window: 'P14D' + fit_every: 'PT1H' + +models: + model_alias1: + # ... + schedulers: ['scheduler_alias'] # if ommited, all the defined schedulers will be attached + queries: ['query_alias1'] # if ommited, all the defined queries will be attached + # https://docs.victoriametrics.com/anomaly-detection/components/models/#provide-series + provide_series: ['anomaly_score'] + # ... other models + +reader: + datasource_url: 'some_url_to_read_data_from' + queries: + query_alias1: 'some_metricsql_query' + sampling_frequency: '1m' # change to whatever you need in data granularity + # other params if needed + # https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader + +writer: + datasource_url: 'some_url_to_write_produced_data_to' + # other params if needed + # https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer + +# optional monitoring section if needed +# https://docs.victoriametrics.com/anomaly-detection/components/monitoring/ +``` + +Configuration above will produce N intervals of full length (`fit_window`=14d + `fit_every`=1h) until `to_iso` timestamp is reached to run N consecutive `fit` calls to train models; Then these models will be used to produce `M = [fit_every / sampling_frequency]` infer datapoints for `fit_every` range at the end of each such interval, imitating M consecutive calls of `infer_every` in `PeriodicScheduler` [config](/anomaly-detection/components/scheduler/#periodic-scheduler). These datapoints then will be written back to VictoriaMetrics TSDB, defined in `writer` [section](/anomaly-detection/components/writer/#vm-writer) for further visualization (i.e. in VMUI or Grafana) + ## Resource consumption of vmanomaly `vmanomaly` itself is a lightweight service, resource usage is primarily dependent on [scheduling](/anomaly-detection/components/scheduler.html) (how often and on what data to fit/infer your models), [# and size of timeseries returned by your queries](/anomaly-detection/components/reader.html#vm-reader), and the complexity of the employed [models](anomaly-detection/components/models.html). Its resource usage is directly related to these factors, making it adaptable to various operational scales. ## Scaling vmanomaly -`vmanomaly` can be scaled horizontally by launching multiple independent instances, each with its own [MetricsQL](https://docs.victoriametrics.com/metricsql/) queries and [configurations](/anomaly-detection/components/). This flexibility allows it to handle varying data volumes and throughput demands efficiently. +> **Note:** As of latest release we don't support cluster or auto-scaled version yet (though, it's in our roadmap for - better backends, more parallelization, etc.), so proposed workarounds should be addressed manually. + +`vmanomaly` can be scaled horizontally by launching multiple independent instances, each with its own [MetricsQL](https://docs.victoriametrics.com/metricsql/) queries and [configurations](/anomaly-detection/components/): + +- By splitting **queries**, [defined in reader section](/anomaly-detection/components/reader/?highlight=queries#vm-reader) and spawn separate service around it. Also in case you have *only 1 query returning huge amount of timeseries*, you can further split it by applying MetricsQL filters, i.e. using "extra_filters" [param in reader](/anomaly-detection/components/reader/?highlight=extra_filters#vm-reader) + +- or **models** (in case you decide to run several models for each timeseries received i.e. for averaging anomaly scores in your alerting rules of `vmalert` or using a vote approach to reduce false positives) - see `queries` arg in [model config](/anomaly-detection/components/models/#queries) + +- or **schedulers** (in case you want the same models to be trained under several schedules) - see `schedulers` arg [model section](/anomaly-detection/components/models/#schedulers) and `scheduler` [component itself](/anomaly-detection/components/scheduler/) + + +```yaml +# config file #1, for 1st vmanomaly instance +# ... +reader: + # ... + queries: + extra_big_query: metricsql_expression_returning_too_many_timeseries + extra_filters: + # suppose you have a label `region` with values to deterministically define such subsets + - '{region="region_name_1"}' + # ... +``` + +```yaml +# config file #2, for 2nd vmanomaly instance +# ... +reader: + # ... + queries: + extra_big_query: metricsql_expression_returning_too_many_timeseries + extra_filters: + # suppose you have a label `region` with values to deterministically define such subsets + - '{region="region_name_2"}' + # ... +``` diff --git a/docs/anomaly-detection/README.md b/docs/anomaly-detection/README.md index bda519909..682e69ddd 100644 --- a/docs/anomaly-detection/README.md +++ b/docs/anomaly-detection/README.md @@ -49,7 +49,7 @@ Got questions about VictoriaMetrics Anomaly Detection? Chances are, we've got th Dive into [our FAQ section](/anomaly-detection/FAQ) to find responses to common questions. ## Get in Touch -We're eager to connect with you and tailor our solutions to your specific needs. Here's how you can engage with us: +We are eager to connect with you and adapt our solutions to your specific needs. Here's how you can engage with us: * [Book a Demo](https://calendly.com/victoriametrics-anomaly-detection) to discover what our product can do. * Interested in exploring our [Enterprise features](https://victoriametrics.com/products/enterprise), including [Anomaly Detection](https://victoriametrics.com/products/enterprise/anomaly-detection)? [Request your trial license](https://victoriametrics.com/products/enterprise/trial/) today and take the first step towards advanced system observability. diff --git a/docs/managed-victoriametrics/create_deployment_continue.webp b/docs/managed-victoriametrics/create_deployment_continue.webp new file mode 100644 index 000000000..eb0133922 Binary files /dev/null and b/docs/managed-victoriametrics/create_deployment_continue.webp differ diff --git a/docs/managed-victoriametrics/create_deployment_start.webp b/docs/managed-victoriametrics/create_deployment_start.webp index 040bae4de..17cccddfd 100644 Binary files a/docs/managed-victoriametrics/create_deployment_start.webp and b/docs/managed-victoriametrics/create_deployment_start.webp differ diff --git a/docs/managed-victoriametrics/quickstart.md b/docs/managed-victoriametrics/quickstart.md index f8656239b..ef2c488c7 100644 --- a/docs/managed-victoriametrics/quickstart.md +++ b/docs/managed-victoriametrics/quickstart.md @@ -183,11 +183,15 @@ If you forgot your password, it can be restored in the following way: On the [Deployments](https://cloud.victoriametrics.com/deployments?utm_source=website&utm_campaign=docs_quickstart) page you will see a list of your existing deployments and will be able to manage them. -To create a deployment click on the button `Create Deployment` button: +To create a deployment click on the button `Start sending metrics` button: +When you already have at least one deployment you can create a new one by clicking on the `Create deployment` button: + + + On the opened screen, choose parameters of your new deployment: diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md index c29a80d5d..dd9dc82e6 100644 --- a/docs/operator/CHANGELOG.md +++ b/docs/operator/CHANGELOG.md @@ -20,6 +20,7 @@ aliases: - [operator](./README.md): remove finalizer for child objects with non-empty `DeletetionTimestamp`. See this [issue](https://github.com/VictoriaMetrics/operator/issues/953) for details. - [operator](./README.md): skip storageClass check if there is no PVC size change. See this [issue](https://github.com/VictoriaMetrics/operator/issues/957) for details. - [vmauth](./api.md#vmauth): fix url when default http port is changed in targetRef. See this [issue](https://github.com/VictoriaMetrics/operator/issues/960) for details. +- [vmauth](./api.html#vmauth): fix deployment when custom reloader is used. See [this pull request](https://github.com/VictoriaMetrics/operator/pull/964). ## [v0.44.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.44.0) - 9 May 2024 diff --git a/docs/operator/vars.md b/docs/operator/vars.md index b7bedfa5b..1ad6bab10 100644 --- a/docs/operator/vars.md +++ b/docs/operator/vars.md @@ -10,7 +10,7 @@ menu: # Auto Generated vars for package config - updated at Thu May 23 09:10:16 UTC 2024 + updated at Wed May 29 11:18:39 UTC 2024 | varible name | variable default value | variable required | variable description | diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md index f94b42e49..daaa59e4b 100644 --- a/docs/stream-aggregation.md +++ b/docs/stream-aggregation.md @@ -1073,7 +1073,7 @@ This may lead to the following issues: since they ignore the first sample in a new time series. - Unexpected spikes for [total](#total) and [increase](#increase) outputs, since they assume that new time series start from 0. -These issues can be be fixed in the following ways: +These issues can be fixed in the following ways: - By increasing the `interval` option at [stream aggregation config](#stream-aggregation-config), so it covers the expected delays in data ingestion pipelines. diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go index 848f90cb1..ebd8ad2ab 100644 --- a/lib/storage/index_db.go +++ b/lib/storage/index_db.go @@ -747,10 +747,18 @@ func (is *indexSearch) getLabelNamesForMetricIDs(qt *querytracer.Tracer, metricI if len(metricIDs) > 0 { lns["__name__"] = struct{}{} } + + dmis := is.db.s.getDeletedMetricIDs() + checkDeleted := dmis.Len() > 0 + var mn MetricName foundLabelNames := 0 var buf []byte for _, metricID := range metricIDs { + if checkDeleted && dmis.Has(metricID) { + // skip deleted IDs from result + continue + } var ok bool buf, ok = is.searchMetricNameWithCache(buf[:0], metricID) if !ok { @@ -946,10 +954,18 @@ func (is *indexSearch) getLabelValuesForMetricIDs(qt *querytracer.Tracer, lvs ma if labelName == "" { labelName = "__name__" } + + dmis := is.db.s.getDeletedMetricIDs() + checkDeleted := dmis.Len() > 0 + var mn MetricName foundLabelValues := 0 var buf []byte for _, metricID := range metricIDs { + if checkDeleted && dmis.Has(metricID) { + // skip deleted IDs from result + continue + } var ok bool buf, ok = is.searchMetricNameWithCache(buf[:0], metricID) if !ok { diff --git a/lib/storage/index_db_test.go b/lib/storage/index_db_test.go index f5ebc9336..127365c3a 100644 --- a/lib/storage/index_db_test.go +++ b/lib/storage/index_db_test.go @@ -1562,30 +1562,34 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { "testMetric", } sort.Strings(labelNames) + + newMN := func(name string, day, metric int) MetricName { + var mn MetricName + mn.MetricGroup = []byte(name) + mn.AddTag( + "constant", + "const", + ) + mn.AddTag( + "day", + fmt.Sprintf("%v", day), + ) + mn.AddTag( + "UniqueId", + fmt.Sprintf("%v", metric), + ) + mn.AddTag( + "some_unique_id", + fmt.Sprintf("%v", day), + ) + mn.sortTags() + return mn + } for day := 0; day < days; day++ { date := baseDate - uint64(day) var metricIDs uint64set.Set for metric := 0; metric < metricsPerDay; metric++ { - var mn MetricName - mn.MetricGroup = []byte("testMetric") - mn.AddTag( - "constant", - "const", - ) - mn.AddTag( - "day", - fmt.Sprintf("%v", day), - ) - mn.AddTag( - "UniqueId", - fmt.Sprintf("%v", metric), - ) - mn.AddTag( - "some_unique_id", - fmt.Sprintf("%v", day), - ) - mn.sortTags() - + mn := newMN("testMetric", day, metric) metricNameBuf = mn.Marshal(metricNameBuf[:0]) var genTSID generationTSID if !is.getTSIDByMetricName(&genTSID, metricNameBuf, date) { @@ -1626,6 +1630,29 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { } db.putIndexSearch(is2) + // add a metric that will be deleted shortly + is3 := db.getIndexSearch(noDeadline) + day := days + date := baseDate - uint64(day) + mn := newMN("deletedMetric", day, 999) + mn.AddTag( + "labelToDelete", + fmt.Sprintf("%v", day), + ) + mn.sortTags() + metricNameBuf = mn.Marshal(metricNameBuf[:0]) + var genTSID generationTSID + if !is3.getTSIDByMetricName(&genTSID, metricNameBuf, date) { + generateTSID(&genTSID.TSID, &mn) + createAllIndexesForMetricName(is3, &mn, &genTSID.TSID, date) + } + // delete the added metric. It is expected it won't be returned during searches + deletedSet := &uint64set.Set{} + deletedSet.Add(genTSID.TSID.MetricID) + s.setDeletedMetricIDs(deletedSet) + db.putIndexSearch(is3) + s.DebugFlush() + // Check SearchLabelNamesWithFiltersOnTimeRange with the specified time range. tr := TimeRange{ MinTimestamp: int64(now) - msecPerDay,