github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: optimize /api/v1/labels and /api/v1/label/.../values when match[] contains metric name

Browse source 
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2978
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5055
This commit is contained in:
Aliaksandr Valialkin 2024-03-12 01:43:27 +02:00
parent 1ed6df7901
commit d1d2771bee
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
4 changed files with 83 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/CHANGELOG.md
View file

@ -46,6 +46,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
* FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): expose `vm_streamaggr_flush_timeouts_total` and `vm_streamaggr_dedup_flush_timeouts_total` [counters](https://docs.victoriametrics.com/keyconcepts/#counter) at [`/metrics` page](https://docs.victoriametrics.com/#monitoring), which can be used for detecting flush timeouts for stream aggregation states. Expose also `vm_streamaggr_flush_duration_seconds` and `vm_streamaggr_dedup_flush_duration_seconds` [histograms](https://docs.victoriametrics.com/keyconcepts/#histogram) for monitoring the real flush durations of stream aggregation states.
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): improve trace display for better visual separation of branches. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5926).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): use the provided `-remoteWrite.tlsServerName` as `Host` header in requests to `-remoteWrite.url`. This allows sending data to https remote storage by IP address instead of hostname. Thanks to @minor-fixes for initial idea and [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5802).
* FEATURE: optimize [`/api/v1/labels`](https://docs.victoriametrics.com/url-examples/#apiv1labels) and [`/api/v1/label/.../values`](https://docs.victoriametrics.com/url-examples/#apiv1labelvalues) when `match[]` filters contains metric name. For example, `/api/v1/label/instance/values?match[]=up` now works much faster than before. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5055).
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for [native protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-from-victoriametrics). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5824).
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for VictoriaMetrics destination specified via `--vm-*` cmd-line flags used in [InfluxDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-influxdb-1x), [Remote Read protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-by-remote-read-protocol), [OpenTSDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-opentsdb), [Prometheus](https://docs.victoriametrics.com/vmctl/#migrating-data-from-prometheus) and [Promscale](https://docs.victoriametrics.com/vmctl/#migrating-data-from-promscale) migration modes.

40
lib/storage/index_db.go
View file

@ -663,6 +663,7 @@ func (is *indexSearch) searchLabelNamesWithFiltersOnDate(qt *querytracer.Tracer,
is.getLabelNamesForMetricIDs(qt, metricIDs, lns, maxLabelNames)
return nil
}
var prevLabelName []byte
ts := &is.ts
kb := &is.kb
@ -674,8 +675,18 @@ func (is *indexSearch) searchLabelNamesWithFiltersOnDate(qt *querytracer.Tracer,
if date == 0 {
nsPrefixExpected = nsPrefixTagToMetricIDs
}
hasCompositeLabelName := false
kb.B = is.marshalCommonPrefixForDate(kb.B[:0], date)
prefix := kb.B
if name := getCommonMetricNameForTagFilterss(tfss); len(name) > 0 {
compositeLabelName := marshalCompositeTagKey(nil, name, nil)
kb.B = marshalTagValue(kb.B, compositeLabelName)
// Drop trailing tagSeparator
kb.B = kb.B[:len(kb.B)-1]
hasCompositeLabelName = true
}
prefix := append([]byte{}, kb.B...)
ts.Seek(prefix)
for len(lns) < maxLabelNames && ts.NextItem() {
if loopsPaceLimiter&paceLimiterFastIterationsMask == 0 {
@ -695,15 +706,15 @@ func (is *indexSearch) searchLabelNamesWithFiltersOnDate(qt *querytracer.Tracer,
continue
}
labelName := mp.Tag.Key
if len(labelName) == 0 {
if len(labelName) == 0 || hasCompositeLabelName {
underscoreNameSeen = true
}
if isArtificialTagKey(labelName) || string(labelName) == string(prevLabelName) {
if (!hasCompositeLabelName && isArtificialTagKey(labelName)) || string(labelName) == string(prevLabelName) {
// Search for the next tag key.
// The last char in kb.B must be tagSeparatorChar.
// Just increment it in order to jump to the next tag key.
kb.B = is.marshalCommonPrefixForDate(kb.B[:0], date)
if len(labelName) > 0 && labelName[0] == compositeTagKeyPrefix {
if !hasCompositeLabelName && len(labelName) > 0 && labelName[0] == compositeTagKeyPrefix {
// skip composite tag entries
kb.B = append(kb.B, compositeTagKeyPrefix)
} else {
@ -713,7 +724,15 @@ func (is *indexSearch) searchLabelNamesWithFiltersOnDate(qt *querytracer.Tracer,
ts.Seek(kb.B)
continue
}
lns[string(labelName)] = struct{}{}
if !hasCompositeLabelName {
lns[string(labelName)] = struct{}{}
} else {
_, key, err := unmarshalCompositeTagKey(labelName)
if err != nil {
return fmt.Errorf("cannot unmarshal composite tag key: %s", err)
}
lns[string(key)] = struct{}{}
}
prevLabelName = append(prevLabelName[:0], labelName...)
}
if underscoreNameSeen {
@ -853,7 +872,7 @@ func (is *indexSearch) searchLabelValuesWithFiltersOnDate(qt *querytracer.Tracer
if err != nil {
return err
}
if filter != nil && filter.Len() < 100e3 {
if filter != nil && filter.Len() <= 100e3 {
// It is faster to obtain label values by metricIDs from the filter
// instead of scanning the inverted index for the matching filters.
// This would help https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2978
@ -866,7 +885,12 @@ func (is *indexSearch) searchLabelValuesWithFiltersOnDate(qt *querytracer.Tracer
// __name__ label is encoded as empty string in indexdb.
labelName = ""
}
labelNameBytes := bytesutil.ToUnsafeBytes(labelName)
if name := getCommonMetricNameForTagFilterss(tfss); len(name) > 0 && labelName != "" {
labelNameBytes = marshalCompositeTagKey(nil, name, labelNameBytes)
}
var prevLabelValue []byte
ts := &is.ts
kb := &is.kb
@ -879,7 +903,7 @@ func (is *indexSearch) searchLabelValuesWithFiltersOnDate(qt *querytracer.Tracer
}
kb.B = is.marshalCommonPrefixForDate(kb.B[:0], date)
kb.B = marshalTagValue(kb.B, labelNameBytes)
prefix := kb.B
prefix := append([]byte{}, kb.B...)
ts.Seek(prefix)
for len(lvs) < maxLabelValues && ts.NextItem() {
if loopsPaceLimiter&paceLimiterFastIterationsMask == 0 {
@ -1233,7 +1257,7 @@ func (is *indexSearch) getTSDBStatus(qt *querytracer.Tracer, tfss []*TagFilters,
nsPrefixExpected = nsPrefixTagToMetricIDs
}
kb.B = is.marshalCommonPrefixForDate(kb.B[:0], date)
prefix := kb.B
prefix := append([]byte{}, kb.B...)
ts.Seek(prefix)
for ts.NextItem() {
if loopsPaceLimiter&paceLimiterFastIterationsMask == 0 {

27
lib/storage/index_db_test.go
View file

@ -1655,6 +1655,13 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
if err := tfs.Add([]byte("constant"), []byte("const"), false, false); err != nil {
t.Fatalf("cannot add filter: %s", err)
}
tfsMetricName := NewTagFilters()
if err := tfsMetricName.Add([]byte("constant"), []byte("const"), false, false); err != nil {
t.Fatalf("cannot add filter on label: %s", err)
}
if err := tfsMetricName.Add(nil, []byte("testMetric"), false, false); err != nil {
t.Fatalf("cannot add filter on metric name: %s", err)
}
// Perform a search within a day.
// This should return the metrics for the day
@ -1690,6 +1697,16 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
t.Fatalf("unexpected labelNames; got\n%s\nwant\n%s", lns, labelNames)
}
// Check SearchLabelNamesWithFiltersOnTimeRange with filters on metric name and time range.
lns, err = db.SearchLabelNamesWithFiltersOnTimeRange(nil, []*TagFilters{tfsMetricName}, tr, 10000, 1e9, noDeadline)
if err != nil {
t.Fatalf("unexpected error in SearchLabelNamesWithFiltersOnTimeRange(filters=%s, timeRange=%s): %s", tfs, &tr, err)
}
sort.Strings(lns)
if !reflect.DeepEqual(lns, labelNames) {
t.Fatalf("unexpected labelNames; got\n%s\nwant\n%s", lns, labelNames)
}
// Check SearchLabelValuesWithFiltersOnTimeRange with the specified filter.
lvs, err = db.SearchLabelValuesWithFiltersOnTimeRange(nil, "", []*TagFilters{tfs}, TimeRange{}, 10000, 1e9, noDeadline)
if err != nil {
@ -1710,6 +1727,16 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
t.Fatalf("unexpected labelValues; got\n%s\nwant\n%s", lvs, labelValues)
}
// Check SearchLabelValuesWithFiltersOnTimeRange with filters on metric name and time range.
lvs, err = db.SearchLabelValuesWithFiltersOnTimeRange(nil, "", []*TagFilters{tfsMetricName}, tr, 10000, 1e9, noDeadline)
if err != nil {
t.Fatalf("unexpected error in SearchLabelValuesWithFiltersOnTimeRange(filters=%s, timeRange=%s): %s", tfs, &tr, err)
}
sort.Strings(lvs)
if !reflect.DeepEqual(lvs, labelValues) {
t.Fatalf("unexpected labelValues; got\n%s\nwant\n%s", lvs, labelValues)
}
// Perform a search across all the days, should match all metrics
tr = TimeRange{
MinTimestamp: int64(now - msecPerDay*days),

23
lib/storage/tag_filters.go
View file

@ -19,6 +19,29 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
)
func getCommonMetricNameForTagFilterss(tfss []*TagFilters) []byte {
if len(tfss) == 0 {
return nil
}
prevName := getMetricNameFilter(tfss[0])
for _, tfs := range tfss[1:] {
name := getMetricNameFilter(tfs)
if string(prevName) != string(name) {
return nil
}
}
return prevName
}
func getMetricNameFilter(tfs *TagFilters) []byte {
for _, tf := range tfs.tfs {
if len(tf.key) == 0 && !tf.isNegative && !tf.isRegexp {
return tf.value
}
}
return nil
}
// convertToCompositeTagFilterss converts tfss to composite filters.
//
// This converts `foo{bar="baz",x=~"a.+"}` to `{foo=bar="baz",foo=x=~"a.+"} filter.