From 97f9397687f51507af75beb708811d2ac6dc32a5 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 12 Jul 2019 02:20:29 +0300
Subject: [PATCH] lib/storage: do not reduce `maxMetrics` on time ranges
exceeding `maxDaysForDateMetricIDs`
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/95
---
lib/storage/index_db.go | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go
index f911f28323..55f0bc4753 100644
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@@ -1269,7 +1269,6 @@ func (is *indexSearch) updateMetricIDsByMetricNameMatch(metricIDs, srcMetricIDs
}
func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) {
- maxMetrics = is.adjustMaxMetricsAdaptive(maxMetrics)
kb := &is.kb
kb.B = append(kb.B[:0], uselessMultiTagFiltersKeyPrefix)
kb.B = encoding.MarshalUint64(kb.B, uint64(maxMetrics))
@@ -1317,7 +1316,22 @@ func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters
var errTooManyMetrics = errors.New("all the tag filters match too many metrics")
-func (is *indexSearch) adjustMaxMetricsAdaptive(maxMetrics int) int {
+func isTooBigTimeRangeForDateMetricIDs(tr TimeRange) bool {
+ minDate := uint64(tr.MinTimestamp) / msecPerDay
+ maxDate := uint64(tr.MaxTimestamp) / msecPerDay
+ return maxDate-minDate > 40
+}
+
+const maxDaysForDateMetricIDs = 40
+
+func (is *indexSearch) adjustMaxMetricsAdaptive(tr TimeRange, maxMetrics int) int {
+ minDate := uint64(tr.MinTimestamp) / msecPerDay
+ maxDate := uint64(tr.MaxTimestamp) / msecPerDay
+ if maxDate-minDate > maxDaysForDateMetricIDs {
+ // Cannot reduce maxMetrics for the given time range,
+ // since the it is expensive extracting metricIDs for the given tr.
+ return maxMetrics
+ }
hmPrev := is.db.prevHourMetricIDs.Load().(*hourMetricIDs)
if !hmPrev.isFull {
return maxMetrics
@@ -1504,7 +1518,8 @@ func (is *indexSearch) updateMetricIDsForTagFilters(metricIDs map[uint64]struct{
// Sort tag filters for faster ts.Seek below.
sort.Slice(tfs.tfs, func(i, j int) bool { return bytes.Compare(tfs.tfs[i].prefix, tfs.tfs[j].prefix) < 0 })
- minTf, minMetricIDs, err := is.getTagFilterWithMinMetricIDsCountAdaptive(tfs, maxMetrics)
+ maxMetricsAdjusted := is.adjustMaxMetricsAdaptive(tr, maxMetrics)
+ minTf, minMetricIDs, err := is.getTagFilterWithMinMetricIDsCountAdaptive(tfs, maxMetricsAdjusted)
if err != nil {
if err != errTooManyMetrics {
return err
@@ -1809,7 +1824,7 @@ func (is *indexSearch) getMetricIDsForTimeRange(tr TimeRange, maxMetrics int, ac
atomic.AddUint64(&is.db.dateMetricIDsSearchCalls, 1)
minDate := uint64(tr.MinTimestamp) / msecPerDay
maxDate := uint64(tr.MaxTimestamp) / msecPerDay
- if maxDate-minDate > 40 {
+ if maxDate-minDate > maxDaysForDateMetricIDs {
// Too much dates must be covered. Give up.
return nil, errMissingMetricIDsForDate
}