diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index 303b31d3eb..e8917e076f 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -426,7 +426,7 @@ func GetLabels(deadline Deadline) ([]string, error) { if deadline.Exceeded() { return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String()) } - labels, err := vmstorage.SearchTagKeys(*maxTagKeysPerSearch) + labels, err := vmstorage.SearchTagKeys(*maxTagKeysPerSearch, deadline.deadline) if err != nil { return nil, fmt.Errorf("error during labels search: %w", err) } @@ -455,7 +455,7 @@ func GetLabelValues(labelName string, deadline Deadline) ([]string, error) { } // Search for tag values - labelValues, err := vmstorage.SearchTagValues([]byte(labelName), *maxTagValuesPerSearch) + labelValues, err := vmstorage.SearchTagValues([]byte(labelName), *maxTagValuesPerSearch, deadline.deadline) if err != nil { return nil, fmt.Errorf("error during label values search for labelName=%q: %w", labelName, err) } @@ -471,7 +471,7 @@ func GetLabelEntries(deadline Deadline) ([]storage.TagEntry, error) { if deadline.Exceeded() { return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String()) } - labelEntries, err := vmstorage.SearchTagEntries(*maxTagKeysPerSearch, *maxTagValuesPerSearch) + labelEntries, err := vmstorage.SearchTagEntries(*maxTagKeysPerSearch, *maxTagValuesPerSearch, deadline.deadline) if err != nil { return nil, fmt.Errorf("error during label entries request: %w", err) } @@ -501,7 +501,7 @@ func GetTSDBStatusForDate(deadline Deadline, date uint64, topN int) (*storage.TS if deadline.Exceeded() { return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String()) } - status, err := vmstorage.GetTSDBStatusForDate(date, topN) + status, err := vmstorage.GetTSDBStatusForDate(date, topN, deadline.deadline) if err != nil { return nil, fmt.Errorf("error during tsdb status request: %w", err) } @@ -513,7 +513,7 @@ func GetSeriesCount(deadline Deadline) (uint64, error) { if deadline.Exceeded() { return 0, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String()) } - n, err := vmstorage.GetSeriesCount() + n, err := vmstorage.GetSeriesCount(deadline.deadline) if err != nil { return 0, fmt.Errorf("error during series count request: %w", err) } @@ -560,7 +560,7 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadli defer vmstorage.WG.Done() sr := getStorageSearch() - sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch) + sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch, deadline.deadline) m := make(map[string][]storage.BlockRef) var orderedMetricNames []string diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index 5a078951f2..d23f2fa3ad 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -117,41 +117,41 @@ func DeleteMetrics(tfss []*storage.TagFilters) (int, error) { } // SearchTagKeys searches for tag keys -func SearchTagKeys(maxTagKeys int) ([]string, error) { +func SearchTagKeys(maxTagKeys int, deadline uint64) ([]string, error) { WG.Add(1) - keys, err := Storage.SearchTagKeys(maxTagKeys) + keys, err := Storage.SearchTagKeys(maxTagKeys, deadline) WG.Done() return keys, err } // SearchTagValues searches for tag values for the given tagKey -func SearchTagValues(tagKey []byte, maxTagValues int) ([]string, error) { +func SearchTagValues(tagKey []byte, maxTagValues int, deadline uint64) ([]string, error) { WG.Add(1) - values, err := Storage.SearchTagValues(tagKey, maxTagValues) + values, err := Storage.SearchTagValues(tagKey, maxTagValues, deadline) WG.Done() return values, err } // SearchTagEntries searches for tag entries. -func SearchTagEntries(maxTagKeys, maxTagValues int) ([]storage.TagEntry, error) { +func SearchTagEntries(maxTagKeys, maxTagValues int, deadline uint64) ([]storage.TagEntry, error) { WG.Add(1) - tagEntries, err := Storage.SearchTagEntries(maxTagKeys, maxTagValues) + tagEntries, err := Storage.SearchTagEntries(maxTagKeys, maxTagValues, deadline) WG.Done() return tagEntries, err } // GetTSDBStatusForDate returns TSDB status for the given date. -func GetTSDBStatusForDate(date uint64, topN int) (*storage.TSDBStatus, error) { +func GetTSDBStatusForDate(date uint64, topN int, deadline uint64) (*storage.TSDBStatus, error) { WG.Add(1) - status, err := Storage.GetTSDBStatusForDate(date, topN) + status, err := Storage.GetTSDBStatusForDate(date, topN, deadline) WG.Done() return status, err } // GetSeriesCount returns the number of time series in the storage. -func GetSeriesCount() (uint64, error) { +func GetSeriesCount(deadline uint64) (uint64, error) { WG.Add(1) - n, err := Storage.GetSeriesCount() + n, err := Storage.GetSeriesCount(deadline) WG.Done() return n, err } diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go index 36f969cbab..bce0d5660a 100644 --- a/lib/storage/index_db.go +++ b/lib/storage/index_db.go @@ -20,7 +20,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/memory" "github.com/VictoriaMetrics/VictoriaMetrics/lib/mergeset" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter" "github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set" "github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache" "github.com/VictoriaMetrics/fastcache" @@ -202,7 +201,7 @@ func openIndexDB(path string, metricIDCache, metricNameCache, tsidCache *working prevHourMetricIDs: prevHourMetricIDs, } - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) dmis, err := is.loadDeletedMetricIDs() db.putIndexSearch(is) if err != nil { @@ -210,7 +209,7 @@ func openIndexDB(path string, metricIDCache, metricNameCache, tsidCache *working } db.setDeletedMetricIDs(dmis) - is = db.getIndexSearch() + is = db.getIndexSearch(noDeadline) date, err := is.getStartDateForPerDayInvertedIndex() db.putIndexSearch(is) if err != nil { @@ -221,6 +220,8 @@ func openIndexDB(path string, metricIDCache, metricNameCache, tsidCache *working return db, nil } +const noDeadline = 1<<64 - 1 + // IndexDBMetrics contains essential metrics for indexDB. type IndexDBMetrics struct { TagCacheSize uint64 @@ -512,7 +513,7 @@ var tagFiltersKeyGen uint64 // // It returns io.EOF if the given mn isn't found locally. func (db *indexDB) getTSIDByNameNoCreate(dst *TSID, metricName []byte) error { - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) err := is.getTSIDByMetricName(dst, metricName) db.putIndexSearch(is) if err == nil { @@ -535,6 +536,9 @@ type indexSearch struct { kb bytesutil.ByteBuffer mp tagToMetricIDsRowParser + // deadline in unix timestamp seconds for the given search. + deadline uint64 + // tsidByNameMisses and tsidByNameSkips is used for a performance // hack in GetOrCreateTSIDByName. See the comment there. tsidByNameMisses int @@ -573,7 +577,7 @@ func (is *indexSearch) GetOrCreateTSIDByName(dst *TSID, metricName []byte) error return nil } -func (db *indexDB) getIndexSearch() *indexSearch { +func (db *indexDB) getIndexSearch(deadline uint64) *indexSearch { v := db.indexSearchPool.Get() if v == nil { v = &indexSearch{ @@ -582,6 +586,7 @@ func (db *indexDB) getIndexSearch() *indexSearch { } is := v.(*indexSearch) is.ts.Init(db.tb, shouldCacheBlock) + is.deadline = deadline return is } @@ -589,6 +594,7 @@ func (db *indexDB) putIndexSearch(is *indexSearch) { is.ts.MustClose() is.kb.Reset() is.mp.Reset() + is.deadline = 0 // Do not reset tsidByNameMisses and tsidByNameSkips, // since they are used in GetOrCreateTSIDByName across call boundaries. @@ -732,12 +738,12 @@ func putIndexItems(ii *indexItems) { var indexItemsPool sync.Pool // SearchTagKeys returns all the tag keys. -func (db *indexDB) SearchTagKeys(maxTagKeys int) ([]string, error) { +func (db *indexDB) SearchTagKeys(maxTagKeys int, deadline uint64) ([]string, error) { // TODO: cache results? tks := make(map[string]struct{}) - is := db.getIndexSearch() + is := db.getIndexSearch(deadline) err := is.searchTagKeys(tks, maxTagKeys) db.putIndexSearch(is) if err != nil { @@ -745,7 +751,7 @@ func (db *indexDB) SearchTagKeys(maxTagKeys int) ([]string, error) { } ok := db.doExtDB(func(extDB *indexDB) { - is := extDB.getIndexSearch() + is := extDB.getIndexSearch(deadline) err = is.searchTagKeys(tks, maxTagKeys) extDB.putIndexSearch(is) }) @@ -775,7 +781,9 @@ func (is *indexSearch) searchTagKeys(tks map[string]struct{}, maxTagKeys int) er ts.Seek(prefix) for len(tks) < maxTagKeys && ts.NextItem() { if loopsPaceLimiter&(1<<16) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return err + } } loopsPaceLimiter++ item := ts.Item @@ -807,18 +815,18 @@ func (is *indexSearch) searchTagKeys(tks map[string]struct{}, maxTagKeys int) er } // SearchTagValues returns all the tag values for the given tagKey -func (db *indexDB) SearchTagValues(tagKey []byte, maxTagValues int) ([]string, error) { +func (db *indexDB) SearchTagValues(tagKey []byte, maxTagValues int, deadline uint64) ([]string, error) { // TODO: cache results? tvs := make(map[string]struct{}) - is := db.getIndexSearch() + is := db.getIndexSearch(deadline) err := is.searchTagValues(tvs, tagKey, maxTagValues) db.putIndexSearch(is) if err != nil { return nil, err } ok := db.doExtDB(func(extDB *indexDB) { - is := extDB.getIndexSearch() + is := extDB.getIndexSearch(deadline) err = is.searchTagValues(tvs, tagKey, maxTagValues) extDB.putIndexSearch(is) }) @@ -853,7 +861,9 @@ func (is *indexSearch) searchTagValues(tvs map[string]struct{}, tagKey []byte, m ts.Seek(prefix) for len(tvs) < maxTagValues && ts.NextItem() { if loopsPaceLimiter&(1<<16) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return err + } } loopsPaceLimiter++ item := ts.Item @@ -895,8 +905,8 @@ func (is *indexSearch) searchTagValues(tvs map[string]struct{}, tagKey []byte, m // // It includes the deleted series too and may count the same series // up to two times - in db and extDB. -func (db *indexDB) GetSeriesCount() (uint64, error) { - is := db.getIndexSearch() +func (db *indexDB) GetSeriesCount(deadline uint64) (uint64, error) { + is := db.getIndexSearch(deadline) n, err := is.getSeriesCount() db.putIndexSearch(is) if err != nil { @@ -905,7 +915,7 @@ func (db *indexDB) GetSeriesCount() (uint64, error) { var nExt uint64 ok := db.doExtDB(func(extDB *indexDB) { - is := extDB.getIndexSearch() + is := extDB.getIndexSearch(deadline) nExt, err = is.getSeriesCount() extDB.putIndexSearch(is) }) @@ -927,7 +937,9 @@ func (is *indexSearch) getSeriesCount() (uint64, error) { ts.Seek(kb.B) for ts.NextItem() { if loopsPaceLimiter&(1<<16) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return 0, err + } } loopsPaceLimiter++ item := ts.Item @@ -955,8 +967,8 @@ func (is *indexSearch) getSeriesCount() (uint64, error) { } // GetTSDBStatusForDate returns topN entries for tsdb status for the given date. -func (db *indexDB) GetTSDBStatusForDate(date uint64, topN int) (*TSDBStatus, error) { - is := db.getIndexSearch() +func (db *indexDB) GetTSDBStatusForDate(date uint64, topN int, deadline uint64) (*TSDBStatus, error) { + is := db.getIndexSearch(deadline) status, err := is.getTSDBStatusForDate(date, topN) db.putIndexSearch(is) if err != nil { @@ -969,7 +981,7 @@ func (db *indexDB) GetTSDBStatusForDate(date uint64, topN int) (*TSDBStatus, err // The entries weren't found in the db. Try searching them in extDB. ok := db.doExtDB(func(extDB *indexDB) { - is := extDB.getIndexSearch() + is := extDB.getIndexSearch(deadline) status, err = is.getTSDBStatusForDate(date, topN) extDB.putIndexSearch(is) }) @@ -997,7 +1009,9 @@ func (is *indexSearch) getTSDBStatusForDate(date uint64, topN int) (*TSDBStatus, ts.Seek(prefix) for ts.NextItem() { if loopsPaceLimiter&(1<<16) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return nil, err + } } loopsPaceLimiter++ item := ts.Item @@ -1149,7 +1163,7 @@ func (th *topHeap) Pop() interface{} { // searchMetricName appends metric name for the given metricID to dst // and returns the result. func (db *indexDB) searchMetricName(dst []byte, metricID uint64) ([]byte, error) { - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) dst, err := is.searchMetricName(dst, metricID) db.putIndexSearch(is) @@ -1159,7 +1173,7 @@ func (db *indexDB) searchMetricName(dst []byte, metricID uint64) ([]byte, error) // Try searching in the external indexDB. if db.doExtDB(func(extDB *indexDB) { - is := extDB.getIndexSearch() + is := extDB.getIndexSearch(noDeadline) dst, err = is.searchMetricName(dst, metricID) extDB.putIndexSearch(is) }) { @@ -1194,7 +1208,7 @@ func (db *indexDB) DeleteTSIDs(tfss []*TagFilters) (int, error) { MinTimestamp: 0, MaxTimestamp: (1 << 63) - 1, } - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) metricIDs, err := is.searchMetricIDs(tfss, tr, 2e9) db.putIndexSearch(is) if err != nil { @@ -1324,7 +1338,7 @@ func (is *indexSearch) loadDeletedMetricIDs() (*uint64set.Set, error) { } // searchTSIDs returns sorted tsids matching the given tfss over the given tr. -func (db *indexDB) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int) ([]TSID, error) { +func (db *indexDB) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int, deadline uint64) ([]TSID, error) { if len(tfss) == 0 { return nil, nil } @@ -1340,7 +1354,7 @@ func (db *indexDB) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int) } // Slow path - search for tsids in the db and extDB. - is := db.getIndexSearch() + is := db.getIndexSearch(deadline) localTSIDs, err := is.searchTSIDs(tfss, tr, maxMetrics) db.putIndexSearch(is) if err != nil { @@ -1359,7 +1373,7 @@ func (db *indexDB) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int) extTSIDs = tsids return } - is := extDB.getIndexSearch() + is := extDB.getIndexSearch(deadline) extTSIDs, err = is.searchTSIDs(tfss, tr, maxMetrics) extDB.putIndexSearch(is) @@ -1519,7 +1533,9 @@ func (is *indexSearch) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics i := 0 for loopsPaceLimiter, metricID := range metricIDs { if loopsPaceLimiter&(1<<10) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return nil, err + } } // Try obtaining TSIDs from MetricID->TSID cache. This is much faster // than scanning the mergeset if it contains a lot of metricIDs. @@ -1589,7 +1605,9 @@ func (is *indexSearch) updateMetricIDsByMetricNameMatch(metricIDs, srcMetricIDs defer PutMetricName(mn) for loopsPaceLimiter, metricID := range sortedMetricIDs { if loopsPaceLimiter&(1<<10) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return err + } } var err error metricName.B, err = is.searchMetricName(metricName.B[:0], metricID) @@ -2075,7 +2093,9 @@ func (is *indexSearch) getMetricIDsForTagFilterSlow(tf *tagFilter, maxLoops int, ts.Seek(prefix) for ts.NextItem() { if loopsPaceLimiter&(1<<14) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return err + } } loopsPaceLimiter++ item := ts.Item @@ -2197,7 +2217,9 @@ func (is *indexSearch) updateMetricIDsForOrSuffixNoFilter(prefix []byte, maxMetr ts.Seek(prefix) for metricIDs.Len() < maxMetrics && ts.NextItem() { if loopsPaceLimiter&(1<<16) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return err + } } loopsPaceLimiter++ item := ts.Item @@ -2237,7 +2259,9 @@ func (is *indexSearch) updateMetricIDsForOrSuffixWithFilter(prefix []byte, metri var metricID uint64 for ts.NextItem() { if loopsPaceLimiter&(1<<12) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return err + } } loopsPaceLimiter++ item := ts.Item @@ -2347,7 +2371,7 @@ func (is *indexSearch) getMetricIDsForTimeRange(tr TimeRange, maxMetrics int) (* wg.Add(1) go func(date uint64) { defer wg.Done() - isLocal := is.db.getIndexSearch() + isLocal := is.db.getIndexSearch(is.deadline) defer is.db.putIndexSearch(isLocal) m, err := isLocal.getMetricIDsForDate(date, maxMetrics) mu.Lock() @@ -2404,7 +2428,7 @@ func (is *indexSearch) tryUpdatingMetricIDsForDateRange(metricIDs *uint64set.Set wg.Add(1) go func(date uint64) { defer wg.Done() - isLocal := is.db.getIndexSearch() + isLocal := is.db.getIndexSearch(is.deadline) defer is.db.putIndexSearch(isLocal) m, err := isLocal.getMetricIDsForDateAndFilters(date, tfs, maxMetrics) mu.Lock() @@ -2776,7 +2800,9 @@ func (is *indexSearch) updateMetricIDsForPrefix(prefix []byte, metricIDs *uint64 ts.Seek(prefix) for ts.NextItem() { if loopsPaceLimiter&(1<<16) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return err + } } loopsPaceLimiter++ item := ts.Item diff --git a/lib/storage/index_db_test.go b/lib/storage/index_db_test.go index b897cc56e3..add05b537f 100644 --- a/lib/storage/index_db_test.go +++ b/lib/storage/index_db_test.go @@ -611,7 +611,7 @@ func testIndexDBBigMetricName(db *indexDB) error { var mn MetricName var tsid TSID - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) defer db.putIndexSearch(is) // Try creating too big metric group @@ -672,7 +672,7 @@ func testIndexDBGetOrCreateTSIDByName(db *indexDB, metricGroups int) ([]MetricNa var mns []MetricName var tsids []TSID - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) defer db.putIndexSearch(is) var metricNameBuf []byte @@ -773,7 +773,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC } // Test SearchTagValues - tvs, err := db.SearchTagValues(nil, 1e5) + tvs, err := db.SearchTagValues(nil, 1e5, noDeadline) if err != nil { return fmt.Errorf("error in SearchTagValues for __name__: %w", err) } @@ -782,7 +782,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC } for i := range mn.Tags { tag := &mn.Tags[i] - tvs, err := db.SearchTagValues(tag.Key, 1e5) + tvs, err := db.SearchTagValues(tag.Key, 1e5, noDeadline) if err != nil { return fmt.Errorf("error in SearchTagValues for __name__: %w", err) } @@ -794,7 +794,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC } // Test SearchTagKeys - tks, err := db.SearchTagKeys(1e5) + tks, err := db.SearchTagKeys(1e5, noDeadline) if err != nil { return fmt.Errorf("error in SearchTagKeys: %w", err) } @@ -811,7 +811,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC // Concurrent test may create duplicate timeseries, so GetSeriesCount // would return more timeseries than needed. if !isConcurrent { - n, err := db.GetSeriesCount() + n, err := db.GetSeriesCount(noDeadline) if err != nil { return fmt.Errorf("unexpected error in GetSeriesCount(): %w", err) } @@ -842,7 +842,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC if err := tfs.Add(nil, nil, true, false); err != nil { return fmt.Errorf("cannot add no-op negative filter: %w", err) } - tsidsFound, err := db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5) + tsidsFound, err := db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search by exact tag filter: %w", err) } @@ -851,7 +851,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC } // Verify tag cache. - tsidsCached, err := db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5) + tsidsCached, err := db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search by exact tag filter: %w", err) } @@ -863,7 +863,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC if err := tfs.Add(nil, mn.MetricGroup, true, false); err != nil { return fmt.Errorf("cannot add negative filter for zeroing search results: %w", err) } - tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5) + tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search by exact tag filter with full negative: %w", err) } @@ -884,7 +884,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC if tfsNew := tfs.Finalize(); len(tfsNew) > 0 { return fmt.Errorf("unexpected non-empty tag filters returned by TagFilters.Finalize: %v", tfsNew) } - tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5) + tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search by regexp tag filter for Graphite wildcard: %w", err) } @@ -912,7 +912,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC if err := tfs.Add(nil, nil, true, true); err != nil { return fmt.Errorf("cannot add no-op negative filter with regexp: %w", err) } - tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5) + tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search by regexp tag filter: %w", err) } @@ -922,7 +922,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC if err := tfs.Add(nil, mn.MetricGroup, true, true); err != nil { return fmt.Errorf("cannot add negative filter for zeroing search results: %w", err) } - tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5) + tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search by regexp tag filter with full negative: %w", err) } @@ -938,7 +938,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC if err := tfs.Add(nil, mn.MetricGroup, false, true); err != nil { return fmt.Errorf("cannot create tag filter for MetricGroup matching zero results: %w", err) } - tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5) + tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search by non-existing tag filter: %w", err) } @@ -954,7 +954,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC // Search with empty filter. It should match all the results. tfs.Reset() - tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5) + tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search for common prefix: %w", err) } @@ -967,7 +967,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC if err := tfs.Add(nil, nil, false, false); err != nil { return fmt.Errorf("cannot create tag filter for empty metricGroup: %w", err) } - tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5) + tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search for empty metricGroup: %w", err) } @@ -984,7 +984,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC if err := tfs2.Add(nil, mn.MetricGroup, false, false); err != nil { return fmt.Errorf("cannot create tag filter for MetricGroup: %w", err) } - tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs1, tfs2}, TimeRange{}, 1e5) + tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs1, tfs2}, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search for empty metricGroup: %w", err) } @@ -993,7 +993,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC } // Verify empty tfss - tsidsFound, err = db.searchTSIDs(nil, TimeRange{}, 1e5) + tsidsFound, err = db.searchTSIDs(nil, TimeRange{}, 1e5, noDeadline) if err != nil { return fmt.Errorf("cannot search for nil tfss: %w", err) } @@ -1501,7 +1501,7 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { } }() - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) defer db.putIndexSearch(is) // Create a bunch of per-day time series @@ -1573,7 +1573,7 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { MinTimestamp: int64(now - msecPerHour + 1), MaxTimestamp: int64(now), } - matchedTSIDs, err := db.searchTSIDs([]*TagFilters{tfs}, tr, 10000) + matchedTSIDs, err := db.searchTSIDs([]*TagFilters{tfs}, tr, 10000, noDeadline) if err != nil { t.Fatalf("error searching tsids: %v", err) } @@ -1587,7 +1587,7 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { MinTimestamp: int64(now - 2*msecPerHour - 1), MaxTimestamp: int64(now), } - matchedTSIDs, err = db.searchTSIDs([]*TagFilters{tfs}, tr, 10000) + matchedTSIDs, err = db.searchTSIDs([]*TagFilters{tfs}, tr, 10000, noDeadline) if err != nil { t.Fatalf("error searching tsids: %v", err) } @@ -1601,7 +1601,7 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { MaxTimestamp: int64(now), } - matchedTSIDs, err = db.searchTSIDs([]*TagFilters{tfs}, tr, 10000) + matchedTSIDs, err = db.searchTSIDs([]*TagFilters{tfs}, tr, 10000, noDeadline) if err != nil { t.Fatalf("error searching tsids: %v", err) } @@ -1610,7 +1610,7 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { } // Check GetTSDBStatusForDate - status, err := db.GetTSDBStatusForDate(baseDate, 5) + status, err := db.GetTSDBStatusForDate(baseDate, 5, noDeadline) if err != nil { t.Fatalf("error in GetTSDBStatusForDate: %s", err) } diff --git a/lib/storage/index_db_timing_test.go b/lib/storage/index_db_timing_test.go index b19e7b2a4f..984ee107a2 100644 --- a/lib/storage/index_db_timing_test.go +++ b/lib/storage/index_db_timing_test.go @@ -95,7 +95,7 @@ func BenchmarkIndexDBAddTSIDs(b *testing.B) { func benchmarkIndexDBAddTSIDs(db *indexDB, tsid *TSID, mn *MetricName, startOffset, recordsPerLoop int) { var metricName []byte - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) defer db.putIndexSearch(is) for i := 0; i < recordsPerLoop; i++ { mn.MetricGroup = strconv.AppendUint(mn.MetricGroup[:0], uint64(i+startOffset), 10) @@ -170,7 +170,7 @@ func BenchmarkHeadPostingForMatchers(b *testing.B) { b.ResetTimer() benchSearch := func(b *testing.B, tfs *TagFilters, expectedMetricIDs int) { - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) defer db.putIndexSearch(is) tfss := []*TagFilters{tfs} tr := TimeRange{ @@ -335,7 +335,7 @@ func BenchmarkIndexDBGetTSIDs(b *testing.B) { var tsid TSID var metricName []byte - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) defer db.putIndexSearch(is) for i := 0; i < recordsCount; i++ { mn.sortTags() @@ -352,7 +352,7 @@ func BenchmarkIndexDBGetTSIDs(b *testing.B) { var tsidLocal TSID var metricNameLocal []byte mnLocal := mn - is := db.getIndexSearch() + is := db.getIndexSearch(noDeadline) defer db.putIndexSearch(is) for pb.Next() { for i := 0; i < recordsPerLoop; i++ { diff --git a/lib/storage/search.go b/lib/storage/search.go index 6f4fe0d253..ed570e073a 100644 --- a/lib/storage/search.go +++ b/lib/storage/search.go @@ -6,6 +6,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter" ) @@ -64,6 +65,9 @@ type Search struct { ts tableSearch + // deadline in unix timestamp seconds for the current search. + deadline uint64 + err error needClosing bool @@ -77,6 +81,7 @@ func (s *Search) reset() { s.storage = nil s.ts.reset() + s.deadline = 0 s.err = nil s.needClosing = false s.loops = 0 @@ -85,17 +90,18 @@ func (s *Search) reset() { // Init initializes s from the given storage, tfss and tr. // // MustClose must be called when the search is done. -func (s *Search) Init(storage *Storage, tfss []*TagFilters, tr TimeRange, maxMetrics int) { +func (s *Search) Init(storage *Storage, tfss []*TagFilters, tr TimeRange, maxMetrics int, deadline uint64) { if s.needClosing { logger.Panicf("BUG: missing MustClose call before the next call to Init") } s.reset() + s.deadline = deadline s.needClosing = true - tsids, err := storage.searchTSIDs(tfss, tr, maxMetrics) + tsids, err := storage.searchTSIDs(tfss, tr, maxMetrics, deadline) if err == nil { - err = storage.prefetchMetricNames(tsids) + err = storage.prefetchMetricNames(tsids, deadline) } // It is ok to call Init on error from storage.searchTSIDs. // Init must be called before returning because it will fail @@ -134,7 +140,10 @@ func (s *Search) NextMetricBlock() bool { } for s.ts.NextBlock() { if s.loops&(1<<10) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(s.deadline); err != nil { + s.err = err + return false + } } s.loops++ tsid := &s.ts.BlockRef.bh.TSID @@ -320,3 +329,11 @@ func (sq *SearchQuery) Unmarshal(src []byte) ([]byte, error) { return src, nil } + +func checkSearchDeadlineAndPace(deadline uint64) error { + if fasttime.UnixTimestamp() > deadline { + return errDeadlineExceeded + } + storagepacelimiter.Search.WaitIfNeeded() + return nil +} diff --git a/lib/storage/search_test.go b/lib/storage/search_test.go index c4e41b8619..6396acad61 100644 --- a/lib/storage/search_test.go +++ b/lib/storage/search_test.go @@ -218,7 +218,7 @@ func testSearchInternal(st *Storage, tr TimeRange, mrs []MetricRow, accountsCoun } // Search - s.Init(st, []*TagFilters{tfs}, tr, 1e5) + s.Init(st, []*TagFilters{tfs}, tr, 1e5, noDeadline) var mbs []metricBlock for s.NextMetricBlock() { var b Block diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 0034cfd444..58ed79c6c7 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -799,10 +799,10 @@ func nextRetentionDuration(retentionMonths int) time.Duration { } // searchTSIDs returns sorted TSIDs for the given tfss and the given tr. -func (s *Storage) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int) ([]TSID, error) { +func (s *Storage) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int, deadline uint64) ([]TSID, error) { // Do not cache tfss -> tsids here, since the caching is performed // on idb level. - tsids, err := s.idb().searchTSIDs(tfss, tr, maxMetrics) + tsids, err := s.idb().searchTSIDs(tfss, tr, maxMetrics, deadline) if err != nil { return nil, fmt.Errorf("error when searching tsids for tfss %q: %w", tfss, err) } @@ -812,7 +812,7 @@ func (s *Storage) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int) // prefetchMetricNames pre-fetches metric names for the given tsids into metricID->metricName cache. // // This should speed-up further searchMetricName calls for metricIDs from tsids. -func (s *Storage) prefetchMetricNames(tsids []TSID) error { +func (s *Storage) prefetchMetricNames(tsids []TSID, deadline uint64) error { if len(tsids) == 0 { return nil } @@ -837,11 +837,13 @@ func (s *Storage) prefetchMetricNames(tsids []TSID) error { var metricName []byte var err error idb := s.idb() - is := idb.getIndexSearch() + is := idb.getIndexSearch(deadline) defer idb.putIndexSearch(is) for loops, metricID := range metricIDs { if loops&(1<<10) == 0 { - storagepacelimiter.Search.WaitIfNeeded() + if err := checkSearchDeadlineAndPace(is.deadline); err != nil { + return err + } } metricName, err = is.searchMetricName(metricName[:0], metricID) if err != nil && err != io.EOF { @@ -856,6 +858,8 @@ func (s *Storage) prefetchMetricNames(tsids []TSID) error { return nil } +var errDeadlineExceeded = fmt.Errorf("deadline exceeded") + // DeleteMetrics deletes all the metrics matching the given tfss. // // Returns the number of metrics deleted. @@ -880,19 +884,19 @@ func (s *Storage) searchMetricName(dst []byte, metricID uint64) ([]byte, error) } // SearchTagKeys searches for tag keys -func (s *Storage) SearchTagKeys(maxTagKeys int) ([]string, error) { - return s.idb().SearchTagKeys(maxTagKeys) +func (s *Storage) SearchTagKeys(maxTagKeys int, deadline uint64) ([]string, error) { + return s.idb().SearchTagKeys(maxTagKeys, deadline) } // SearchTagValues searches for tag values for the given tagKey -func (s *Storage) SearchTagValues(tagKey []byte, maxTagValues int) ([]string, error) { - return s.idb().SearchTagValues(tagKey, maxTagValues) +func (s *Storage) SearchTagValues(tagKey []byte, maxTagValues int, deadline uint64) ([]string, error) { + return s.idb().SearchTagValues(tagKey, maxTagValues, deadline) } // SearchTagEntries returns a list of (tagName -> tagValues) -func (s *Storage) SearchTagEntries(maxTagKeys, maxTagValues int) ([]TagEntry, error) { +func (s *Storage) SearchTagEntries(maxTagKeys, maxTagValues int, deadline uint64) ([]TagEntry, error) { idb := s.idb() - keys, err := idb.SearchTagKeys(maxTagKeys) + keys, err := idb.SearchTagKeys(maxTagKeys, deadline) if err != nil { return nil, fmt.Errorf("cannot search tag keys: %w", err) } @@ -902,7 +906,7 @@ func (s *Storage) SearchTagEntries(maxTagKeys, maxTagValues int) ([]TagEntry, er tes := make([]TagEntry, len(keys)) for i, key := range keys { - values, err := idb.SearchTagValues([]byte(key), maxTagValues) + values, err := idb.SearchTagValues([]byte(key), maxTagValues, deadline) if err != nil { return nil, fmt.Errorf("cannot search values for tag %q: %w", key, err) } @@ -926,15 +930,15 @@ type TagEntry struct { // // It includes the deleted series too and may count the same series // up to two times - in db and extDB. -func (s *Storage) GetSeriesCount() (uint64, error) { - return s.idb().GetSeriesCount() +func (s *Storage) GetSeriesCount(deadline uint64) (uint64, error) { + return s.idb().GetSeriesCount(deadline) } // GetTSDBStatusForDate returns TSDB status data for /api/v1/status/tsdb. // // See https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats -func (s *Storage) GetTSDBStatusForDate(date uint64, topN int) (*TSDBStatus, error) { - return s.idb().GetTSDBStatusForDate(date, topN) +func (s *Storage) GetTSDBStatusForDate(date uint64, topN int, deadline uint64) (*TSDBStatus, error) { + return s.idb().GetTSDBStatusForDate(date, topN, deadline) } // MetricRow is a metric to insert into storage. @@ -1131,7 +1135,7 @@ func (s *Storage) add(rows []rawRow, mrs []MetricRow, precisionBits uint8) ([]ra sort.Slice(pendingMetricRows, func(i, j int) bool { return string(pendingMetricRows[i].MetricName) < string(pendingMetricRows[j].MetricName) }) - is := idb.getIndexSearch() + is := idb.getIndexSearch(noDeadline) prevMetricNameRaw = nil for i := range pendingMetricRows { pmr := &pendingMetricRows[i] @@ -1339,7 +1343,7 @@ func (s *Storage) updatePerDateData(rows []rawRow) error { return a.metricID < b.metricID }) idb := s.idb() - is := idb.getIndexSearch() + is := idb.getIndexSearch(noDeadline) defer idb.putIndexSearch(is) var firstError error prevMetricID = 0 diff --git a/lib/storage/storage_test.go b/lib/storage/storage_test.go index 39e45a584f..3d4eb4cabc 100644 --- a/lib/storage/storage_test.go +++ b/lib/storage/storage_test.go @@ -453,7 +453,7 @@ func TestStorageDeleteMetrics(t *testing.T) { } // Verify no tag keys exist - tks, err := s.SearchTagKeys(1e5) + tks, err := s.SearchTagKeys(1e5, noDeadline) if err != nil { t.Fatalf("error in SearchTagKeys at the start: %s", err) } @@ -504,7 +504,7 @@ func TestStorageDeleteMetrics(t *testing.T) { }) // Verify no more tag keys exist - tks, err = s.SearchTagKeys(1e5) + tks, err = s.SearchTagKeys(1e5, noDeadline) if err != nil { t.Fatalf("error in SearchTagKeys after the test: %s", err) } @@ -560,7 +560,7 @@ func testStorageDeleteMetrics(s *Storage, workerNum int) error { s.debugFlush() // Verify tag values exist - tvs, err := s.SearchTagValues(workerTag, 1e5) + tvs, err := s.SearchTagValues(workerTag, 1e5, noDeadline) if err != nil { return fmt.Errorf("error in SearchTagValues before metrics removal: %w", err) } @@ -569,7 +569,7 @@ func testStorageDeleteMetrics(s *Storage, workerNum int) error { } // Verify tag keys exist - tks, err := s.SearchTagKeys(1e5) + tks, err := s.SearchTagKeys(1e5, noDeadline) if err != nil { return fmt.Errorf("error in SearchTagKeys before metrics removal: %w", err) } @@ -585,7 +585,7 @@ func testStorageDeleteMetrics(s *Storage, workerNum int) error { metricBlocksCount := func(tfs *TagFilters) int { // Verify the number of blocks n := 0 - sr.Init(s, []*TagFilters{tfs}, tr, 1e5) + sr.Init(s, []*TagFilters{tfs}, tr, 1e5, noDeadline) for sr.NextMetricBlock() { n++ } @@ -633,7 +633,7 @@ func testStorageDeleteMetrics(s *Storage, workerNum int) error { if n := metricBlocksCount(tfs); n != 0 { return fmt.Errorf("expecting zero metric blocks after deleting all the metrics; got %d blocks", n) } - tvs, err = s.SearchTagValues(workerTag, 1e5) + tvs, err = s.SearchTagValues(workerTag, 1e5, noDeadline) if err != nil { return fmt.Errorf("error in SearchTagValues after all the metrics are removed: %w", err) }