github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: skip searching in extDB if it doesn't contain items for the given time range

Browse source 
This should improve inverted index search performance for big amount
of unique time series when the search is performed only on recent data.
This commit is contained in:
Aliaksandr Valialkin 2019-06-25 12:55:27 +03:00
parent feacfffe89
commit cf63669303
2 changed files with 44 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
lib/storage/index_db.go
View file

@ -1057,6 +1057,15 @@ func mergeTSIDs(a, b []TSID) []TSID {
}
func (is *indexSearch) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int) ([]TSID, error) {
// Verify whether `is` contains data for the given tr.
ok, err := is.containsTimeRange(tr)
if err != nil {
return nil, fmt.Errorf("error in containsTimeRange(%s): %s", &tr, err)
}
if !ok {
// Fast path: nothing to search.
return nil, nil
}
metricIDs, err := is.searchMetricIDs(tfss, tr, maxMetrics)
if err != nil {
return nil, err
@ -1617,15 +1626,15 @@ func (is *indexSearch) getMetricIDsForTimeRange(tr TimeRange, maxMetrics int) (m
// Slow path: collect the metric ids for all the days covering the given tr.
atomic.AddUint64(&is.db.dateMetricIDsSearchCalls, 1)
minDate := tr.MinTimestamp / msecPerDay
maxDate := tr.MaxTimestamp / msecPerDay
minDate := uint64(tr.MinTimestamp) / msecPerDay
maxDate := uint64(tr.MaxTimestamp) / msecPerDay
if maxDate-minDate > 40 {
// Too much dates must be covered. Give up.
return nil, errMissingMetricIDsForDate
}
metricIDs := make(map[uint64]struct{}, maxMetrics)
for minDate <= maxDate {
if err := is.getMetricIDsForDate(uint64(minDate), metricIDs, maxMetrics); err != nil {
if err := is.getMetricIDsForDate(minDate, metricIDs, maxMetrics); err != nil {
return nil, err
}
minDate++
@ -1760,6 +1769,28 @@ func (is *indexSearch) getMetricIDsForDate(date uint64, metricIDs map[uint64]str
return nil
}
func (is *indexSearch) containsTimeRange(tr TimeRange) (bool, error) {
ts := &is.ts
kb := &is.kb
// Verify whether the maximum date in `ts` covers tr.MinTimestamp.
minDate := uint64(tr.MinTimestamp) / msecPerDay
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixDateToMetricID)
kb.B = encoding.MarshalUint64(kb.B, minDate)
ts.Seek(kb.B)
if !ts.NextItem() {
if err := ts.Error(); err != nil {
return false, fmt.Errorf("error when searching for minDate=%d, prefix %q: %s", minDate, kb.B, err)
}
return false, nil
}
if !bytes.HasPrefix(ts.Item, kb.B[:1]) {
// minDate exceeds max date from ts.
return false, nil
}
return true, nil
}
func (is *indexSearch) updateMetricIDsForCommonPrefix(metricIDs map[uint64]struct{}, commonPrefix []byte, maxMetrics int) error {
ts := &is.ts
ts.Seek(commonPrefix)

10
lib/storage/index_db_test.go
View file

@ -293,6 +293,16 @@ func testIndexDBGetOrCreateTSIDByName(db *indexDB, accountsCount, projectsCount,
}
func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isConcurrent bool) error {
// fill Date -> MetricID cache
date := uint64(timestampFromTime(time.Now())) / msecPerDay
for i := range tsids {
tsid := &tsids[i]
if err := db.storeDateMetricID(date, tsid.MetricID); err != nil {
return fmt.Errorf("error in storeDateMetricID(%d, %d): %s", date, tsid.MetricID, err)
}
}
db.tb.DebugFlush()
hasValue := func(tvs []string, v []byte) bool {
for _, tv := range tvs {
if string(v) == tv {