github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/logstorage: lazily read column headers metadata during queries

Browse source 
This improves performance for analytical queries, which do not need column headers metadata.
For example, the following query doesn't need column headers metadata, since _stream and min(_time)
are stored in block header, which is read separately from colum headers metadata:

  _time:1w | stats by (_stream) min(_time) min_time

This commit significantly improves the performance for this query.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7070
This commit is contained in:
Aliaksandr Valialkin 2024-09-25 16:59:58 +02:00
parent 4599429f51
commit 65b93b17b1
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
17 changed files with 73 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
lib/logstorage/block_result.go
View file

@ -305,10 +305,11 @@ func (br *blockResult) initAllColumns() {
if !slices.Contains(unneededColumnNames, "_msg") {
// Add _msg column
v := br.bs.csh.getConstColumnValue("_msg")
csh := br.bs.getColumnsHeader()
v := csh.getConstColumnValue("_msg")
if v != "" {
br.addConstColumn("_msg", v)
} else if ch := br.bs.csh.getColumnHeader("_msg"); ch != nil {
} else if ch := csh.getColumnHeader("_msg"); ch != nil {
br.addColumn(ch)
} else {
br.addConstColumn("_msg", "")
@ -316,7 +317,8 @@ func (br *blockResult) initAllColumns() {
}
// Add other const columns
for _, cc := range br.bs.csh.constColumns {
csh := br.bs.getColumnsHeader()
for _, cc := range csh.constColumns {
if isMsgFieldName(cc.Name) {
continue
}
@ -326,7 +328,7 @@ func (br *blockResult) initAllColumns() {
}
// Add other non-const columns
chs := br.bs.csh.columnHeaders
chs := csh.columnHeaders
for i := range chs {
ch := &chs[i]
if isMsgFieldName(ch.name) {
@ -355,10 +357,11 @@ func (br *blockResult) initRequestedColumns() {
case "_time":
br.addTimeColumn()
default:
v := br.bs.csh.getConstColumnValue(columnName)
csh := br.bs.getColumnsHeader()
v := csh.getConstColumnValue(columnName)
if v != "" {
br.addConstColumn(columnName, v)
} else if ch := br.bs.csh.getColumnHeader(columnName); ch != nil {
} else if ch := csh.getColumnHeader(columnName); ch != nil {
br.addColumn(ch)
} else {
br.addConstColumn(columnName, "")

25
lib/logstorage/block_search.go
View file

@ -113,10 +113,15 @@ type blockSearch struct {
// sbu is used for unmarshaling local columns
sbu stringsBlockUnmarshaler
// csh is the columnsHeader associated with the given block
csh columnsHeader
// cshCached is the columnsHeader associated with the given block
//
// it is initialized lazily by calling getColumnsHeader().
cshCached columnsHeader
// a is used for storing unmarshaled data in csh
// cshInitialized is set to true if cshCached is initialized.
cshInitialized bool
// a is used for storing unmarshaled data in cshCached
a arena
// seenStreams contains seen streamIDs for the recent searches.
@ -146,7 +151,10 @@ func (bs *blockSearch) reset() {
}
bs.sbu.reset()
bs.csh.reset()
bs.cshCached.reset()
bs.cshInitialized = false
bs.a.reset()
// Do not reset seenStreams, since its' lifetime is managed by blockResult.addStreamColumn() code.
@ -161,8 +169,6 @@ func (bs *blockSearch) search(bsw *blockSearchWork, bm *bitmap) {
bs.bsw = bsw
bs.csh.initFromBlockHeader(&bs.a, bsw.p, &bsw.bh)
// search rows matching the given filter
bm.init(int(bsw.bh.rowsCount))
bm.setBits()
@ -183,6 +189,13 @@ func (bs *blockSearch) search(bsw *blockSearchWork, bm *bitmap) {
}
}
func (bs *blockSearch) getColumnsHeader() *columnsHeader {
if !bs.cshInitialized {
bs.cshCached.initFromBlockHeader(&bs.a, bs.bsw.p, &bs.bsw.bh)
}
return &bs.cshCached
}
func (csh *columnsHeader) initFromBlockHeader(a *arena, p *part, bh *blockHeader) {
bb := longTermBufPool.Get()
columnsHeaderSize := bh.columnsHeaderSize

5
lib/logstorage/filter_and.go
View file

@ -81,7 +81,8 @@ func (fa *filterAnd) matchBloomFilters(bs *blockSearch) bool {
fieldName := ft.field
tokens := ft.tokens
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if matchStringByAllTokens(v, tokens) {
continue
@ -89,7 +90,7 @@ func (fa *filterAnd) matchBloomFilters(bs *blockSearch) bool {
return false
}
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
return false
}

5
lib/logstorage/filter_any_case_phrase.go
View file

@ -86,7 +86,8 @@ func (fp *filterAnyCasePhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
phraseLowercase := fp.getPhraseLowercase()
// Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !matchAnyCasePhrase(v, phraseLowercase) {
bm.resetBits()
@ -95,7 +96,7 @@ func (fp *filterAnyCasePhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty phrase.

5
lib/logstorage/filter_any_case_prefix.go
View file

@ -90,7 +90,8 @@ func (fp *filterAnyCasePrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
prefixLowercase := fp.getPrefixLowercase()
// Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !matchAnyCasePrefix(v, prefixLowercase) {
bm.resetBits()
@ -99,7 +100,7 @@ func (fp *filterAnyCasePrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
bm.resetBits()

5
lib/logstorage/filter_exact.go
View file

@ -174,7 +174,8 @@ func (fe *filterExact) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
fieldName := fe.fieldName
value := fe.value
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if value != v {
bm.resetBits()
@ -183,7 +184,7 @@ func (fe *filterExact) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty value.

5
lib/logstorage/filter_exact_prefix.go
View file

@ -51,7 +51,8 @@ func (fep *filterExactPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
fieldName := fep.fieldName
prefix := fep.prefix
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !matchExactPrefix(v, prefix) {
bm.resetBits()
@ -60,7 +61,7 @@ func (fep *filterExactPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
if !matchExactPrefix("", prefix) {

5
lib/logstorage/filter_in.go
View file

@ -358,7 +358,8 @@ func (fi *filterIn) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return
}
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
stringValues := fi.getStringValues()
if _, ok := stringValues[v]; !ok {
@ -368,7 +369,7 @@ func (fi *filterIn) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty phrase.

5
lib/logstorage/filter_ipv4_range.go
View file

@ -102,7 +102,8 @@ func (fr *filterIPv4Range) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return
}
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !matchIPv4Range(v, minValue, maxValue) {
bm.resetBits()
@ -111,7 +112,7 @@ func (fr *filterIPv4Range) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
bm.resetBits()

5
lib/logstorage/filter_len_range.go
View file

@ -125,7 +125,8 @@ func (fr *filterLenRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return
}
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !matchLenRange(v, minLen, maxLen) {
bm.resetBits()
@ -134,7 +135,7 @@ func (fr *filterLenRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
if !matchLenRange("", minLen, maxLen) {

5
lib/logstorage/filter_or.go
View file

@ -93,7 +93,8 @@ func (fo *filterOr) matchBloomFilters(bs *blockSearch) bool {
fieldName := ft.field
tokens := ft.tokens
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if matchStringByAllTokens(v, tokens) {
return true
@ -101,7 +102,7 @@ func (fo *filterOr) matchBloomFilters(bs *blockSearch) bool {
continue
}
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
continue
}

5
lib/logstorage/filter_phrase.go
View file

@ -61,7 +61,8 @@ func (fp *filterPhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
phrase := fp.phrase
// Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !matchPhrase(v, phrase) {
bm.resetBits()
@ -70,7 +71,7 @@ func (fp *filterPhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty phrase.

5
lib/logstorage/filter_prefix.go
View file

@ -59,7 +59,8 @@ func (fp *filterPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
prefix := fp.prefix
// Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !matchPrefix(v, prefix) {
bm.resetBits()
@ -68,7 +69,7 @@ func (fp *filterPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
bm.resetBits()

5
lib/logstorage/filter_range.go
View file

@ -173,7 +173,8 @@ func (fr *filterRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return
}
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !matchRange(v, minValue, maxValue) {
bm.resetBits()
@ -182,7 +183,7 @@ func (fr *filterRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
bm.resetBits()

5
lib/logstorage/filter_regexp.go
View file

@ -78,7 +78,8 @@ func (fr *filterRegexp) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
re := fr.re
// Verify whether filter matches const column
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !re.MatchString(v) {
bm.resetBits()
@ -87,7 +88,7 @@ func (fr *filterRegexp) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
if !re.MatchString("") {

5
lib/logstorage/filter_sequence.go
View file

@ -87,7 +87,8 @@ func (fs *filterSequence) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return
}
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !matchSequence(v, phrases) {
bm.resetBits()
@ -96,7 +97,7 @@ func (fs *filterSequence) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty phrase.

5
lib/logstorage/filter_string_range.go
View file

@ -52,7 +52,8 @@ func (fr *filterStringRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return
}
v := bs.csh.getConstColumnValue(fieldName)
csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" {
if !matchStringRange(v, minValue, maxValue) {
bm.resetBits()
@ -61,7 +62,7 @@ func (fr *filterStringRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
}
// Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName)
ch := csh.getColumnHeader(fieldName)
if ch == nil {
if !matchStringRange("", minValue, maxValue) {
bm.resetBits()