github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-02-09 15:27:11 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/logstorage: lazily read column headers metadata during queries

Browse source 
This improves performance for analytical queries, which do not need column headers metadata.
For example, the following query doesn't need column headers metadata, since _stream and min(_time)
are stored in block header, which is read separately from colum headers metadata:

  _time:1w | stats by (_stream) min(_time) min_time

This commit significantly improves the performance for this query.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7070
This commit is contained in:
Aliaksandr Valialkin 2024-09-25 16:59:58 +02:00
parent 4599429f51
commit 65b93b17b1
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
17 changed files with 73 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
lib/logstorage/block_result.go
View file

@ -305,10 +305,11 @@ func (br *blockResult) initAllColumns() {
if !slices.Contains(unneededColumnNames, "_msg") { if !slices.Contains(unneededColumnNames, "_msg") {
// Add _msg column // Add _msg column
v := br.bs.csh.getConstColumnValue("_msg") csh := br.bs.getColumnsHeader()
v := csh.getConstColumnValue("_msg")
if v != "" { if v != "" {
br.addConstColumn("_msg", v) br.addConstColumn("_msg", v)
} else if ch := br.bs.csh.getColumnHeader("_msg"); ch != nil { } else if ch := csh.getColumnHeader("_msg"); ch != nil {
br.addColumn(ch) br.addColumn(ch)
} else { } else {
br.addConstColumn("_msg", "") br.addConstColumn("_msg", "")
@ -316,7 +317,8 @@ func (br *blockResult) initAllColumns() {
} }
// Add other const columns // Add other const columns
for _, cc := range br.bs.csh.constColumns { csh := br.bs.getColumnsHeader()
for _, cc := range csh.constColumns {
if isMsgFieldName(cc.Name) { if isMsgFieldName(cc.Name) {
continue continue
} }
@ -326,7 +328,7 @@ func (br *blockResult) initAllColumns() {
} }
// Add other non-const columns // Add other non-const columns
chs := br.bs.csh.columnHeaders chs := csh.columnHeaders
for i := range chs { for i := range chs {
ch := &chs[i] ch := &chs[i]
if isMsgFieldName(ch.name) { if isMsgFieldName(ch.name) {
@ -355,10 +357,11 @@ func (br *blockResult) initRequestedColumns() {
case "_time": case "_time":
br.addTimeColumn() br.addTimeColumn()
default: default:
v := br.bs.csh.getConstColumnValue(columnName) csh := br.bs.getColumnsHeader()
v := csh.getConstColumnValue(columnName)
if v != "" { if v != "" {
br.addConstColumn(columnName, v) br.addConstColumn(columnName, v)
} else if ch := br.bs.csh.getColumnHeader(columnName); ch != nil { } else if ch := csh.getColumnHeader(columnName); ch != nil {
br.addColumn(ch) br.addColumn(ch)
} else { } else {
br.addConstColumn(columnName, "") br.addConstColumn(columnName, "")

25
lib/logstorage/block_search.go
View file

@ -113,10 +113,15 @@ type blockSearch struct {
// sbu is used for unmarshaling local columns // sbu is used for unmarshaling local columns
sbu stringsBlockUnmarshaler sbu stringsBlockUnmarshaler
// csh is the columnsHeader associated with the given block // cshCached is the columnsHeader associated with the given block
csh columnsHeader //
// it is initialized lazily by calling getColumnsHeader().
cshCached columnsHeader
// a is used for storing unmarshaled data in csh // cshInitialized is set to true if cshCached is initialized.
cshInitialized bool
// a is used for storing unmarshaled data in cshCached
a arena a arena
// seenStreams contains seen streamIDs for the recent searches. // seenStreams contains seen streamIDs for the recent searches.
@ -146,7 +151,10 @@ func (bs *blockSearch) reset() {
} }
bs.sbu.reset() bs.sbu.reset()
bs.csh.reset()
bs.cshCached.reset()
bs.cshInitialized = false
bs.a.reset() bs.a.reset()
// Do not reset seenStreams, since its' lifetime is managed by blockResult.addStreamColumn() code. // Do not reset seenStreams, since its' lifetime is managed by blockResult.addStreamColumn() code.
@ -161,8 +169,6 @@ func (bs *blockSearch) search(bsw *blockSearchWork, bm *bitmap) {
bs.bsw = bsw bs.bsw = bsw
bs.csh.initFromBlockHeader(&bs.a, bsw.p, &bsw.bh)
// search rows matching the given filter // search rows matching the given filter
bm.init(int(bsw.bh.rowsCount)) bm.init(int(bsw.bh.rowsCount))
bm.setBits() bm.setBits()
@ -183,6 +189,13 @@ func (bs *blockSearch) search(bsw *blockSearchWork, bm *bitmap) {
} }
} }
func (bs *blockSearch) getColumnsHeader() *columnsHeader {
if !bs.cshInitialized {
bs.cshCached.initFromBlockHeader(&bs.a, bs.bsw.p, &bs.bsw.bh)
}
return &bs.cshCached
}
func (csh *columnsHeader) initFromBlockHeader(a *arena, p *part, bh *blockHeader) { func (csh *columnsHeader) initFromBlockHeader(a *arena, p *part, bh *blockHeader) {
bb := longTermBufPool.Get() bb := longTermBufPool.Get()
columnsHeaderSize := bh.columnsHeaderSize columnsHeaderSize := bh.columnsHeaderSize

5
lib/logstorage/filter_and.go
View file

@ -81,7 +81,8 @@ func (fa *filterAnd) matchBloomFilters(bs *blockSearch) bool {
fieldName := ft.field fieldName := ft.field
tokens := ft.tokens tokens := ft.tokens
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if matchStringByAllTokens(v, tokens) { if matchStringByAllTokens(v, tokens) {
continue continue
@ -89,7 +90,7 @@ func (fa *filterAnd) matchBloomFilters(bs *blockSearch) bool {
return false return false
} }
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
return false return false
} }

5
lib/logstorage/filter_any_case_phrase.go
View file

@ -86,7 +86,8 @@ func (fp *filterAnyCasePhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
phraseLowercase := fp.getPhraseLowercase() phraseLowercase := fp.getPhraseLowercase()
// Verify whether fp matches const column // Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchAnyCasePhrase(v, phraseLowercase) { if !matchAnyCasePhrase(v, phraseLowercase) {
bm.resetBits() bm.resetBits()
@ -95,7 +96,7 @@ func (fp *filterAnyCasePhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether fp matches other columns // Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
// It matches anything only for empty phrase. // It matches anything only for empty phrase.

5
lib/logstorage/filter_any_case_prefix.go
View file

@ -90,7 +90,8 @@ func (fp *filterAnyCasePrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
prefixLowercase := fp.getPrefixLowercase() prefixLowercase := fp.getPrefixLowercase()
// Verify whether fp matches const column // Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchAnyCasePrefix(v, prefixLowercase) { if !matchAnyCasePrefix(v, prefixLowercase) {
bm.resetBits() bm.resetBits()
@ -99,7 +100,7 @@ func (fp *filterAnyCasePrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether fp matches other columns // Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
bm.resetBits() bm.resetBits()

5
lib/logstorage/filter_exact.go
View file

@ -174,7 +174,8 @@ func (fe *filterExact) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
fieldName := fe.fieldName fieldName := fe.fieldName
value := fe.value value := fe.value
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if value != v { if value != v {
bm.resetBits() bm.resetBits()
@ -183,7 +184,7 @@ func (fe *filterExact) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether filter matches other columns // Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
// It matches anything only for empty value. // It matches anything only for empty value.

5
lib/logstorage/filter_exact_prefix.go
View file

@ -51,7 +51,8 @@ func (fep *filterExactPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
fieldName := fep.fieldName fieldName := fep.fieldName
prefix := fep.prefix prefix := fep.prefix
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchExactPrefix(v, prefix) { if !matchExactPrefix(v, prefix) {
bm.resetBits() bm.resetBits()
@ -60,7 +61,7 @@ func (fep *filterExactPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether filter matches other columns // Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
if !matchExactPrefix("", prefix) { if !matchExactPrefix("", prefix) {

5
lib/logstorage/filter_in.go
View file

@ -358,7 +358,8 @@ func (fi *filterIn) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return return
} }
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
stringValues := fi.getStringValues() stringValues := fi.getStringValues()
if _, ok := stringValues[v]; !ok { if _, ok := stringValues[v]; !ok {
@ -368,7 +369,7 @@ func (fi *filterIn) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether filter matches other columns // Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
// It matches anything only for empty phrase. // It matches anything only for empty phrase.

5
lib/logstorage/filter_ipv4_range.go
View file

@ -102,7 +102,8 @@ func (fr *filterIPv4Range) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return return
} }
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchIPv4Range(v, minValue, maxValue) { if !matchIPv4Range(v, minValue, maxValue) {
bm.resetBits() bm.resetBits()
@ -111,7 +112,7 @@ func (fr *filterIPv4Range) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether filter matches other columns // Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
bm.resetBits() bm.resetBits()

5
lib/logstorage/filter_len_range.go
View file

@ -125,7 +125,8 @@ func (fr *filterLenRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return return
} }
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchLenRange(v, minLen, maxLen) { if !matchLenRange(v, minLen, maxLen) {
bm.resetBits() bm.resetBits()
@ -134,7 +135,7 @@ func (fr *filterLenRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether filter matches other columns // Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
if !matchLenRange("", minLen, maxLen) { if !matchLenRange("", minLen, maxLen) {

5
lib/logstorage/filter_or.go
View file

@ -93,7 +93,8 @@ func (fo *filterOr) matchBloomFilters(bs *blockSearch) bool {
fieldName := ft.field fieldName := ft.field
tokens := ft.tokens tokens := ft.tokens
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if matchStringByAllTokens(v, tokens) { if matchStringByAllTokens(v, tokens) {
return true return true
@ -101,7 +102,7 @@ func (fo *filterOr) matchBloomFilters(bs *blockSearch) bool {
continue continue
} }
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
continue continue
} }

5
lib/logstorage/filter_phrase.go
View file

@ -61,7 +61,8 @@ func (fp *filterPhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
phrase := fp.phrase phrase := fp.phrase
// Verify whether fp matches const column // Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchPhrase(v, phrase) { if !matchPhrase(v, phrase) {
bm.resetBits() bm.resetBits()
@ -70,7 +71,7 @@ func (fp *filterPhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether fp matches other columns // Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
// It matches anything only for empty phrase. // It matches anything only for empty phrase.

5
lib/logstorage/filter_prefix.go
View file

@ -59,7 +59,8 @@ func (fp *filterPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
prefix := fp.prefix prefix := fp.prefix
// Verify whether fp matches const column // Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchPrefix(v, prefix) { if !matchPrefix(v, prefix) {
bm.resetBits() bm.resetBits()
@ -68,7 +69,7 @@ func (fp *filterPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether fp matches other columns // Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
bm.resetBits() bm.resetBits()

5
lib/logstorage/filter_range.go
View file

@ -173,7 +173,8 @@ func (fr *filterRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return return
} }
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchRange(v, minValue, maxValue) { if !matchRange(v, minValue, maxValue) {
bm.resetBits() bm.resetBits()
@ -182,7 +183,7 @@ func (fr *filterRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether filter matches other columns // Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
bm.resetBits() bm.resetBits()

5
lib/logstorage/filter_regexp.go
View file

@ -78,7 +78,8 @@ func (fr *filterRegexp) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
re := fr.re re := fr.re
// Verify whether filter matches const column // Verify whether filter matches const column
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !re.MatchString(v) { if !re.MatchString(v) {
bm.resetBits() bm.resetBits()
@ -87,7 +88,7 @@ func (fr *filterRegexp) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether filter matches other columns // Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
if !re.MatchString("") { if !re.MatchString("") {

5
lib/logstorage/filter_sequence.go
View file

@ -87,7 +87,8 @@ func (fs *filterSequence) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return return
} }
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchSequence(v, phrases) { if !matchSequence(v, phrases) {
bm.resetBits() bm.resetBits()
@ -96,7 +97,7 @@ func (fs *filterSequence) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether filter matches other columns // Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
// Fast path - there are no matching columns. // Fast path - there are no matching columns.
// It matches anything only for empty phrase. // It matches anything only for empty phrase.

5
lib/logstorage/filter_string_range.go
View file

@ -52,7 +52,8 @@ func (fr *filterStringRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
return return
} }
v := bs.csh.getConstColumnValue(fieldName) csh := bs.getColumnsHeader()
v := csh.getConstColumnValue(fieldName)
if v != "" { if v != "" {
if !matchStringRange(v, minValue, maxValue) { if !matchStringRange(v, minValue, maxValue) {
bm.resetBits() bm.resetBits()
@ -61,7 +62,7 @@ func (fr *filterStringRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
} }
// Verify whether filter matches other columns // Verify whether filter matches other columns
ch := bs.csh.getColumnHeader(fieldName) ch := csh.getColumnHeader(fieldName)
if ch == nil { if ch == nil {
if !matchStringRange("", minValue, maxValue) { if !matchStringRange("", minValue, maxValue) {
bm.resetBits() bm.resetBits()