From 507b206a7de2b088ec1a6b46d80d6ce34acc08be Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sun, 13 Oct 2024 14:28:59 +0200 Subject: [PATCH] lib/logstorage: move getConstColumnValue() and getColumnHeader() methods from columnsHeader to blockSearch This localizes blockSearch.getColumnsHeader() call at block_search.go . This call is going to be optimized in the next commits in order to avoid unmarshaling of header data for unneeded columns, which weren't requested by getConstColumnValue() / getColumnHeader(). --- lib/logstorage/block_header.go | 28 ----------------- lib/logstorage/block_result.go | 20 ++++++------ lib/logstorage/block_search.go | 40 ++++++++++++++++++++++++ lib/logstorage/filter_and.go | 5 ++- lib/logstorage/filter_any_case_phrase.go | 5 ++- lib/logstorage/filter_any_case_prefix.go | 5 ++- lib/logstorage/filter_exact.go | 5 ++- lib/logstorage/filter_exact_prefix.go | 5 ++- lib/logstorage/filter_in.go | 5 ++- lib/logstorage/filter_ipv4_range.go | 5 ++- lib/logstorage/filter_len_range.go | 5 ++- lib/logstorage/filter_or.go | 5 ++- lib/logstorage/filter_phrase.go | 5 ++- lib/logstorage/filter_prefix.go | 5 ++- lib/logstorage/filter_range.go | 5 ++- lib/logstorage/filter_regexp.go | 5 ++- lib/logstorage/filter_sequence.go | 5 ++- lib/logstorage/filter_string_range.go | 5 ++- 18 files changed, 79 insertions(+), 84 deletions(-) diff --git a/lib/logstorage/block_header.go b/lib/logstorage/block_header.go index a2c771e56..96ae05282 100644 --- a/lib/logstorage/block_header.go +++ b/lib/logstorage/block_header.go @@ -235,34 +235,6 @@ func (csh *columnsHeader) reset() { csh.constColumns = ccs[:0] } -func (csh *columnsHeader) getConstColumnValue(name string) string { - if name == "_msg" { - name = "" - } - ccs := csh.constColumns - for i := range ccs { - cc := &ccs[i] - if cc.Name == name { - return cc.Value - } - } - return "" -} - -func (csh *columnsHeader) getColumnHeader(name string) *columnHeader { - if name == "_msg" { - name = "" - } - chs := csh.columnHeaders - for i := range chs { - ch := &chs[i] - if ch.name == name { - return ch - } - } - return nil -} - func (csh *columnsHeader) resizeConstColumns(columnsLen int) []Field { csh.constColumns = slicesutil.SetLength(csh.constColumns, columnsLen) return csh.constColumns diff --git a/lib/logstorage/block_result.go b/lib/logstorage/block_result.go index c95fe8597..8eb2b1c72 100644 --- a/lib/logstorage/block_result.go +++ b/lib/logstorage/block_result.go @@ -305,11 +305,10 @@ func (br *blockResult) initAllColumns() { if !slices.Contains(unneededColumnNames, "_msg") { // Add _msg column - csh := br.bs.getColumnsHeader() - v := csh.getConstColumnValue("_msg") + v := br.bs.getConstColumnValue("_msg") if v != "" { br.addConstColumn("_msg", v) - } else if ch := csh.getColumnHeader("_msg"); ch != nil { + } else if ch := br.bs.getColumnHeader("_msg"); ch != nil { br.addColumn(ch) } else { br.addConstColumn("_msg", "") @@ -317,9 +316,9 @@ func (br *blockResult) initAllColumns() { } // Add other const columns - csh := br.bs.getColumnsHeader() - for _, cc := range csh.constColumns { - if isMsgFieldName(cc.Name) { + ccs := br.bs.getConstColumns() + for _, cc := range ccs { + if cc.Name == "" { continue } if !slices.Contains(unneededColumnNames, cc.Name) { @@ -328,10 +327,10 @@ func (br *blockResult) initAllColumns() { } // Add other non-const columns - chs := csh.columnHeaders + chs := br.bs.getColumnHeaders() for i := range chs { ch := &chs[i] - if isMsgFieldName(ch.name) { + if ch.name == "" { continue } if !slices.Contains(unneededColumnNames, ch.name) { @@ -357,11 +356,10 @@ func (br *blockResult) initRequestedColumns() { case "_time": br.addTimeColumn() default: - csh := br.bs.getColumnsHeader() - v := csh.getConstColumnValue(columnName) + v := br.bs.getConstColumnValue(columnName) if v != "" { br.addConstColumn(columnName, v) - } else if ch := csh.getColumnHeader(columnName); ch != nil { + } else if ch := br.bs.getColumnHeader(columnName); ch != nil { br.addColumn(ch) } else { br.addConstColumn(columnName, "") diff --git a/lib/logstorage/block_search.go b/lib/logstorage/block_search.go index 3b4ed9ed3..d954bf0fb 100644 --- a/lib/logstorage/block_search.go +++ b/lib/logstorage/block_search.go @@ -190,6 +190,46 @@ func (bs *blockSearch) search(bsw *blockSearchWork, bm *bitmap) { } } +func (bs *blockSearch) getConstColumnValue(name string) string { + if name == "_msg" { + name = "" + } + + csh := bs.getColumnsHeader() + for _, cc := range csh.constColumns { + if cc.Name == name { + return cc.Value + } + } + return "" +} + +func (bs *blockSearch) getColumnHeader(name string) *columnHeader { + if name == "_msg" { + name = "" + } + + csh := bs.getColumnsHeader() + chs := csh.columnHeaders + for i := range chs { + ch := &chs[i] + if ch.name == name { + return ch + } + } + return nil +} + +func (bs *blockSearch) getConstColumns() []Field { + csh := bs.getColumnsHeader() + return csh.constColumns +} + +func (bs *blockSearch) getColumnHeaders() []columnHeader { + csh := bs.getColumnsHeader() + return csh.columnHeaders +} + func (bs *blockSearch) getColumnsHeader() *columnsHeader { if bs.cshCache == nil { bs.cshBlockCache = readColumnsHeaderBlock(bs.cshBlockCache[:0], bs.bsw.p, &bs.bsw.bh) diff --git a/lib/logstorage/filter_and.go b/lib/logstorage/filter_and.go index 3d0ac2091..624cb9c59 100644 --- a/lib/logstorage/filter_and.go +++ b/lib/logstorage/filter_and.go @@ -81,8 +81,7 @@ func (fa *filterAnd) matchBloomFilters(bs *blockSearch) bool { fieldName := ft.field tokens := ft.tokens - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if matchStringByAllTokens(v, tokens) { continue @@ -90,7 +89,7 @@ func (fa *filterAnd) matchBloomFilters(bs *blockSearch) bool { return false } - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { return false } diff --git a/lib/logstorage/filter_any_case_phrase.go b/lib/logstorage/filter_any_case_phrase.go index 63f2ee338..d1d54feae 100644 --- a/lib/logstorage/filter_any_case_phrase.go +++ b/lib/logstorage/filter_any_case_phrase.go @@ -86,8 +86,7 @@ func (fp *filterAnyCasePhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) { phraseLowercase := fp.getPhraseLowercase() // Verify whether fp matches const column - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !matchAnyCasePhrase(v, phraseLowercase) { bm.resetBits() @@ -96,7 +95,7 @@ func (fp *filterAnyCasePhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether fp matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. // It matches anything only for empty phrase. diff --git a/lib/logstorage/filter_any_case_prefix.go b/lib/logstorage/filter_any_case_prefix.go index 1e569612d..c471b6f3f 100644 --- a/lib/logstorage/filter_any_case_prefix.go +++ b/lib/logstorage/filter_any_case_prefix.go @@ -90,8 +90,7 @@ func (fp *filterAnyCasePrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) { prefixLowercase := fp.getPrefixLowercase() // Verify whether fp matches const column - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !matchAnyCasePrefix(v, prefixLowercase) { bm.resetBits() @@ -100,7 +99,7 @@ func (fp *filterAnyCasePrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether fp matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. bm.resetBits() diff --git a/lib/logstorage/filter_exact.go b/lib/logstorage/filter_exact.go index 3d7359047..d68f96b62 100644 --- a/lib/logstorage/filter_exact.go +++ b/lib/logstorage/filter_exact.go @@ -174,8 +174,7 @@ func (fe *filterExact) applyToBlockSearch(bs *blockSearch, bm *bitmap) { fieldName := fe.fieldName value := fe.value - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if value != v { bm.resetBits() @@ -184,7 +183,7 @@ func (fe *filterExact) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether filter matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. // It matches anything only for empty value. diff --git a/lib/logstorage/filter_exact_prefix.go b/lib/logstorage/filter_exact_prefix.go index bf894166f..efc75d007 100644 --- a/lib/logstorage/filter_exact_prefix.go +++ b/lib/logstorage/filter_exact_prefix.go @@ -51,8 +51,7 @@ func (fep *filterExactPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) { fieldName := fep.fieldName prefix := fep.prefix - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !matchExactPrefix(v, prefix) { bm.resetBits() @@ -61,7 +60,7 @@ func (fep *filterExactPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether filter matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. if !matchExactPrefix("", prefix) { diff --git a/lib/logstorage/filter_in.go b/lib/logstorage/filter_in.go index 0c062f28d..fa3855e7e 100644 --- a/lib/logstorage/filter_in.go +++ b/lib/logstorage/filter_in.go @@ -358,8 +358,7 @@ func (fi *filterIn) applyToBlockSearch(bs *blockSearch, bm *bitmap) { return } - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { stringValues := fi.getStringValues() if _, ok := stringValues[v]; !ok { @@ -369,7 +368,7 @@ func (fi *filterIn) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether filter matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. // It matches anything only for empty phrase. diff --git a/lib/logstorage/filter_ipv4_range.go b/lib/logstorage/filter_ipv4_range.go index a382cd5de..8c9b6fb57 100644 --- a/lib/logstorage/filter_ipv4_range.go +++ b/lib/logstorage/filter_ipv4_range.go @@ -102,8 +102,7 @@ func (fr *filterIPv4Range) applyToBlockSearch(bs *blockSearch, bm *bitmap) { return } - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !matchIPv4Range(v, minValue, maxValue) { bm.resetBits() @@ -112,7 +111,7 @@ func (fr *filterIPv4Range) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether filter matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. bm.resetBits() diff --git a/lib/logstorage/filter_len_range.go b/lib/logstorage/filter_len_range.go index b80ab877e..d6fb7dad0 100644 --- a/lib/logstorage/filter_len_range.go +++ b/lib/logstorage/filter_len_range.go @@ -125,8 +125,7 @@ func (fr *filterLenRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) { return } - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !matchLenRange(v, minLen, maxLen) { bm.resetBits() @@ -135,7 +134,7 @@ func (fr *filterLenRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether filter matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. if !matchLenRange("", minLen, maxLen) { diff --git a/lib/logstorage/filter_or.go b/lib/logstorage/filter_or.go index 7e67717ef..1987ee9f4 100644 --- a/lib/logstorage/filter_or.go +++ b/lib/logstorage/filter_or.go @@ -93,8 +93,7 @@ func (fo *filterOr) matchBloomFilters(bs *blockSearch) bool { fieldName := ft.field tokens := ft.tokens - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if matchStringByAllTokens(v, tokens) { return true @@ -102,7 +101,7 @@ func (fo *filterOr) matchBloomFilters(bs *blockSearch) bool { continue } - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { continue } diff --git a/lib/logstorage/filter_phrase.go b/lib/logstorage/filter_phrase.go index 676711b52..f25f1e971 100644 --- a/lib/logstorage/filter_phrase.go +++ b/lib/logstorage/filter_phrase.go @@ -61,8 +61,7 @@ func (fp *filterPhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) { phrase := fp.phrase // Verify whether fp matches const column - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !matchPhrase(v, phrase) { bm.resetBits() @@ -71,7 +70,7 @@ func (fp *filterPhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether fp matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. // It matches anything only for empty phrase. diff --git a/lib/logstorage/filter_prefix.go b/lib/logstorage/filter_prefix.go index a6a086c00..50897a090 100644 --- a/lib/logstorage/filter_prefix.go +++ b/lib/logstorage/filter_prefix.go @@ -59,8 +59,7 @@ func (fp *filterPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) { prefix := fp.prefix // Verify whether fp matches const column - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !matchPrefix(v, prefix) { bm.resetBits() @@ -69,7 +68,7 @@ func (fp *filterPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether fp matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. bm.resetBits() diff --git a/lib/logstorage/filter_range.go b/lib/logstorage/filter_range.go index b173c2deb..a4ed1e936 100644 --- a/lib/logstorage/filter_range.go +++ b/lib/logstorage/filter_range.go @@ -173,8 +173,7 @@ func (fr *filterRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) { return } - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !matchRange(v, minValue, maxValue) { bm.resetBits() @@ -183,7 +182,7 @@ func (fr *filterRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether filter matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. bm.resetBits() diff --git a/lib/logstorage/filter_regexp.go b/lib/logstorage/filter_regexp.go index de9121ea1..1f71ea9c9 100644 --- a/lib/logstorage/filter_regexp.go +++ b/lib/logstorage/filter_regexp.go @@ -78,8 +78,7 @@ func (fr *filterRegexp) applyToBlockSearch(bs *blockSearch, bm *bitmap) { re := fr.re // Verify whether filter matches const column - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !re.MatchString(v) { bm.resetBits() @@ -88,7 +87,7 @@ func (fr *filterRegexp) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether filter matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. if !re.MatchString("") { diff --git a/lib/logstorage/filter_sequence.go b/lib/logstorage/filter_sequence.go index fd8cdccc5..2eec23a58 100644 --- a/lib/logstorage/filter_sequence.go +++ b/lib/logstorage/filter_sequence.go @@ -87,8 +87,7 @@ func (fs *filterSequence) applyToBlockSearch(bs *blockSearch, bm *bitmap) { return } - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !matchSequence(v, phrases) { bm.resetBits() @@ -97,7 +96,7 @@ func (fs *filterSequence) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether filter matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { // Fast path - there are no matching columns. // It matches anything only for empty phrase. diff --git a/lib/logstorage/filter_string_range.go b/lib/logstorage/filter_string_range.go index 071ade0be..988aced50 100644 --- a/lib/logstorage/filter_string_range.go +++ b/lib/logstorage/filter_string_range.go @@ -52,8 +52,7 @@ func (fr *filterStringRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) { return } - csh := bs.getColumnsHeader() - v := csh.getConstColumnValue(fieldName) + v := bs.getConstColumnValue(fieldName) if v != "" { if !matchStringRange(v, minValue, maxValue) { bm.resetBits() @@ -62,7 +61,7 @@ func (fr *filterStringRange) applyToBlockSearch(bs *blockSearch, bm *bitmap) { } // Verify whether filter matches other columns - ch := csh.getColumnHeader(fieldName) + ch := bs.getColumnHeader(fieldName) if ch == nil { if !matchStringRange("", minValue, maxValue) { bm.resetBits()