From 8d0eb0c43c8e367fa3ca68ab9cd570ca4abae918 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 29 Apr 2024 06:17:14 +0200 Subject: [PATCH] wip --- lib/logstorage/filter.go | 90 -------------------------- lib/logstorage/filter_ipv4_range.go | 98 +++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 90 deletions(-) create mode 100644 lib/logstorage/filter_ipv4_range.go diff --git a/lib/logstorage/filter.go b/lib/logstorage/filter.go index 576908325..92d04ea3e 100644 --- a/lib/logstorage/filter.go +++ b/lib/logstorage/filter.go @@ -72,71 +72,6 @@ func (fs *streamFilter) apply(bs *blockSearch, bm *bitmap) { } } -// filterIPv4Range matches the given ipv4 range [minValue..maxValue]. -// -// Example LogsQL: `fieldName:ipv4_range(127.0.0.1, 127.0.0.255)` -type filterIPv4Range struct { - fieldName string - minValue uint32 - maxValue uint32 -} - -func (fr *filterIPv4Range) String() string { - minValue := string(encoding.MarshalUint32(nil, fr.minValue)) - maxValue := string(encoding.MarshalUint32(nil, fr.maxValue)) - return fmt.Sprintf("%sipv4_range(%s, %s)", quoteFieldNameIfNeeded(fr.fieldName), toIPv4String(nil, minValue), toIPv4String(nil, maxValue)) -} - -func (fr *filterIPv4Range) apply(bs *blockSearch, bm *bitmap) { - fieldName := fr.fieldName - minValue := fr.minValue - maxValue := fr.maxValue - - if minValue > maxValue { - bm.resetBits() - return - } - - v := bs.csh.getConstColumnValue(fieldName) - if v != "" { - if !matchIPv4Range(v, minValue, maxValue) { - bm.resetBits() - } - return - } - - // Verify whether filter matches other columns - ch := bs.csh.getColumnHeader(fieldName) - if ch == nil { - // Fast path - there are no matching columns. - bm.resetBits() - return - } - - switch ch.valueType { - case valueTypeString: - matchStringByIPv4Range(bs, ch, bm, minValue, maxValue) - case valueTypeDict: - matchValuesDictByIPv4Range(bs, ch, bm, minValue, maxValue) - case valueTypeUint8: - bm.resetBits() - case valueTypeUint16: - bm.resetBits() - case valueTypeUint32: - bm.resetBits() - case valueTypeUint64: - bm.resetBits() - case valueTypeFloat64: - bm.resetBits() - case valueTypeIPv4: - matchIPv4ByRange(bs, ch, bm, minValue, maxValue) - case valueTypeTimestampISO8601: - bm.resetBits() - default: - logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) - } -} - // stringRangeFilter matches tie given string range [minValue..maxValue) // // Note that the minValue is included in the range, while the maxValue isn't included in the range. @@ -1010,17 +945,6 @@ func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase bbPool.Put(bb) } -func matchValuesDictByIPv4Range(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue uint32) { - bb := bbPool.Get() - for i, v := range ch.valuesDict.values { - if matchIPv4Range(v, minValue, maxValue) { - bb.B = append(bb.B, byte(i)) - } - } - matchEncodedValuesDict(bs, ch, bm, bb.B) - bbPool.Put(bb) -} - func matchValuesDictByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { bb := bbPool.Get() for i, v := range ch.valuesDict.values { @@ -1136,12 +1060,6 @@ func matchEncodedValuesDict(bs *blockSearch, ch *columnHeader, bm *bitmap, encod }) } -func matchStringByIPv4Range(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue uint32) { - visitValues(bs, ch, bm, func(v string) bool { - return matchIPv4Range(v, minValue, maxValue) - }) -} - func matchStringByStringRange(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) { visitValues(bs, ch, bm, func(v string) bool { return matchStringRange(v, minValue, maxValue) @@ -1608,14 +1526,6 @@ func matchPrefix(s, prefix string) bool { } } -func matchIPv4Range(s string, minValue, maxValue uint32) bool { - n, ok := tryParseIPv4(s) - if !ok { - return false - } - return n >= minValue && n <= maxValue -} - func matchStringRange(s, minValue, maxValue string) bool { return s >= minValue && s < maxValue } diff --git a/lib/logstorage/filter_ipv4_range.go b/lib/logstorage/filter_ipv4_range.go new file mode 100644 index 000000000..ac3c97a3e --- /dev/null +++ b/lib/logstorage/filter_ipv4_range.go @@ -0,0 +1,98 @@ +package logstorage + +import ( + "fmt" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" +) + +// filterIPv4Range matches the given ipv4 range [minValue..maxValue]. +// +// Example LogsQL: `fieldName:ipv4_range(127.0.0.1, 127.0.0.255)` +type filterIPv4Range struct { + fieldName string + minValue uint32 + maxValue uint32 +} + +func (fr *filterIPv4Range) String() string { + minValue := string(encoding.MarshalUint32(nil, fr.minValue)) + maxValue := string(encoding.MarshalUint32(nil, fr.maxValue)) + return fmt.Sprintf("%sipv4_range(%s, %s)", quoteFieldNameIfNeeded(fr.fieldName), toIPv4String(nil, minValue), toIPv4String(nil, maxValue)) +} + +func (fr *filterIPv4Range) apply(bs *blockSearch, bm *bitmap) { + fieldName := fr.fieldName + minValue := fr.minValue + maxValue := fr.maxValue + + if minValue > maxValue { + bm.resetBits() + return + } + + v := bs.csh.getConstColumnValue(fieldName) + if v != "" { + if !matchIPv4Range(v, minValue, maxValue) { + bm.resetBits() + } + return + } + + // Verify whether filter matches other columns + ch := bs.csh.getColumnHeader(fieldName) + if ch == nil { + // Fast path - there are no matching columns. + bm.resetBits() + return + } + + switch ch.valueType { + case valueTypeString: + matchStringByIPv4Range(bs, ch, bm, minValue, maxValue) + case valueTypeDict: + matchValuesDictByIPv4Range(bs, ch, bm, minValue, maxValue) + case valueTypeUint8: + bm.resetBits() + case valueTypeUint16: + bm.resetBits() + case valueTypeUint32: + bm.resetBits() + case valueTypeUint64: + bm.resetBits() + case valueTypeFloat64: + bm.resetBits() + case valueTypeIPv4: + matchIPv4ByRange(bs, ch, bm, minValue, maxValue) + case valueTypeTimestampISO8601: + bm.resetBits() + default: + logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType) + } +} + +func matchValuesDictByIPv4Range(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue uint32) { + bb := bbPool.Get() + for i, v := range ch.valuesDict.values { + if matchIPv4Range(v, minValue, maxValue) { + bb.B = append(bb.B, byte(i)) + } + } + matchEncodedValuesDict(bs, ch, bm, bb.B) + bbPool.Put(bb) +} + +func matchStringByIPv4Range(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue uint32) { + visitValues(bs, ch, bm, func(v string) bool { + return matchIPv4Range(v, minValue, maxValue) + }) +} + +func matchIPv4Range(s string, minValue, maxValue uint32) bool { + n, ok := tryParseIPv4(s) + if !ok { + return false + } + return n >= minValue && n <= maxValue +}