lib/logstorage: drop all the pipes from the query when calculating the number of matching logs at /select/logsql/hits API

This commit is contained in:
Aliaksandr Valialkin 2024-07-10 00:39:16 +02:00
parent 3c02937a34
commit aa9bb99527
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
4 changed files with 32 additions and 2 deletions

View file

@ -70,9 +70,10 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
fieldsLimit = 0 fieldsLimit = 0
} }
// Prepare the query // Prepare the query for hits count.
q.AddCountByTimePipe(int64(step), int64(offset), fields)
q.Optimize() q.Optimize()
q.DropAllPipes()
q.AddCountByTimePipe(int64(step), int64(offset), fields)
var mLock sync.Mutex var mLock sync.Mutex
m := make(map[string]*hitsSeries) m := make(map[string]*hitsSeries)

View file

@ -20,6 +20,8 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip ## tip
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): show a spinner on top of bar chart until user's request is finished. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6558). * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): show a spinner on top of bar chart until user's request is finished. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6558).
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): use compact representation of JSON lines at `JSON` tab if only a single [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) is queried.
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): properly show the number of matching logs on the selected time range at bar chart for queries with arbitrary [pipes](https://docs.victoriametrics.com/victorialogs/logsql/#pipes), including [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe) and [`top` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#top-pipe).
## [v0.27.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.27.1-victorialogs) ## [v0.27.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.27.1-victorialogs)

View file

@ -281,6 +281,11 @@ func getStreamIDsFromFilterOr(f filter) ([]streamID, bool) {
} }
} }
// DropAllPipes drops all the pipes from q.
func (q *Query) DropAllPipes() {
q.pipes = nil
}
// AddCountByTimePipe adds '| stats by (_time:step offset off, field1, ..., fieldN) count() hits' to the end of q. // AddCountByTimePipe adds '| stats by (_time:step offset off, field1, ..., fieldN) count() hits' to the end of q.
func (q *Query) AddCountByTimePipe(step, off int64, fields []string) { func (q *Query) AddCountByTimePipe(step, off int64, fields []string) {
{ {

View file

@ -2078,3 +2078,25 @@ func TestQueryCanLiveTail(t *testing.T) {
f("* | unpack_syslog", true) f("* | unpack_syslog", true)
f("* | unroll by (a)", true) f("* | unroll by (a)", true)
} }
func TestQueryDropAllPipes(t *testing.T) {
f := func(qStr, resultExpected string) {
t.Helper()
q, err := ParseQuery(qStr)
if err != nil {
t.Fatalf("cannot parse [%s]: %s", qStr, err)
}
q.Optimize()
q.DropAllPipes()
result := q.String()
if result != resultExpected {
t.Fatalf("unexpected result\ngot\n%s\nwant\n%s", result, resultExpected)
}
}
f(`*`, `*`)
f(`foo | stats count()`, `foo`)
f(`foo or bar and baz | top 5 by (x)`, `foo or bar baz`)
f(`foo | filter bar:baz | stats by (x) min(y)`, `foo bar:baz`)
}