diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index 3c35dbf01..1a83eefe6 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -1552,7 +1552,7 @@ func TestQueryGetNeededColumns(t *testing.T) { f(`* | fields f1, f2 | sort by(f3)`, `f1,f2`, ``) f(`* | fields f1, f2 | sort by(f1,f3)`, `f1,f2`, ``) f(`* | fields f1, f2 | sort by(f3) | stats count() r1`, ``, ``) - f(`* | fields f1, f2 | sort by(f1) | stats count() r1`, `f1`, ``) + f(`* | fields f1, f2 | sort by(f1) | stats count() r1`, ``, ``) f(`* | fields f1, f2 | sort by(f1) | stats count(f2,f3) r1`, `f1,f2`, ``) f(`* | fields f1, f2 | sort by(f3) | fields f2`, `f2`, ``) f(`* | fields f1, f2 | sort by(f1,f3) | fields f2`, `f1,f2`, ``) @@ -1783,17 +1783,41 @@ func TestQueryGetNeededColumns(t *testing.T) { f(`* | rm f1, f2 | stats by(f3) count(f4) r1`, `f3,f4`, ``) // Verify that fields are correctly tracked before count(*) - f(`* | copy a b, c d | count() r1`, `a`, ``) - f(`* | delete a, b | count() r1`, `*`, `a`) - f(`* | extract "bar" from x | count() r1`, `x`, ``) - f(`* | extract if (q:w p:a) "bar" from x | count() r1`, `p,q,x`, ``) - f(`* | extract_regexp "(?P.*)bar" from x | count() r1`, `x`, ``) - f(`* | extract_regexp if (q:w p:a) "(?P.*)bar" from x | count() r1`, `p,q,x`, ``) + f(`* | copy a b, c d | count() r1`, ``, ``) + f(`* | delete a, b | count() r1`, ``, ``) + f(`* | extract "bar" from x | count() r1`, ``, ``) + f(`* | extract if (q:w p:a) "bar" from x | count() r1`, `p,q`, ``) + f(`* | extract_regexp "(?P.*)bar" from x | count() r1`, ``, ``) + f(`* | extract_regexp if (q:w p:a) "(?P.*)bar" from x | count() r1`, `p,q`, ``) f(`* | field_names | count() r1`, `*`, `_time`) f(`* | limit 10 | field_names as abc | count() r1`, `*`, ``) - f(`* | fields a, b | count() r1`, `a`, ``) + f(`* | fields a, b | count() r1`, ``, ``) f(`* | field_values a | count() r1`, `a`, ``) f(`* | limit 10 | filter a:b c:d | count() r1`, `a,c`, ``) - f(`* | format "" as c | count() r1`, `a,b`, ``) - f(`* | format if (q:w p:a) "" as c | count() r1`, `a,b,p,q`, ``) + f(`* | limit 10 | count() r1`, ``, ``) + f(`* | format "" as c | count() r1`, ``, ``) + f(`* | format if (q:w p:a) "" as c | count() r1`, `p,q`, ``) + f(`* | math (a + b) as c, d * 2 as x | count() r1`, ``, ``) + f(`* | offset 10 | count() r1`, ``, ``) + f(`* | pack_json | count() r1`, ``, ``) + f(`* | pack_json fields(a,b) | count() r1`, ``, ``) + f(`* | rename a b, c d | count() r1`, ``, ``) + f(`* | replace ("a", "b") at x | count() r1`, ``, ``) + f(`* | replace if (q:w p:a) ("a", "b") at x | count() r1`, `p,q`, ``) + f(`* | replace_regexp ("a", "b") at x | count() r1`, ``, ``) + f(`* | replace_regexp if (q:w p:a) ("a", "b") at x | count() r1`, `p,q`, ``) + f(`* | sort by (a,b) | count() r1`, ``, ``) + f(`* | stats count_uniq(a, b) as c | count() r1`, ``, ``) + f(`* | stats count_uniq(a, b) if (q:w p:a) as c | count() r1`, ``, ``) + f(`* | stats by (a1,a2) count_uniq(a, b) as c | count() r1`, `a1,a2`, ``) + f(`* | stats by (a1,a2) count_uniq(a, b) if (q:w p:a) as c | count() r1`, `a1,a2`, ``) + f(`* | uniq by (a, b) | count() r1`, `a,b`, ``) + f(`* | unpack_json from x | count() r1`, ``, ``) + f(`* | unpack_json from x fields (a,b) | count() r1`, ``, ``) + f(`* | unpack_json if (q:w p:a) from x | count() r1`, `p,q`, ``) + f(`* | unpack_json if (q:w p:a) from x fields(a,b) | count() r1`, `p,q`, ``) + f(`* | unpack_logfmt from x | count() r1`, ``, ``) + f(`* | unpack_logfmt from x fields (a,b) | count() r1`, ``, ``) + f(`* | unpack_logfmt if (q:w p:a) from x | count() r1`, `p,q`, ``) + f(`* | unpack_logfmt if (q:w p:a) from x fields(a,b) | count() r1`, `p,q`, ``) } diff --git a/lib/logstorage/pipe_copy.go b/lib/logstorage/pipe_copy.go index c3d4ffdf1..00d91edca 100644 --- a/lib/logstorage/pipe_copy.go +++ b/lib/logstorage/pipe_copy.go @@ -32,11 +32,6 @@ func (pc *pipeCopy) String() string { } func (pc *pipeCopy) updateNeededFields(neededFields, unneededFields fieldsSet) { - if neededFields.isEmpty() { - neededFields.add(pc.srcFields[0]) - return - } - for i := len(pc.srcFields) - 1; i >= 0; i-- { srcField := pc.srcFields[i] dstField := pc.dstFields[i] diff --git a/lib/logstorage/pipe_delete.go b/lib/logstorage/pipe_delete.go index 2530fb39e..9f6b22e1e 100644 --- a/lib/logstorage/pipe_delete.go +++ b/lib/logstorage/pipe_delete.go @@ -23,12 +23,6 @@ func (pd *pipeDelete) String() string { } func (pd *pipeDelete) updateNeededFields(neededFields, unneededFields fieldsSet) { - if neededFields.isEmpty() { - neededFields.add("*") - unneededFields.add(pd.fields[0]) - return - } - if neededFields.contains("*") { unneededFields.addFields(pd.fields) } else { diff --git a/lib/logstorage/pipe_extract.go b/lib/logstorage/pipe_extract.go index 4591fa20f..99d21fad5 100644 --- a/lib/logstorage/pipe_extract.go +++ b/lib/logstorage/pipe_extract.go @@ -61,7 +61,6 @@ func (pe *pipeExtract) initFilterInValues(cache map[string][]string, getFieldVal func (pe *pipeExtract) updateNeededFields(neededFields, unneededFields fieldsSet) { if neededFields.isEmpty() { - neededFields.add(pe.fromField) if pe.iff != nil { neededFields.addFields(pe.iff.neededFields) } diff --git a/lib/logstorage/pipe_extract_regexp.go b/lib/logstorage/pipe_extract_regexp.go index 6f1a4b935..0df49dd7d 100644 --- a/lib/logstorage/pipe_extract_regexp.go +++ b/lib/logstorage/pipe_extract_regexp.go @@ -63,7 +63,6 @@ func (pe *pipeExtractRegexp) initFilterInValues(cache map[string][]string, getFi func (pe *pipeExtractRegexp) updateNeededFields(neededFields, unneededFields fieldsSet) { if neededFields.isEmpty() { - neededFields.add(pe.fromField) if pe.iff != nil { neededFields.addFields(pe.iff.neededFields) } diff --git a/lib/logstorage/pipe_fields.go b/lib/logstorage/pipe_fields.go index f9944432a..0c08d57a9 100644 --- a/lib/logstorage/pipe_fields.go +++ b/lib/logstorage/pipe_fields.go @@ -29,10 +29,6 @@ func (pf *pipeFields) updateNeededFields(neededFields, unneededFields fieldsSet) if pf.containsStar { return } - if neededFields.isEmpty() { - neededFields.add(pf.fields[0]) - return - } if neededFields.contains("*") { // subtract unneeded fields from pf.fields diff --git a/lib/logstorage/pipe_format.go b/lib/logstorage/pipe_format.go index fd85ff21e..aa91a0677 100644 --- a/lib/logstorage/pipe_format.go +++ b/lib/logstorage/pipe_format.go @@ -44,11 +44,6 @@ func (pf *pipeFormat) String() string { func (pf *pipeFormat) updateNeededFields(neededFields, unneededFields fieldsSet) { if neededFields.isEmpty() { - for _, step := range pf.steps { - if step.field != "" { - neededFields.add(step.field) - } - } if pf.iff != nil { neededFields.addFields(pf.iff.neededFields) } diff --git a/lib/logstorage/pipe_math.go b/lib/logstorage/pipe_math.go index 47f7ed73a..07b66f446 100644 --- a/lib/logstorage/pipe_math.go +++ b/lib/logstorage/pipe_math.go @@ -175,26 +175,19 @@ func (pm *pipeMath) updateNeededFields(neededFields, unneededFields fieldsSet) { if !unneededFields.contains(e.resultField) { unneededFields.add(e.resultField) - entryFields := e.getNeededFields() - unneededFields.removeFields(entryFields) + fs := newFieldsSet() + e.expr.updateNeededFields(fs) + unneededFields.removeFields(fs.getAll()) } } else { if neededFields.contains(e.resultField) { neededFields.remove(e.resultField) - - entryFields := e.getNeededFields() - neededFields.addFields(entryFields) + e.expr.updateNeededFields(neededFields) } } } } -func (me *mathEntry) getNeededFields() []string { - neededFields := newFieldsSet() - me.expr.updateNeededFields(neededFields) - return neededFields.getAll() -} - func (me *mathExpr) updateNeededFields(neededFields fieldsSet) { if me.isConst { return diff --git a/lib/logstorage/pipe_sort.go b/lib/logstorage/pipe_sort.go index 0b6d3cbbd..1bcee3670 100644 --- a/lib/logstorage/pipe_sort.go +++ b/lib/logstorage/pipe_sort.go @@ -56,6 +56,10 @@ func (ps *pipeSort) String() string { } func (ps *pipeSort) updateNeededFields(neededFields, unneededFields fieldsSet) { + if neededFields.isEmpty() { + return + } + if len(ps.byFields) == 0 { neededFields.add("*") unneededFields.reset() diff --git a/lib/logstorage/pipe_unpack.go b/lib/logstorage/pipe_unpack.go index fde04b369..fe2e36d78 100644 --- a/lib/logstorage/pipe_unpack.go +++ b/lib/logstorage/pipe_unpack.go @@ -7,6 +7,13 @@ import ( ) func updateNeededFieldsForUnpackPipe(fromField string, outFields []string, keepOriginalFields, skipEmptyResults bool, iff *ifFilter, neededFields, unneededFields fieldsSet) { + if neededFields.isEmpty() { + if iff != nil { + neededFields.addFields(iff.neededFields) + } + return + } + if neededFields.contains("*") { unneededFieldsOrig := unneededFields.clone() unneededFieldsCount := 0 diff --git a/lib/logstorage/pipe_update.go b/lib/logstorage/pipe_update.go index 718c2e37b..2684bd9a5 100644 --- a/lib/logstorage/pipe_update.go +++ b/lib/logstorage/pipe_update.go @@ -5,6 +5,13 @@ import ( ) func updateNeededFieldsForUpdatePipe(neededFields, unneededFields fieldsSet, field string, iff *ifFilter) { + if neededFields.isEmpty() { + if iff != nil { + neededFields.addFields(iff.neededFields) + } + return + } + if neededFields.contains("*") { if !unneededFields.contains(field) && iff != nil { unneededFields.removeFields(iff.neededFields)