VictoriaMetrics/lib/logstorage/pipe_unpack_json.go

110 lines
2.5 KiB
Go
Raw Normal View History

2024-05-19 19:25:52 +00:00
package logstorage
import (
"fmt"
2024-05-20 12:09:39 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
2024-05-19 19:25:52 +00:00
)
// pipeUnpackJSON processes '| unpack_json ...' pipe.
//
// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_json-pipe
type pipeUnpackJSON struct {
2024-05-21 10:55:11 +00:00
// fromField is the field to unpack json fields from
2024-05-19 19:25:52 +00:00
fromField string
2024-05-21 10:55:11 +00:00
// resultPrefix is prefix to add to unpacked field names
2024-05-19 19:25:52 +00:00
resultPrefix string
2024-05-21 10:55:11 +00:00
// iff is an optional filter for skipping unpacking json
iff *ifFilter
2024-05-19 19:25:52 +00:00
}
func (pu *pipeUnpackJSON) String() string {
s := "unpack_json"
if !isMsgFieldName(pu.fromField) {
s += " from " + quoteTokenIfNeeded(pu.fromField)
}
if pu.resultPrefix != "" {
2024-05-19 21:27:52 +00:00
s += " result_prefix " + quoteTokenIfNeeded(pu.resultPrefix)
2024-05-19 19:25:52 +00:00
}
2024-05-21 10:55:11 +00:00
if pu.iff != nil {
s += " " + pu.iff.String()
}
2024-05-19 19:25:52 +00:00
return s
}
func (pu *pipeUnpackJSON) updateNeededFields(neededFields, unneededFields fieldsSet) {
if neededFields.contains("*") {
unneededFields.remove(pu.fromField)
2024-05-21 10:55:11 +00:00
if pu.iff != nil {
unneededFields.removeFields(pu.iff.neededFields)
}
2024-05-19 19:25:52 +00:00
} else {
neededFields.add(pu.fromField)
2024-05-21 10:55:11 +00:00
if pu.iff != nil {
neededFields.addFields(pu.iff.neededFields)
}
2024-05-19 19:25:52 +00:00
}
}
2024-05-19 19:28:11 +00:00
func (pu *pipeUnpackJSON) newPipeProcessor(workersCount int, _ <-chan struct{}, _ func(), ppBase pipeProcessor) pipeProcessor {
2024-05-21 10:55:11 +00:00
return newPipeUnpackProcessor(workersCount, unpackJSON, ppBase, pu.fromField, pu.resultPrefix, pu.iff)
2024-05-19 19:25:52 +00:00
}
2024-05-20 12:09:39 +00:00
func unpackJSON(uctx *fieldsUnpackerContext, s, fieldPrefix string) {
if len(s) == 0 || s[0] != '{' {
2024-05-20 00:41:03 +00:00
// This isn't a JSON object
2024-05-19 19:25:52 +00:00
return
}
2024-05-20 12:09:39 +00:00
p := GetJSONParser()
if err := p.ParseLogMessage(bytesutil.ToUnsafeBytes(s)); err == nil {
for _, f := range p.Fields {
uctx.addField(f.Name, f.Value, fieldPrefix)
2024-05-19 19:25:52 +00:00
}
}
2024-05-20 12:09:39 +00:00
PutJSONParser(p)
2024-05-19 19:25:52 +00:00
}
func parsePipeUnpackJSON(lex *lexer) (*pipeUnpackJSON, error) {
if !lex.isKeyword("unpack_json") {
return nil, fmt.Errorf("unexpected token: %q; want %q", lex.token, "unpack_json")
}
lex.nextToken()
fromField := "_msg"
if lex.isKeyword("from") {
lex.nextToken()
f, err := parseFieldName(lex)
if err != nil {
return nil, fmt.Errorf("cannot parse 'from' field name: %w", err)
}
fromField = f
}
resultPrefix := ""
if lex.isKeyword("result_prefix") {
lex.nextToken()
p, err := getCompoundToken(lex)
if err != nil {
return nil, fmt.Errorf("cannot parse 'result_prefix': %w", err)
}
resultPrefix = p
}
pu := &pipeUnpackJSON{
fromField: fromField,
resultPrefix: resultPrefix,
}
2024-05-21 10:55:11 +00:00
if lex.isKeyword("if") {
iff, err := parseIfFilter(lex)
if err != nil {
return nil, err
}
pu.iff = iff
}
2024-05-19 19:25:52 +00:00
return pu, nil
}