2024-05-19 19:25:52 +00:00
|
|
|
package logstorage
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
2024-05-20 12:09:39 +00:00
|
|
|
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
2024-05-19 19:25:52 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// pipeUnpackJSON processes '| unpack_json ...' pipe.
|
|
|
|
//
|
|
|
|
// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_json-pipe
|
|
|
|
type pipeUnpackJSON struct {
|
2024-05-21 10:55:11 +00:00
|
|
|
// fromField is the field to unpack json fields from
|
2024-05-19 19:25:52 +00:00
|
|
|
fromField string
|
|
|
|
|
2024-05-21 10:55:11 +00:00
|
|
|
// resultPrefix is prefix to add to unpacked field names
|
2024-05-19 19:25:52 +00:00
|
|
|
resultPrefix string
|
2024-05-21 10:55:11 +00:00
|
|
|
|
|
|
|
// iff is an optional filter for skipping unpacking json
|
|
|
|
iff *ifFilter
|
2024-05-19 19:25:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (pu *pipeUnpackJSON) String() string {
|
|
|
|
s := "unpack_json"
|
|
|
|
if !isMsgFieldName(pu.fromField) {
|
|
|
|
s += " from " + quoteTokenIfNeeded(pu.fromField)
|
|
|
|
}
|
|
|
|
if pu.resultPrefix != "" {
|
2024-05-19 21:27:52 +00:00
|
|
|
s += " result_prefix " + quoteTokenIfNeeded(pu.resultPrefix)
|
2024-05-19 19:25:52 +00:00
|
|
|
}
|
2024-05-21 10:55:11 +00:00
|
|
|
if pu.iff != nil {
|
|
|
|
s += " " + pu.iff.String()
|
|
|
|
}
|
2024-05-19 19:25:52 +00:00
|
|
|
return s
|
|
|
|
}
|
|
|
|
|
|
|
|
func (pu *pipeUnpackJSON) updateNeededFields(neededFields, unneededFields fieldsSet) {
|
|
|
|
if neededFields.contains("*") {
|
|
|
|
unneededFields.remove(pu.fromField)
|
2024-05-21 10:55:11 +00:00
|
|
|
if pu.iff != nil {
|
|
|
|
unneededFields.removeFields(pu.iff.neededFields)
|
|
|
|
}
|
2024-05-19 19:25:52 +00:00
|
|
|
} else {
|
|
|
|
neededFields.add(pu.fromField)
|
2024-05-21 10:55:11 +00:00
|
|
|
if pu.iff != nil {
|
|
|
|
neededFields.addFields(pu.iff.neededFields)
|
|
|
|
}
|
2024-05-19 19:25:52 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-05-19 19:28:11 +00:00
|
|
|
func (pu *pipeUnpackJSON) newPipeProcessor(workersCount int, _ <-chan struct{}, _ func(), ppBase pipeProcessor) pipeProcessor {
|
2024-05-21 10:55:11 +00:00
|
|
|
return newPipeUnpackProcessor(workersCount, unpackJSON, ppBase, pu.fromField, pu.resultPrefix, pu.iff)
|
2024-05-19 19:25:52 +00:00
|
|
|
}
|
|
|
|
|
2024-05-20 12:09:39 +00:00
|
|
|
func unpackJSON(uctx *fieldsUnpackerContext, s, fieldPrefix string) {
|
|
|
|
if len(s) == 0 || s[0] != '{' {
|
2024-05-20 00:41:03 +00:00
|
|
|
// This isn't a JSON object
|
2024-05-19 19:25:52 +00:00
|
|
|
return
|
|
|
|
}
|
2024-05-20 12:09:39 +00:00
|
|
|
p := GetJSONParser()
|
|
|
|
if err := p.ParseLogMessage(bytesutil.ToUnsafeBytes(s)); err == nil {
|
|
|
|
for _, f := range p.Fields {
|
|
|
|
uctx.addField(f.Name, f.Value, fieldPrefix)
|
2024-05-19 19:25:52 +00:00
|
|
|
}
|
|
|
|
}
|
2024-05-20 12:09:39 +00:00
|
|
|
PutJSONParser(p)
|
2024-05-19 19:25:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func parsePipeUnpackJSON(lex *lexer) (*pipeUnpackJSON, error) {
|
|
|
|
if !lex.isKeyword("unpack_json") {
|
|
|
|
return nil, fmt.Errorf("unexpected token: %q; want %q", lex.token, "unpack_json")
|
|
|
|
}
|
|
|
|
lex.nextToken()
|
|
|
|
|
|
|
|
fromField := "_msg"
|
|
|
|
if lex.isKeyword("from") {
|
|
|
|
lex.nextToken()
|
|
|
|
f, err := parseFieldName(lex)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("cannot parse 'from' field name: %w", err)
|
|
|
|
}
|
|
|
|
fromField = f
|
|
|
|
}
|
|
|
|
|
|
|
|
resultPrefix := ""
|
|
|
|
if lex.isKeyword("result_prefix") {
|
|
|
|
lex.nextToken()
|
|
|
|
p, err := getCompoundToken(lex)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("cannot parse 'result_prefix': %w", err)
|
|
|
|
}
|
|
|
|
resultPrefix = p
|
|
|
|
}
|
|
|
|
|
|
|
|
pu := &pipeUnpackJSON{
|
|
|
|
fromField: fromField,
|
|
|
|
resultPrefix: resultPrefix,
|
|
|
|
}
|
2024-05-21 10:55:11 +00:00
|
|
|
|
|
|
|
if lex.isKeyword("if") {
|
|
|
|
iff, err := parseIfFilter(lex)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
pu.iff = iff
|
|
|
|
}
|
|
|
|
|
2024-05-19 19:25:52 +00:00
|
|
|
return pu, nil
|
|
|
|
}
|