From 492190885d7fdab1cea5dc1a2a5199b3decba411 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 7 Oct 2024 12:18:18 +0200 Subject: [PATCH] app/vlogscli: add ability to display query results in logfmt, single-line and multi-line json modes --- app/vlogscli/json_prettifier.go | 124 +++++++++++++++++-------- app/vlogscli/main.go | 67 +++++++++++-- docs/VictoriaLogs/CHANGELOG.md | 1 + docs/VictoriaLogs/querying/vlogscli.md | 14 ++- 4 files changed, 159 insertions(+), 47 deletions(-) diff --git a/app/vlogscli/json_prettifier.go b/app/vlogscli/json_prettifier.go index e4f0edc5a..0fe4f6d5f 100644 --- a/app/vlogscli/json_prettifier.go +++ b/app/vlogscli/json_prettifier.go @@ -1,33 +1,70 @@ package main import ( + "bufio" "encoding/json" "fmt" "io" "sort" "sync" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage" ) +type outputMode int + +const ( + outputModeJSONMultiline = outputMode(0) + outputModeJSONSingleline = outputMode(1) + outputModeLogfmt = outputMode(2) +) + +func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error { + switch outputMode { + case outputModeJSONMultiline: + return func(w io.Writer, fields []logstorage.Field) error { + return writeJSONObject(w, fields, true) + } + case outputModeJSONSingleline: + return func(w io.Writer, fields []logstorage.Field) error { + return writeJSONObject(w, fields, false) + } + case outputModeLogfmt: + return writeLogfmtObject + default: + panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode)) + } +} + type jsonPrettifier struct { - rOriginal io.ReadCloser + r io.ReadCloser + formatter func(w io.Writer, fields []logstorage.Field) error d *json.Decoder pr *io.PipeReader pw *io.PipeWriter + bw *bufio.Writer wg sync.WaitGroup } -func newJSONPrettifier(r io.ReadCloser) *jsonPrettifier { +func newJSONPrettifier(r io.ReadCloser, outputMode outputMode) *jsonPrettifier { d := json.NewDecoder(r) pr, pw := io.Pipe() + bw := bufio.NewWriter(pw) + + formatter := getOutputFormatter(outputMode) jp := &jsonPrettifier{ - rOriginal: r, - d: d, - pr: pr, - pw: pw, + r: r, + formatter: formatter, + + d: d, + + pr: pr, + pw: pw, + bw: bw, } jp.wg.Add(1) @@ -47,20 +84,23 @@ func (jp *jsonPrettifier) closePipesWithError(err error) { func (jp *jsonPrettifier) prettifyJSONLines() error { for jp.d.More() { - kvs, err := readNextJSONObject(jp.d) + fields, err := readNextJSONObject(jp.d) if err != nil { return err } - if err := writeJSONObject(jp.pw, kvs); err != nil { + sort.Slice(fields, func(i, j int) bool { + return fields[i].Name < fields[j].Name + }) + if err := jp.formatter(jp.bw, fields); err != nil { return err } } - return nil + return jp.bw.Flush() } func (jp *jsonPrettifier) Close() error { jp.closePipesWithError(io.ErrUnexpectedEOF) - err := jp.rOriginal.Close() + err := jp.r.Close() jp.wg.Wait() return err } @@ -69,7 +109,7 @@ func (jp *jsonPrettifier) Read(p []byte) (int, error) { return jp.pr.Read(p) } -func readNextJSONObject(d *json.Decoder) ([]kv, error) { +func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) { t, err := d.Token() if err != nil { return nil, fmt.Errorf("cannot read '{': %w", err) @@ -79,7 +119,7 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) { return nil, fmt.Errorf("unexpected token read; got %q; want '{'", delim) } - var kvs []kv + var fields []logstorage.Field for { // Read object key t, err := d.Token() @@ -89,7 +129,7 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) { delim, ok := t.(json.Delim) if ok { if delim.String() == "}" { - return kvs, nil + return fields, nil } return nil, fmt.Errorf("unexpected delimiter read; got %q; want '}'", delim) } @@ -108,41 +148,56 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) { return nil, fmt.Errorf("unexpected token read for oject value: %v; want string", t) } - kvs = append(kvs, kv{ - key: key, - value: value, + fields = append(fields, logstorage.Field{ + Name: key, + Value: value, }) } } -func writeJSONObject(w io.Writer, kvs []kv) error { - if len(kvs) == 0 { +func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error { + data := logstorage.MarshalFieldsToLogfmt(nil, fields) + _, err := fmt.Fprintf(w, "%s\n", data) + return err +} + +func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error { + if len(fields) == 0 { fmt.Fprintf(w, "{}\n") return nil } - sort.Slice(kvs, func(i, j int) bool { - return kvs[i].key < kvs[j].key - }) - - fmt.Fprintf(w, "{\n") - if err := writeJSONObjectKeyValue(w, kvs[0]); err != nil { + fmt.Fprintf(w, "{") + writeNewlineIfNeeded(w, isMultiline) + if err := writeJSONObjectKeyValue(w, fields[0], isMultiline); err != nil { return err } - for _, kv := range kvs[1:] { - fmt.Fprintf(w, ",\n") - if err := writeJSONObjectKeyValue(w, kv); err != nil { + for _, f := range fields[1:] { + fmt.Fprintf(w, ",") + writeNewlineIfNeeded(w, isMultiline) + if err := writeJSONObjectKeyValue(w, f, isMultiline); err != nil { return err } } - fmt.Fprintf(w, "\n}\n") + writeNewlineIfNeeded(w, isMultiline) + fmt.Fprintf(w, "}\n") return nil } -func writeJSONObjectKeyValue(w io.Writer, kv kv) error { - key := getJSONString(kv.key) - value := getJSONString(kv.value) - _, err := fmt.Fprintf(w, " %s: %s", key, value) +func writeNewlineIfNeeded(w io.Writer, isMultiline bool) { + if isMultiline { + fmt.Fprintf(w, "\n") + } +} + +func writeJSONObjectKeyValue(w io.Writer, f logstorage.Field, isMultiline bool) error { + key := getJSONString(f.Name) + value := getJSONString(f.Value) + if isMultiline { + _, err := fmt.Fprintf(w, " %s: %s", key, value) + return err + } + _, err := fmt.Fprintf(w, "%s:%s", key, value) return err } @@ -153,8 +208,3 @@ func getJSONString(s string) string { } return string(data) } - -type kv struct { - key string - value string -} diff --git a/app/vlogscli/main.go b/app/vlogscli/main.go index 26c8d9d5b..055cea13f 100644 --- a/app/vlogscli/main.go +++ b/app/vlogscli/main.go @@ -86,6 +86,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) { } } + outputMode := outputModeJSONMultiline s := "" for { line, err := rl.ReadLine() @@ -94,7 +95,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) { case io.EOF: if s != "" { // This is non-interactive query execution. - if err := executeQuery(context.Background(), rl, s); err != nil { + if err := executeQuery(context.Background(), rl, s, outputMode); err != nil { fmt.Fprintf(rl, "%s\n", err) } } @@ -116,14 +117,43 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) { } s += line - if isQuitCommand(s) { - fmt.Fprintf(rl, "bye!\n") - return - } if s == "" { // Skip empty lines continue } + + if isQuitCommand(s) { + fmt.Fprintf(rl, "bye!\n") + _ = pushToHistory(rl, historyLines, s) + return + } + if isHelpCommand(s) { + printCommandsHelp(rl) + historyLines = pushToHistory(rl, historyLines, s) + s = "" + continue + } + if s == `\s` { + fmt.Fprintf(rl, "singleline json output mode\n") + outputMode = outputModeJSONSingleline + historyLines = pushToHistory(rl, historyLines, s) + s = "" + continue + } + if s == `\m` { + fmt.Fprintf(rl, "multiline json output mode\n") + outputMode = outputModeJSONMultiline + historyLines = pushToHistory(rl, historyLines, s) + s = "" + continue + } + if s == `\logfmt` { + fmt.Fprintf(rl, "logfmt output mode\n") + outputMode = outputModeLogfmt + historyLines = pushToHistory(rl, historyLines, s) + s = "" + continue + } if line != "" && !strings.HasSuffix(line, ";") { // Assume the query is incomplete and allow the user finishing the query on the next line s += "\n" @@ -133,7 +163,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) { // Execute the query ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) - err = executeQuery(ctx, rl, s) + err = executeQuery(ctx, rl, s, outputMode) cancel() if err != nil { @@ -205,14 +235,33 @@ func saveToHistory(filePath string, lines []string) error { func isQuitCommand(s string) bool { switch s { - case "q", "quit", "exit", "\\q": + case `\q`, "q", "quit", "exit": return true default: return false } } -func executeQuery(ctx context.Context, output io.Writer, s string) error { +func isHelpCommand(s string) bool { + switch s { + case `\h`, "h", "help", "?": + return true + default: + return false + } +} + +func printCommandsHelp(w io.Writer) { + fmt.Fprintf(w, "%s", `List of available commands: +\q - quit +\h - show this help +\s - singleline json output mode +\m - multiline json output mode +\logfmt - logfmt output mode +`) +} + +func executeQuery(ctx context.Context, output io.Writer, s string, outputMode outputMode) error { // Parse the query and convert it to canonical view. s = strings.TrimSuffix(s, ";") q, err := logstorage.ParseQuery(s) @@ -257,7 +306,7 @@ func executeQuery(ctx context.Context, output io.Writer, s string) error { } // Prettify the response and stream it to 'less'. - jp := newJSONPrettifier(resp.Body) + jp := newJSONPrettifier(resp.Body, outputMode) defer func() { _ = jp.Close() }() diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index e1a8b80df..a215ce8ff 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -15,6 +15,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip +* FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): add ability to display results in `logfmt` mode, single-line and multi-line JSON modes according [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/#output-modes). * FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): preserve `less` output after the exit from scrolling mode. This should help re-using previous query results in subsequent queries. * FEATURE: add [`len` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#len-pipe) for calculating the length for the given [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) value in bytes. diff --git a/docs/VictoriaLogs/querying/vlogscli.md b/docs/VictoriaLogs/querying/vlogscli.md index 09ef673ea..393d8e726 100644 --- a/docs/VictoriaLogs/querying/vlogscli.md +++ b/docs/VictoriaLogs/querying/vlogscli.md @@ -57,7 +57,8 @@ duration: 0.688s Query execution can be interrupted at any time by pressing `Ctrl+C`. -Type `q`, `quit` or `exit` and then press `Enter` for exit from `vlogsql`. +Type `q` and then press `Enter` for exit from `vlogsql` (if you want to search for `q` [word](https://docs.victoriametrics.com/victorialogs/logsql/#word), +then just wrap it into quotes: `"q"` or `'q'`). If the query response exceeds vertical screen space, `vlogsql` pipes query response to `less` utility, so you can scroll the response as needed. This allows executing queries, which potentially @@ -88,3 +89,14 @@ Press `Ctrl+R` multiple times for searching other matching queries in the histor Press `Enter` when the needed query is found in order to execute it. Press `Ctrl+C` for exit from the `search history` mode. See also [other available shortcuts](https://github.com/chzyer/readline/blob/f533ef1caae91a1fcc90875ff9a5a030f0237c6a/doc/shortcut.md). + +## Output modes + +By default `vlogscli` displays query results as prettified JSON object with every field on a separate line. +Fields in every JSON object are sorted in alphabetical order. This simplifies locating the needed fields. + +`vlogscli` supports the following output modes: + +* A single JSON line per every result. Type `\s` and press `enter` for this mode. +* Multline JSON per every result. Type `\m` and press `enter` for this mode. +* [Logfmt output](https://brandur.org/logfmt). Type `\logfmt` and press `enter` for this mode.