From efe5935497bf80971347980dd401ba47acea8fd8 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Mon, 7 Oct 2024 12:18:18 +0200
Subject: [PATCH] app/vlogscli: add ability to display query results in logfmt,
single-line and multi-line json modes
(cherry picked from commit 492190885d7fdab1cea5dc1a2a5199b3decba411)
---
app/vlogscli/json_prettifier.go | 124 +++++++++++++++++--------
app/vlogscli/main.go | 67 +++++++++++--
docs/VictoriaLogs/CHANGELOG.md | 1 +
docs/VictoriaLogs/querying/vlogscli.md | 14 ++-
4 files changed, 159 insertions(+), 47 deletions(-)
diff --git a/app/vlogscli/json_prettifier.go b/app/vlogscli/json_prettifier.go
index e4f0edc5ac..0fe4f6d5f3 100644
--- a/app/vlogscli/json_prettifier.go
+++ b/app/vlogscli/json_prettifier.go
@@ -1,33 +1,70 @@
package main
import (
+ "bufio"
"encoding/json"
"fmt"
"io"
"sort"
"sync"
+
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
)
+type outputMode int
+
+const (
+ outputModeJSONMultiline = outputMode(0)
+ outputModeJSONSingleline = outputMode(1)
+ outputModeLogfmt = outputMode(2)
+)
+
+func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error {
+ switch outputMode {
+ case outputModeJSONMultiline:
+ return func(w io.Writer, fields []logstorage.Field) error {
+ return writeJSONObject(w, fields, true)
+ }
+ case outputModeJSONSingleline:
+ return func(w io.Writer, fields []logstorage.Field) error {
+ return writeJSONObject(w, fields, false)
+ }
+ case outputModeLogfmt:
+ return writeLogfmtObject
+ default:
+ panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode))
+ }
+}
+
type jsonPrettifier struct {
- rOriginal io.ReadCloser
+ r io.ReadCloser
+ formatter func(w io.Writer, fields []logstorage.Field) error
d *json.Decoder
pr *io.PipeReader
pw *io.PipeWriter
+ bw *bufio.Writer
wg sync.WaitGroup
}
-func newJSONPrettifier(r io.ReadCloser) *jsonPrettifier {
+func newJSONPrettifier(r io.ReadCloser, outputMode outputMode) *jsonPrettifier {
d := json.NewDecoder(r)
pr, pw := io.Pipe()
+ bw := bufio.NewWriter(pw)
+
+ formatter := getOutputFormatter(outputMode)
jp := &jsonPrettifier{
- rOriginal: r,
- d: d,
- pr: pr,
- pw: pw,
+ r: r,
+ formatter: formatter,
+
+ d: d,
+
+ pr: pr,
+ pw: pw,
+ bw: bw,
}
jp.wg.Add(1)
@@ -47,20 +84,23 @@ func (jp *jsonPrettifier) closePipesWithError(err error) {
func (jp *jsonPrettifier) prettifyJSONLines() error {
for jp.d.More() {
- kvs, err := readNextJSONObject(jp.d)
+ fields, err := readNextJSONObject(jp.d)
if err != nil {
return err
}
- if err := writeJSONObject(jp.pw, kvs); err != nil {
+ sort.Slice(fields, func(i, j int) bool {
+ return fields[i].Name < fields[j].Name
+ })
+ if err := jp.formatter(jp.bw, fields); err != nil {
return err
}
}
- return nil
+ return jp.bw.Flush()
}
func (jp *jsonPrettifier) Close() error {
jp.closePipesWithError(io.ErrUnexpectedEOF)
- err := jp.rOriginal.Close()
+ err := jp.r.Close()
jp.wg.Wait()
return err
}
@@ -69,7 +109,7 @@ func (jp *jsonPrettifier) Read(p []byte) (int, error) {
return jp.pr.Read(p)
}
-func readNextJSONObject(d *json.Decoder) ([]kv, error) {
+func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
t, err := d.Token()
if err != nil {
return nil, fmt.Errorf("cannot read '{': %w", err)
@@ -79,7 +119,7 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) {
return nil, fmt.Errorf("unexpected token read; got %q; want '{'", delim)
}
- var kvs []kv
+ var fields []logstorage.Field
for {
// Read object key
t, err := d.Token()
@@ -89,7 +129,7 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) {
delim, ok := t.(json.Delim)
if ok {
if delim.String() == "}" {
- return kvs, nil
+ return fields, nil
}
return nil, fmt.Errorf("unexpected delimiter read; got %q; want '}'", delim)
}
@@ -108,41 +148,56 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) {
return nil, fmt.Errorf("unexpected token read for oject value: %v; want string", t)
}
- kvs = append(kvs, kv{
- key: key,
- value: value,
+ fields = append(fields, logstorage.Field{
+ Name: key,
+ Value: value,
})
}
}
-func writeJSONObject(w io.Writer, kvs []kv) error {
- if len(kvs) == 0 {
+func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error {
+ data := logstorage.MarshalFieldsToLogfmt(nil, fields)
+ _, err := fmt.Fprintf(w, "%s\n", data)
+ return err
+}
+
+func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error {
+ if len(fields) == 0 {
fmt.Fprintf(w, "{}\n")
return nil
}
- sort.Slice(kvs, func(i, j int) bool {
- return kvs[i].key < kvs[j].key
- })
-
- fmt.Fprintf(w, "{\n")
- if err := writeJSONObjectKeyValue(w, kvs[0]); err != nil {
+ fmt.Fprintf(w, "{")
+ writeNewlineIfNeeded(w, isMultiline)
+ if err := writeJSONObjectKeyValue(w, fields[0], isMultiline); err != nil {
return err
}
- for _, kv := range kvs[1:] {
- fmt.Fprintf(w, ",\n")
- if err := writeJSONObjectKeyValue(w, kv); err != nil {
+ for _, f := range fields[1:] {
+ fmt.Fprintf(w, ",")
+ writeNewlineIfNeeded(w, isMultiline)
+ if err := writeJSONObjectKeyValue(w, f, isMultiline); err != nil {
return err
}
}
- fmt.Fprintf(w, "\n}\n")
+ writeNewlineIfNeeded(w, isMultiline)
+ fmt.Fprintf(w, "}\n")
return nil
}
-func writeJSONObjectKeyValue(w io.Writer, kv kv) error {
- key := getJSONString(kv.key)
- value := getJSONString(kv.value)
- _, err := fmt.Fprintf(w, " %s: %s", key, value)
+func writeNewlineIfNeeded(w io.Writer, isMultiline bool) {
+ if isMultiline {
+ fmt.Fprintf(w, "\n")
+ }
+}
+
+func writeJSONObjectKeyValue(w io.Writer, f logstorage.Field, isMultiline bool) error {
+ key := getJSONString(f.Name)
+ value := getJSONString(f.Value)
+ if isMultiline {
+ _, err := fmt.Fprintf(w, " %s: %s", key, value)
+ return err
+ }
+ _, err := fmt.Fprintf(w, "%s:%s", key, value)
return err
}
@@ -153,8 +208,3 @@ func getJSONString(s string) string {
}
return string(data)
}
-
-type kv struct {
- key string
- value string
-}
diff --git a/app/vlogscli/main.go b/app/vlogscli/main.go
index 26c8d9d5bb..055cea13ff 100644
--- a/app/vlogscli/main.go
+++ b/app/vlogscli/main.go
@@ -86,6 +86,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
}
}
+ outputMode := outputModeJSONMultiline
s := ""
for {
line, err := rl.ReadLine()
@@ -94,7 +95,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
case io.EOF:
if s != "" {
// This is non-interactive query execution.
- if err := executeQuery(context.Background(), rl, s); err != nil {
+ if err := executeQuery(context.Background(), rl, s, outputMode); err != nil {
fmt.Fprintf(rl, "%s\n", err)
}
}
@@ -116,14 +117,43 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
}
s += line
- if isQuitCommand(s) {
- fmt.Fprintf(rl, "bye!\n")
- return
- }
if s == "" {
// Skip empty lines
continue
}
+
+ if isQuitCommand(s) {
+ fmt.Fprintf(rl, "bye!\n")
+ _ = pushToHistory(rl, historyLines, s)
+ return
+ }
+ if isHelpCommand(s) {
+ printCommandsHelp(rl)
+ historyLines = pushToHistory(rl, historyLines, s)
+ s = ""
+ continue
+ }
+ if s == `\s` {
+ fmt.Fprintf(rl, "singleline json output mode\n")
+ outputMode = outputModeJSONSingleline
+ historyLines = pushToHistory(rl, historyLines, s)
+ s = ""
+ continue
+ }
+ if s == `\m` {
+ fmt.Fprintf(rl, "multiline json output mode\n")
+ outputMode = outputModeJSONMultiline
+ historyLines = pushToHistory(rl, historyLines, s)
+ s = ""
+ continue
+ }
+ if s == `\logfmt` {
+ fmt.Fprintf(rl, "logfmt output mode\n")
+ outputMode = outputModeLogfmt
+ historyLines = pushToHistory(rl, historyLines, s)
+ s = ""
+ continue
+ }
if line != "" && !strings.HasSuffix(line, ";") {
// Assume the query is incomplete and allow the user finishing the query on the next line
s += "\n"
@@ -133,7 +163,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
// Execute the query
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
- err = executeQuery(ctx, rl, s)
+ err = executeQuery(ctx, rl, s, outputMode)
cancel()
if err != nil {
@@ -205,14 +235,33 @@ func saveToHistory(filePath string, lines []string) error {
func isQuitCommand(s string) bool {
switch s {
- case "q", "quit", "exit", "\\q":
+ case `\q`, "q", "quit", "exit":
return true
default:
return false
}
}
-func executeQuery(ctx context.Context, output io.Writer, s string) error {
+func isHelpCommand(s string) bool {
+ switch s {
+ case `\h`, "h", "help", "?":
+ return true
+ default:
+ return false
+ }
+}
+
+func printCommandsHelp(w io.Writer) {
+ fmt.Fprintf(w, "%s", `List of available commands:
+\q - quit
+\h - show this help
+\s - singleline json output mode
+\m - multiline json output mode
+\logfmt - logfmt output mode
+`)
+}
+
+func executeQuery(ctx context.Context, output io.Writer, s string, outputMode outputMode) error {
// Parse the query and convert it to canonical view.
s = strings.TrimSuffix(s, ";")
q, err := logstorage.ParseQuery(s)
@@ -257,7 +306,7 @@ func executeQuery(ctx context.Context, output io.Writer, s string) error {
}
// Prettify the response and stream it to 'less'.
- jp := newJSONPrettifier(resp.Body)
+ jp := newJSONPrettifier(resp.Body, outputMode)
defer func() {
_ = jp.Close()
}()
diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md
index e1a8b80df0..a215ce8ffe 100644
--- a/docs/VictoriaLogs/CHANGELOG.md
+++ b/docs/VictoriaLogs/CHANGELOG.md
@@ -15,6 +15,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip
+* FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): add ability to display results in `logfmt` mode, single-line and multi-line JSON modes according [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/#output-modes).
* FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): preserve `less` output after the exit from scrolling mode. This should help re-using previous query results in subsequent queries.
* FEATURE: add [`len` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#len-pipe) for calculating the length for the given [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) value in bytes.
diff --git a/docs/VictoriaLogs/querying/vlogscli.md b/docs/VictoriaLogs/querying/vlogscli.md
index 09ef673ea8..393d8e726a 100644
--- a/docs/VictoriaLogs/querying/vlogscli.md
+++ b/docs/VictoriaLogs/querying/vlogscli.md
@@ -57,7 +57,8 @@ duration: 0.688s
Query execution can be interrupted at any time by pressing `Ctrl+C`.
-Type `q`, `quit` or `exit` and then press `Enter` for exit from `vlogsql`.
+Type `q` and then press `Enter` for exit from `vlogsql` (if you want to search for `q` [word](https://docs.victoriametrics.com/victorialogs/logsql/#word),
+then just wrap it into quotes: `"q"` or `'q'`).
If the query response exceeds vertical screen space, `vlogsql` pipes query response to `less` utility,
so you can scroll the response as needed. This allows executing queries, which potentially
@@ -88,3 +89,14 @@ Press `Ctrl+R` multiple times for searching other matching queries in the histor
Press `Enter` when the needed query is found in order to execute it.
Press `Ctrl+C` for exit from the `search history` mode.
See also [other available shortcuts](https://github.com/chzyer/readline/blob/f533ef1caae91a1fcc90875ff9a5a030f0237c6a/doc/shortcut.md).
+
+## Output modes
+
+By default `vlogscli` displays query results as prettified JSON object with every field on a separate line.
+Fields in every JSON object are sorted in alphabetical order. This simplifies locating the needed fields.
+
+`vlogscli` supports the following output modes:
+
+* A single JSON line per every result. Type `\s` and press `enter` for this mode.
+* Multline JSON per every result. Type `\m` and press `enter` for this mode.
+* [Logfmt output](https://brandur.org/logfmt). Type `\logfmt` and press `enter` for this mode.