github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vlogscli: add ability to display query results in logfmt, single-line and multi-line json modes

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-10-07 12:18:18 +02:00
parent e144a2b062
commit 492190885d
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
4 changed files with 159 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

124
app/vlogscli/json_prettifier.go
View file

@ -1,33 +1,70 @@
package main
import (
"bufio"
"encoding/json"
"fmt"
"io"
"sort"
"sync"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
)
type outputMode int
const (
outputModeJSONMultiline = outputMode(0)
outputModeJSONSingleline = outputMode(1)
outputModeLogfmt = outputMode(2)
)
func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error {
switch outputMode {
case outputModeJSONMultiline:
return func(w io.Writer, fields []logstorage.Field) error {
return writeJSONObject(w, fields, true)
}
case outputModeJSONSingleline:
return func(w io.Writer, fields []logstorage.Field) error {
return writeJSONObject(w, fields, false)
}
case outputModeLogfmt:
return writeLogfmtObject
default:
panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode))
}
}
type jsonPrettifier struct {
rOriginal io.ReadCloser
r io.ReadCloser
formatter func(w io.Writer, fields []logstorage.Field) error
d *json.Decoder
pr *io.PipeReader
pw *io.PipeWriter
bw *bufio.Writer
wg sync.WaitGroup
}
func newJSONPrettifier(r io.ReadCloser) *jsonPrettifier {
func newJSONPrettifier(r io.ReadCloser, outputMode outputMode) *jsonPrettifier {
d := json.NewDecoder(r)
pr, pw := io.Pipe()
bw := bufio.NewWriter(pw)
formatter := getOutputFormatter(outputMode)
jp := &jsonPrettifier{
rOriginal: r,
d: d,
pr: pr,
pw: pw,
r: r,
formatter: formatter,
d: d,
pr: pr,
pw: pw,
bw: bw,
}
jp.wg.Add(1)
@ -47,20 +84,23 @@ func (jp *jsonPrettifier) closePipesWithError(err error) {
func (jp *jsonPrettifier) prettifyJSONLines() error {
for jp.d.More() {
kvs, err := readNextJSONObject(jp.d)
fields, err := readNextJSONObject(jp.d)
if err != nil {
return err
}
if err := writeJSONObject(jp.pw, kvs); err != nil {
sort.Slice(fields, func(i, j int) bool {
return fields[i].Name < fields[j].Name
})
if err := jp.formatter(jp.bw, fields); err != nil {
return err
}
}
return nil
return jp.bw.Flush()
}
func (jp *jsonPrettifier) Close() error {
jp.closePipesWithError(io.ErrUnexpectedEOF)
err := jp.rOriginal.Close()
err := jp.r.Close()
jp.wg.Wait()
return err
}
@ -69,7 +109,7 @@ func (jp *jsonPrettifier) Read(p []byte) (int, error) {
return jp.pr.Read(p)
}
func readNextJSONObject(d *json.Decoder) ([]kv, error) {
func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
t, err := d.Token()
if err != nil {
return nil, fmt.Errorf("cannot read '{': %w", err)
@ -79,7 +119,7 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) {
return nil, fmt.Errorf("unexpected token read; got %q; want '{'", delim)
}
var kvs []kv
var fields []logstorage.Field
for {
// Read object key
t, err := d.Token()
@ -89,7 +129,7 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) {
delim, ok := t.(json.Delim)
if ok {
if delim.String() == "}" {
return kvs, nil
return fields, nil
}
return nil, fmt.Errorf("unexpected delimiter read; got %q; want '}'", delim)
}
@ -108,41 +148,56 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) {
return nil, fmt.Errorf("unexpected token read for oject value: %v; want string", t)
}
kvs = append(kvs, kv{
key: key,
value: value,
fields = append(fields, logstorage.Field{
Name: key,
Value: value,
})
}
}
func writeJSONObject(w io.Writer, kvs []kv) error {
if len(kvs) == 0 {
func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error {
data := logstorage.MarshalFieldsToLogfmt(nil, fields)
_, err := fmt.Fprintf(w, "%s\n", data)
return err
}
func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error {
if len(fields) == 0 {
fmt.Fprintf(w, "{}\n")
return nil
}
sort.Slice(kvs, func(i, j int) bool {
return kvs[i].key < kvs[j].key
})
fmt.Fprintf(w, "{\n")
if err := writeJSONObjectKeyValue(w, kvs[0]); err != nil {
fmt.Fprintf(w, "{")
writeNewlineIfNeeded(w, isMultiline)
if err := writeJSONObjectKeyValue(w, fields[0], isMultiline); err != nil {
return err
}
for _, kv := range kvs[1:] {
fmt.Fprintf(w, ",\n")
if err := writeJSONObjectKeyValue(w, kv); err != nil {
for _, f := range fields[1:] {
fmt.Fprintf(w, ",")
writeNewlineIfNeeded(w, isMultiline)
if err := writeJSONObjectKeyValue(w, f, isMultiline); err != nil {
return err
}
}
fmt.Fprintf(w, "\n}\n")
writeNewlineIfNeeded(w, isMultiline)
fmt.Fprintf(w, "}\n")
return nil
}
func writeJSONObjectKeyValue(w io.Writer, kv kv) error {
key := getJSONString(kv.key)
value := getJSONString(kv.value)
_, err := fmt.Fprintf(w, " %s: %s", key, value)
func writeNewlineIfNeeded(w io.Writer, isMultiline bool) {
if isMultiline {
fmt.Fprintf(w, "\n")
}
}
func writeJSONObjectKeyValue(w io.Writer, f logstorage.Field, isMultiline bool) error {
key := getJSONString(f.Name)
value := getJSONString(f.Value)
if isMultiline {
_, err := fmt.Fprintf(w, " %s: %s", key, value)
return err
}
_, err := fmt.Fprintf(w, "%s:%s", key, value)
return err
}
@ -153,8 +208,3 @@ func getJSONString(s string) string {
}
return string(data)
}
type kv struct {
key string
value string
}

67
app/vlogscli/main.go
View file

@ -86,6 +86,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
}
}
outputMode := outputModeJSONMultiline
s := ""
for {
line, err := rl.ReadLine()
@ -94,7 +95,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
case io.EOF:
if s != "" {
// This is non-interactive query execution.
if err := executeQuery(context.Background(), rl, s); err != nil {
if err := executeQuery(context.Background(), rl, s, outputMode); err != nil {
fmt.Fprintf(rl, "%s\n", err)
}
}
@ -116,14 +117,43 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
}
s += line
if isQuitCommand(s) {
fmt.Fprintf(rl, "bye!\n")
return
}
if s == "" {
// Skip empty lines
continue
}
if isQuitCommand(s) {
fmt.Fprintf(rl, "bye!\n")
_ = pushToHistory(rl, historyLines, s)
return
}
if isHelpCommand(s) {
printCommandsHelp(rl)
historyLines = pushToHistory(rl, historyLines, s)
s = ""
continue
}
if s == `\s` {
fmt.Fprintf(rl, "singleline json output mode\n")
outputMode = outputModeJSONSingleline
historyLines = pushToHistory(rl, historyLines, s)
s = ""
continue
}
if s == `\m` {
fmt.Fprintf(rl, "multiline json output mode\n")
outputMode = outputModeJSONMultiline
historyLines = pushToHistory(rl, historyLines, s)
s = ""
continue
}
if s == `\logfmt` {
fmt.Fprintf(rl, "logfmt output mode\n")
outputMode = outputModeLogfmt
historyLines = pushToHistory(rl, historyLines, s)
s = ""
continue
}
if line != "" && !strings.HasSuffix(line, ";") {
// Assume the query is incomplete and allow the user finishing the query on the next line
s += "\n"
@ -133,7 +163,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
// Execute the query
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
err = executeQuery(ctx, rl, s)
err = executeQuery(ctx, rl, s, outputMode)
cancel()
if err != nil {
@ -205,14 +235,33 @@ func saveToHistory(filePath string, lines []string) error {
func isQuitCommand(s string) bool {
switch s {
case "q", "quit", "exit", "\\q":
case `\q`, "q", "quit", "exit":
return true
default:
return false
}
}
func executeQuery(ctx context.Context, output io.Writer, s string) error {
func isHelpCommand(s string) bool {
switch s {
case `\h`, "h", "help", "?":
return true
default:
return false
}
}
func printCommandsHelp(w io.Writer) {
fmt.Fprintf(w, "%s", `List of available commands:
\q - quit
\h - show this help
\s - singleline json output mode
\m - multiline json output mode
\logfmt - logfmt output mode
`)
}
func executeQuery(ctx context.Context, output io.Writer, s string, outputMode outputMode) error {
// Parse the query and convert it to canonical view.
s = strings.TrimSuffix(s, ";")
q, err := logstorage.ParseQuery(s)
@ -257,7 +306,7 @@ func executeQuery(ctx context.Context, output io.Writer, s string) error {
}
// Prettify the response and stream it to 'less'.
jp := newJSONPrettifier(resp.Body)
jp := newJSONPrettifier(resp.Body, outputMode)
defer func() {
_ = jp.Close()
}()

1
docs/VictoriaLogs/CHANGELOG.md
View file

@ -15,6 +15,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip
* FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): add ability to display results in `logfmt` mode, single-line and multi-line JSON modes according [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/#output-modes).
* FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): preserve `less` output after the exit from scrolling mode. This should help re-using previous query results in subsequent queries.
* FEATURE: add [`len` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#len-pipe) for calculating the length for the given [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) value in bytes.

14
docs/VictoriaLogs/querying/vlogscli.md
View file

@ -57,7 +57,8 @@ duration: 0.688s
Query execution can be interrupted at any time by pressing `Ctrl+C`.
Type `q`, `quit` or `exit` and then press `Enter` for exit from `vlogsql`.
Type `q` and then press `Enter` for exit from `vlogsql` (if you want to search for `q` [word](https://docs.victoriametrics.com/victorialogs/logsql/#word),
then just wrap it into quotes: `"q"` or `'q'`).
If the query response exceeds vertical screen space, `vlogsql` pipes query response to `less` utility,
so you can scroll the response as needed. This allows executing queries, which potentially
@ -88,3 +89,14 @@ Press `Ctrl+R` multiple times for searching other matching queries in the histor
Press `Enter` when the needed query is found in order to execute it.
Press `Ctrl+C` for exit from the `search history` mode.
See also [other available shortcuts](https://github.com/chzyer/readline/blob/f533ef1caae91a1fcc90875ff9a5a030f0237c6a/doc/shortcut.md).
## Output modes
By default `vlogscli` displays query results as prettified JSON object with every field on a separate line.
Fields in every JSON object are sorted in alphabetical order. This simplifies locating the needed fields.
`vlogscli` supports the following output modes:
* A single JSON line per every result. Type `\s` and press `enter` for this mode.
* Multline JSON per every result. Type `\m` and press `enter` for this mode.
* [Logfmt output](https://brandur.org/logfmt). Type `\logfmt` and press `enter` for this mode.