From b2294d1cf13fb4422f596a2f44c97ee6472a9101 Mon Sep 17 00:00:00 2001 From: Dmytro Kozlov Date: Mon, 2 May 2022 10:06:34 +0300 Subject: [PATCH] vmctl/vm: added datapoints collection bar (#2486) add progress bars to the VM importer The new progress bars supposed to display the processing speed per each VM importer worker. This info should help to identify if there is a bottleneck on the VM side during the import process, without waiting for its finish. The new progress bars can be disabled by passing `vm-disable-progress-bar` flag. Plotting multiple progress bars requires using experimental progress bar pool from github.com/cheggaaa/pb/v3. Switch to progress bar pool required changes in all import modes. The openTSDB mode wasn't changed due to its implementation, which implies individual progress bars per each series. Because of this, using the pool wasn't possible. Signed-off-by: dmitryk-dk Co-authored-by: hagen1778 --- .../notifier/alertmanager_request.qtpl.go | 112 +-- app/vmalert/tpl/footer.qtpl.go | 36 +- app/vmalert/tpl/header.qtpl | 5 - app/vmalert/tpl/header.qtpl.go | 55 +- app/vmalert/tpl/nav.qtpl.go | 62 +- app/vmalert/web.qtpl | 4 +- app/vmalert/web.qtpl.go | 780 +++++++++--------- app/vmctl/barpool/pool.go | 26 + app/vmctl/flags.go | 5 + app/vmctl/influx.go | 10 +- app/vmctl/main.go | 4 + app/vmctl/prometheus.go | 11 +- app/vmctl/utils.go | 2 + app/vmctl/vm/vm.go | 27 +- app/vmctl/vm_native.go | 13 +- go.mod | 2 +- go.sum | 4 +- vendor/github.com/cheggaaa/pb/v3/pb.go | 7 + vendor/github.com/cheggaaa/pb/v3/pool.go | 105 +++ vendor/github.com/cheggaaa/pb/v3/pool_win.go | 46 ++ vendor/github.com/cheggaaa/pb/v3/pool_x.go | 43 + .../cheggaaa/pb/v3/termutil/term.go | 16 +- .../cheggaaa/pb/v3/termutil/term_win.go | 4 +- .../cheggaaa/pb/v3/termutil/term_x.go | 10 +- vendor/modules.txt | 2 +- 25 files changed, 833 insertions(+), 558 deletions(-) create mode 100644 app/vmctl/barpool/pool.go create mode 100644 vendor/github.com/cheggaaa/pb/v3/pool.go create mode 100644 vendor/github.com/cheggaaa/pb/v3/pool_win.go create mode 100644 vendor/github.com/cheggaaa/pb/v3/pool_x.go diff --git a/app/vmalert/notifier/alertmanager_request.qtpl.go b/app/vmalert/notifier/alertmanager_request.qtpl.go index 8a6ce45e2..f3f7b193a 100644 --- a/app/vmalert/notifier/alertmanager_request.qtpl.go +++ b/app/vmalert/notifier/alertmanager_request.qtpl.go @@ -1,135 +1,135 @@ // Code generated by qtc from "alertmanager_request.qtpl". DO NOT EDIT. // See https://github.com/valyala/quicktemplate for details. -//line app/vmalert/notifier/alertmanager_request.qtpl:1 +//line notifier/alertmanager_request.qtpl:1 package notifier -//line app/vmalert/notifier/alertmanager_request.qtpl:1 +//line notifier/alertmanager_request.qtpl:1 import ( "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" ) -//line app/vmalert/notifier/alertmanager_request.qtpl:8 +//line notifier/alertmanager_request.qtpl:8 import ( qtio422016 "io" qt422016 "github.com/valyala/quicktemplate" ) -//line app/vmalert/notifier/alertmanager_request.qtpl:8 +//line notifier/alertmanager_request.qtpl:8 var ( _ = qtio422016.Copy _ = qt422016.AcquireByteBuffer ) -//line app/vmalert/notifier/alertmanager_request.qtpl:8 +//line notifier/alertmanager_request.qtpl:8 func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) { -//line app/vmalert/notifier/alertmanager_request.qtpl:8 +//line notifier/alertmanager_request.qtpl:8 qw422016.N().S(`[`) -//line app/vmalert/notifier/alertmanager_request.qtpl:10 +//line notifier/alertmanager_request.qtpl:10 for i, alert := range alerts { -//line app/vmalert/notifier/alertmanager_request.qtpl:10 +//line notifier/alertmanager_request.qtpl:10 qw422016.N().S(`{"startsAt":`) -//line app/vmalert/notifier/alertmanager_request.qtpl:12 +//line notifier/alertmanager_request.qtpl:12 qw422016.N().Q(alert.Start.Format(time.RFC3339Nano)) -//line app/vmalert/notifier/alertmanager_request.qtpl:12 +//line notifier/alertmanager_request.qtpl:12 qw422016.N().S(`,"generatorURL":`) -//line app/vmalert/notifier/alertmanager_request.qtpl:13 +//line notifier/alertmanager_request.qtpl:13 qw422016.N().Q(generatorURL(alert)) -//line app/vmalert/notifier/alertmanager_request.qtpl:13 +//line notifier/alertmanager_request.qtpl:13 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:14 +//line notifier/alertmanager_request.qtpl:14 if !alert.End.IsZero() { -//line app/vmalert/notifier/alertmanager_request.qtpl:14 +//line notifier/alertmanager_request.qtpl:14 qw422016.N().S(`"endsAt":`) -//line app/vmalert/notifier/alertmanager_request.qtpl:15 +//line notifier/alertmanager_request.qtpl:15 qw422016.N().Q(alert.End.Format(time.RFC3339Nano)) -//line app/vmalert/notifier/alertmanager_request.qtpl:15 +//line notifier/alertmanager_request.qtpl:15 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:16 +//line notifier/alertmanager_request.qtpl:16 } -//line app/vmalert/notifier/alertmanager_request.qtpl:16 +//line notifier/alertmanager_request.qtpl:16 qw422016.N().S(`"labels": {"alertname":`) -//line app/vmalert/notifier/alertmanager_request.qtpl:18 +//line notifier/alertmanager_request.qtpl:18 qw422016.N().Q(alert.Name) -//line app/vmalert/notifier/alertmanager_request.qtpl:19 +//line notifier/alertmanager_request.qtpl:19 lbls := alert.toPromLabels(relabelCfg) -//line app/vmalert/notifier/alertmanager_request.qtpl:20 +//line notifier/alertmanager_request.qtpl:20 for _, l := range lbls { -//line app/vmalert/notifier/alertmanager_request.qtpl:20 +//line notifier/alertmanager_request.qtpl:20 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:21 +//line notifier/alertmanager_request.qtpl:21 qw422016.N().Q(l.Name) -//line app/vmalert/notifier/alertmanager_request.qtpl:21 +//line notifier/alertmanager_request.qtpl:21 qw422016.N().S(`:`) -//line app/vmalert/notifier/alertmanager_request.qtpl:21 +//line notifier/alertmanager_request.qtpl:21 qw422016.N().Q(l.Value) -//line app/vmalert/notifier/alertmanager_request.qtpl:22 +//line notifier/alertmanager_request.qtpl:22 } -//line app/vmalert/notifier/alertmanager_request.qtpl:22 +//line notifier/alertmanager_request.qtpl:22 qw422016.N().S(`},"annotations": {`) -//line app/vmalert/notifier/alertmanager_request.qtpl:25 +//line notifier/alertmanager_request.qtpl:25 c := len(alert.Annotations) -//line app/vmalert/notifier/alertmanager_request.qtpl:26 +//line notifier/alertmanager_request.qtpl:26 for k, v := range alert.Annotations { -//line app/vmalert/notifier/alertmanager_request.qtpl:27 +//line notifier/alertmanager_request.qtpl:27 c = c - 1 -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line notifier/alertmanager_request.qtpl:28 qw422016.N().Q(k) -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line notifier/alertmanager_request.qtpl:28 qw422016.N().S(`:`) -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line notifier/alertmanager_request.qtpl:28 qw422016.N().Q(v) -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line notifier/alertmanager_request.qtpl:28 if c > 0 { -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line notifier/alertmanager_request.qtpl:28 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:28 +//line notifier/alertmanager_request.qtpl:28 } -//line app/vmalert/notifier/alertmanager_request.qtpl:29 +//line notifier/alertmanager_request.qtpl:29 } -//line app/vmalert/notifier/alertmanager_request.qtpl:29 +//line notifier/alertmanager_request.qtpl:29 qw422016.N().S(`}}`) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line notifier/alertmanager_request.qtpl:32 if i != len(alerts)-1 { -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line notifier/alertmanager_request.qtpl:32 qw422016.N().S(`,`) -//line app/vmalert/notifier/alertmanager_request.qtpl:32 +//line notifier/alertmanager_request.qtpl:32 } -//line app/vmalert/notifier/alertmanager_request.qtpl:33 +//line notifier/alertmanager_request.qtpl:33 } -//line app/vmalert/notifier/alertmanager_request.qtpl:33 +//line notifier/alertmanager_request.qtpl:33 qw422016.N().S(`]`) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 } -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) { -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 streamamRequest(qw422016, alerts, generatorURL, relabelCfg) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 } -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string { -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 writeamRequest(qb422016, alerts, generatorURL, relabelCfg) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 qs422016 := string(qb422016.B) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 return qs422016 -//line app/vmalert/notifier/alertmanager_request.qtpl:35 +//line notifier/alertmanager_request.qtpl:35 } diff --git a/app/vmalert/tpl/footer.qtpl.go b/app/vmalert/tpl/footer.qtpl.go index fd5a0a1ce..7100c36bc 100644 --- a/app/vmalert/tpl/footer.qtpl.go +++ b/app/vmalert/tpl/footer.qtpl.go @@ -1,25 +1,25 @@ // Code generated by qtc from "footer.qtpl". DO NOT EDIT. // See https://github.com/valyala/quicktemplate for details. -//line app/vmalert/tpl/footer.qtpl:1 +//line tpl/footer.qtpl:1 package tpl -//line app/vmalert/tpl/footer.qtpl:1 +//line tpl/footer.qtpl:1 import ( qtio422016 "io" qt422016 "github.com/valyala/quicktemplate" ) -//line app/vmalert/tpl/footer.qtpl:1 +//line tpl/footer.qtpl:1 var ( _ = qtio422016.Copy _ = qt422016.AcquireByteBuffer ) -//line app/vmalert/tpl/footer.qtpl:1 +//line tpl/footer.qtpl:1 func StreamFooter(qw422016 *qt422016.Writer) { -//line app/vmalert/tpl/footer.qtpl:1 +//line tpl/footer.qtpl:1 qw422016.N().S(` @@ -56,31 +56,31 @@ func StreamFooter(qw422016 *qt422016.Writer) { `) -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 } -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 func WriteFooter(qq422016 qtio422016.Writer) { -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 StreamFooter(qw422016) -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 qt422016.ReleaseWriter(qw422016) -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 } -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 func Footer() string { -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 WriteFooter(qb422016) -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 qs422016 := string(qb422016.B) -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 return qs422016 -//line app/vmalert/tpl/footer.qtpl:36 +//line tpl/footer.qtpl:36 } diff --git a/app/vmalert/tpl/header.qtpl b/app/vmalert/tpl/header.qtpl index d5edf68e0..00410519c 100644 --- a/app/vmalert/tpl/header.qtpl +++ b/app/vmalert/tpl/header.qtpl @@ -9,11 +9,6 @@ min-height: 75rem; padding-top: 4.5rem; } - pre { - overflow: scroll; - max-width: 600px; - min-height: 30px; - } .group-heading { cursor: pointer; padding: 5px; diff --git a/app/vmalert/tpl/header.qtpl.go b/app/vmalert/tpl/header.qtpl.go index 366a75727..a4f75c3e7 100644 --- a/app/vmalert/tpl/header.qtpl.go +++ b/app/vmalert/tpl/header.qtpl.go @@ -1,39 +1,39 @@ // Code generated by qtc from "header.qtpl". DO NOT EDIT. // See https://github.com/valyala/quicktemplate for details. -//line app/vmalert/tpl/header.qtpl:1 +//line tpl/header.qtpl:1 package tpl -//line app/vmalert/tpl/header.qtpl:1 +//line tpl/header.qtpl:1 import ( qtio422016 "io" qt422016 "github.com/valyala/quicktemplate" ) -//line app/vmalert/tpl/header.qtpl:1 +//line tpl/header.qtpl:1 var ( _ = qtio422016.Copy _ = qt422016.AcquireByteBuffer ) -//line app/vmalert/tpl/header.qtpl:1 +//line tpl/header.qtpl:1 func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) { -//line app/vmalert/tpl/header.qtpl:1 +//line tpl/header.qtpl:1 qw422016.N().S(` vmalert`) -//line app/vmalert/tpl/header.qtpl:5 +//line tpl/header.qtpl:5 if title != "" { -//line app/vmalert/tpl/header.qtpl:5 +//line tpl/header.qtpl:5 qw422016.N().S(` - `) -//line app/vmalert/tpl/header.qtpl:5 +//line tpl/header.qtpl:5 qw422016.E().S(title) -//line app/vmalert/tpl/header.qtpl:5 +//line tpl/header.qtpl:5 } -//line app/vmalert/tpl/header.qtpl:5 +//line tpl/header.qtpl:5 qw422016.N().S(`