diff --git a/README.md b/README.md index 17e55b9a92..1cb330ddd6 100644 --- a/README.md +++ b/README.md @@ -2762,7 +2762,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -cacheExpireDuration duration Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s) -configAuthKey value - Authorization key for accessing /config page. It must be passed via authKey query arg + Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings. Flag value can be read from the given file when using -configAuthKey=file:///abs/path/to/file or -configAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -configAuthKey=http://host/path or -configAuthKey=https://host/path -csvTrimTimestamp duration Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms) @@ -3076,7 +3076,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -relabelConfig string Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal -reloadAuthKey value - Auth key for /-/reload http endpoint. It must be passed as authKey=... + Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings. Flag value can be read from the given file when using -reloadAuthKey=file:///abs/path/to/file or -reloadAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -reloadAuthKey=http://host/path or -reloadAuthKey=https://host/path -retentionFilter array Retention filter in the format 'filter:retention'. For example, '{env="dev"}:3d' configures the retention for time series with env="dev" label to 3 days. See https://docs.victoriametrics.com/#retention-filters for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise/ diff --git a/app/vmagent/main.go b/app/vmagent/main.go index 4429f2458f..c1c53ea7a9 100644 --- a/app/vmagent/main.go +++ b/app/vmagent/main.go @@ -78,8 +78,8 @@ var ( "See also -opentsdbHTTPListenAddr.useProxyProtocol") opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+ "at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt") - configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg") - reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...") + configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings.") + reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.") dryRun = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+ "-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+ "Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag") diff --git a/app/vmalert/web.go b/app/vmalert/web.go index 0dc8ac8b87..43d2562ccc 100644 --- a/app/vmalert/web.go +++ b/app/vmalert/web.go @@ -19,7 +19,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil" ) -var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...") +var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.") var ( apiLinks = [][2]string{ diff --git a/app/vmauth/main.go b/app/vmauth/main.go index 92d906689d..9611b19a54 100644 --- a/app/vmauth/main.go +++ b/app/vmauth/main.go @@ -37,13 +37,15 @@ var ( "With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing") maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+ "See also -maxConcurrentRequests") + idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. + Consider setting this value less than "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`) responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend") maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+ "'429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options") maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+ "Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+ "in per-user config") - reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...") + reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.") logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+ `Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`) failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend") @@ -199,10 +201,8 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) { isDefault = true } maxAttempts := up.getBackendsCount() - if maxAttempts > 1 { - r.Body = &readTrackingBody{ - r: r.Body, - } + r.Body = &readTrackingBody{ + r: r.Body, } for i := 0; i < maxAttempts; i++ { bu := up.getBackendURL() @@ -243,8 +243,10 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url req.Host = targetURL.Host } updateHeadersByConfig(req.Header, hc.RequestHeaders) - res, err := ui.rt.RoundTrip(req) + var trivialRetries int rtb, rtbOK := req.Body.(*readTrackingBody) +again: + res, err := ui.rt.RoundTrip(req) if err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { // Do not retry canceled or timed out requests @@ -267,6 +269,12 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url ui.backendErrors.Inc() return true } + // one time retry trivial network errors, such as proxy idle timeout misconfiguration + // or socket close by OS + if (netutil.IsTrivialNetworkError(err) || errors.Is(err, io.EOF)) && trivialRetries < 1 { + trivialRetries++ + goto again + } // Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable. remoteAddr := httpserver.GetQuotedRemoteAddr(r) // NOTE: do not use httpserver.GetRequestURI @@ -434,6 +442,7 @@ func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, i tr.ResponseHeaderTimeout = *responseTimeout // Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535 tr.DisableCompression = true + tr.IdleConnTimeout = *idleConnTimeout tr.MaxIdleConnsPerHost = *maxIdleConnsPerBackend if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost { tr.MaxIdleConns = tr.MaxIdleConnsPerHost diff --git a/app/vmctl/barpool/pool.go b/app/vmctl/barpool/pool.go index 05fe8b4181..deb20b4fb4 100644 --- a/app/vmctl/barpool/pool.go +++ b/app/vmctl/barpool/pool.go @@ -5,37 +5,88 @@ package barpool import ( "fmt" + "io" "os" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/terminal" "github.com/cheggaaa/pb/v3" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/terminal" ) +var isDisabled bool + +// Disable sets progress bar to be no-op if v==true +func Disable(v bool) { + isDisabled = v +} + var pool = pb.NewPool() -// Add adds bar to the global pool -func Add(bar *pb.ProgressBar) { pool.Add(bar) } +// Bar is an interface for progress bar +type Bar interface { + Add(value int) + Increment() + Start() + Finish() + NewProxyReader(r io.Reader) *pb.Reader +} + +type progressBar struct { + *pb.ProgressBar +} + +func (pb *progressBar) Finish() { pb.ProgressBar.Finish() } +func (pb *progressBar) Start() { pb.ProgressBar.Start() } +func (pb *progressBar) Add(value int) { pb.ProgressBar.Add(value) } +func (pb *progressBar) Increment() { pb.ProgressBar.Increment() } +func (pb *progressBar) NewProxyReader(r io.Reader) *pb.Reader { + return pb.ProgressBar.NewProxyReader(r) +} + +type progressBarNoOp struct{} + +func (pbno *progressBarNoOp) Finish() {} +func (pbno *progressBarNoOp) Start() {} +func (pbno *progressBarNoOp) Add(int) {} +func (pbno *progressBarNoOp) Increment() {} +func (pbno *progressBarNoOp) NewProxyReader(_ io.Reader) *pb.Reader { return nil } // Start starts the global pool // Must be called after all progress bars were added -func Start() error { return pool.Start() } +func Start() error { + if isDisabled { + return nil + } + return pool.Start() +} // Stop stops the global pool -func Stop() { _ = pool.Stop() } +func Stop() { + if isDisabled { + return + } + _ = pool.Stop() +} // AddWithTemplate adds bar with the given template // to the global pool -func AddWithTemplate(format string, total int) *pb.ProgressBar { +func AddWithTemplate(format string, total int) Bar { + if isDisabled { + return &progressBarNoOp{} + } tpl := getTemplate(format) bar := pb.ProgressBarTemplate(tpl).New(total) - Add(bar) - return bar + pool.Add(bar) + return &progressBar{bar} } // NewSingleProgress returns progress bar with given template -func NewSingleProgress(format string, total int) *pb.ProgressBar { +func NewSingleProgress(format string, total int) Bar { + if isDisabled { + return &progressBarNoOp{} + } tpl := getTemplate(format) - return pb.ProgressBarTemplate(tpl).New(total) + return &progressBar{pb.ProgressBarTemplate(tpl).New(total)} } func getTemplate(format string) string { diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go index 51146880ef..e5ae52e542 100644 --- a/app/vmctl/flags.go +++ b/app/vmctl/flags.go @@ -10,8 +10,9 @@ import ( ) const ( - globalSilent = "s" - globalVerbose = "verbose" + globalSilent = "s" + globalVerbose = "verbose" + globalDisableProgressBar = "disable-progress-bar" ) var ( @@ -26,6 +27,11 @@ var ( Value: false, Usage: "Whether to enable verbosity in logs output.", }, + &cli.BoolFlag{ + Name: globalDisableProgressBar, + Value: false, + Usage: "Whether to disable progress bar during the import.", + }, } ) @@ -39,7 +45,6 @@ const ( vmBatchSize = "vm-batch-size" vmSignificantFigures = "vm-significant-figures" vmRoundDigits = "vm-round-digits" - vmDisableProgressBar = "vm-disable-progress-bar" vmCertFile = "vm-cert-file" vmKeyFile = "vm-key-file" vmCAFile = "vm-CA-file" @@ -120,10 +125,6 @@ var ( Usage: "Optional data transfer rate limit in bytes per second.\n" + "By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination.", }, - &cli.BoolFlag{ - Name: vmDisableProgressBar, - Usage: "Whether to disable progress bar per each worker during the import.", - }, &cli.StringFlag{ Name: vmCertFile, Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vmAddr'", diff --git a/app/vmctl/influx.go b/app/vmctl/influx.go index 41cf740830..eeb91e8207 100644 --- a/app/vmctl/influx.go +++ b/app/vmctl/influx.go @@ -18,14 +18,14 @@ type influxProcessor struct { separator string skipDbLabel bool promMode bool - isSilent bool isVerbose bool } -func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator string, skipDbLabel, promMode, silent, verbose bool) *influxProcessor { +func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator string, skipDbLabel, promMode, verbose bool) *influxProcessor { if cc < 1 { cc = 1 } + return &influxProcessor{ ic: ic, im: im, @@ -33,7 +33,6 @@ func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator st separator: separator, skipDbLabel: skipDbLabel, promMode: promMode, - isSilent: silent, isVerbose: verbose, } } @@ -48,7 +47,7 @@ func (ip *influxProcessor) run() error { } question := fmt.Sprintf("Found %d timeseries to import. Continue?", len(series)) - if !ip.isSilent && !prompt(question) { + if !prompt(question) { return nil } diff --git a/app/vmctl/main.go b/app/vmctl/main.go index 10ca56d35f..136c186a96 100644 --- a/app/vmctl/main.go +++ b/app/vmctl/main.go @@ -16,6 +16,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread" @@ -37,15 +38,23 @@ func main() { ctx, cancelCtx := context.WithCancel(context.Background()) start := time.Now() + beforeFn := func(c *cli.Context) error { + isSilent = c.Bool(globalSilent) + if c.Bool(globalDisableProgressBar) { + barpool.Disable(true) + } + return nil + } app := &cli.App{ Name: "vmctl", Usage: "VictoriaMetrics command-line tool", Version: buildinfo.Version, Commands: []*cli.Command{ { - Name: "opentsdb", - Usage: "Migrate time series from OpenTSDB", - Flags: mergeFlags(globalFlags, otsdbFlags, vmFlags), + Name: "opentsdb", + Usage: "Migrate time series from OpenTSDB", + Flags: mergeFlags(globalFlags, otsdbFlags, vmFlags), + Before: beforeFn, Action: func(c *cli.Context) error { fmt.Println("OpenTSDB import mode") @@ -81,22 +90,20 @@ func main() { if err != nil { return fmt.Errorf("failed to init VM configuration: %s", err) } - // disable progress bars since openTSDB implementation - // does not use progress bar pool - vmCfg.DisableProgressBar = true importer, err := vm.NewImporter(ctx, vmCfg) if err != nil { return fmt.Errorf("failed to create VM importer: %s", err) } - otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency), c.Bool(globalSilent), c.Bool(globalVerbose)) + otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency), c.Bool(globalVerbose)) return otsdbProcessor.run() }, }, { - Name: "influx", - Usage: "Migrate time series from InfluxDB", - Flags: mergeFlags(globalFlags, influxFlags, vmFlags), + Name: "influx", + Usage: "Migrate time series from InfluxDB", + Flags: mergeFlags(globalFlags, influxFlags, vmFlags), + Before: beforeFn, Action: func(c *cli.Context) error { fmt.Println("InfluxDB import mode") @@ -148,15 +155,15 @@ func main() { c.String(influxMeasurementFieldSeparator), c.Bool(influxSkipDatabaseLabel), c.Bool(influxPrometheusMode), - c.Bool(globalSilent), c.Bool(globalVerbose)) return processor.run() }, }, { - Name: "remote-read", - Usage: "Migrate time series via Prometheus remote-read protocol", - Flags: mergeFlags(globalFlags, remoteReadFlags, vmFlags), + Name: "remote-read", + Usage: "Migrate time series via Prometheus remote-read protocol", + Flags: mergeFlags(globalFlags, remoteReadFlags, vmFlags), + Before: beforeFn, Action: func(c *cli.Context) error { fmt.Println("Remote-read import mode") @@ -209,16 +216,16 @@ func main() { timeReverse: c.Bool(remoteReadFilterTimeReverse), }, cc: c.Int(remoteReadConcurrency), - isSilent: c.Bool(globalSilent), isVerbose: c.Bool(globalVerbose), } return rmp.run(ctx) }, }, { - Name: "prometheus", - Usage: "Migrate time series from Prometheus", - Flags: mergeFlags(globalFlags, promFlags, vmFlags), + Name: "prometheus", + Usage: "Migrate time series from Prometheus", + Flags: mergeFlags(globalFlags, promFlags, vmFlags), + Before: beforeFn, Action: func(c *cli.Context) error { fmt.Println("Prometheus import mode") @@ -245,17 +252,19 @@ func main() { return fmt.Errorf("failed to create prometheus client: %s", err) } pp := prometheusProcessor{ - cl: cl, - im: importer, - cc: c.Int(promConcurrency), + cl: cl, + im: importer, + cc: c.Int(promConcurrency), + isVerbose: c.Bool(globalVerbose), } - return pp.run(c.Bool(globalSilent), c.Bool(globalVerbose)) + return pp.run() }, }, { - Name: "vm-native", - Usage: "Migrate time series between VictoriaMetrics installations via native binary format", - Flags: mergeFlags(globalFlags, vmNativeFlags), + Name: "vm-native", + Usage: "Migrate time series between VictoriaMetrics installations via native binary format", + Flags: mergeFlags(globalFlags, vmNativeFlags), + Before: beforeFn, Action: func(c *cli.Context) error { fmt.Println("VictoriaMetrics Native import mode") @@ -344,7 +353,6 @@ func main() { backoff: backoff.New(), cc: c.Int(vmConcurrency), disablePerMetricRequests: c.Bool(vmNativeDisablePerMetricMigration), - isSilent: c.Bool(globalSilent), isNative: !c.Bool(vmNativeDisableBinaryProtocol), } return p.run(ctx) @@ -360,6 +368,7 @@ func main() { Value: false, }, }, + Before: beforeFn, Action: func(c *cli.Context) error { common.StartUnmarshalWorkers() blockPath := c.Args().First() @@ -433,6 +442,5 @@ func initConfigVM(c *cli.Context) (vm.Config, error) { RoundDigits: c.Int(vmRoundDigits), ExtraLabels: c.StringSlice(vmExtraLabel), RateLimit: c.Int64(vmRateLimit), - DisableProgressBar: c.Bool(vmDisableProgressBar), }, nil } diff --git a/app/vmctl/opentsdb.go b/app/vmctl/opentsdb.go index 07fc3a6d54..437da9e690 100644 --- a/app/vmctl/opentsdb.go +++ b/app/vmctl/opentsdb.go @@ -15,7 +15,6 @@ type otsdbProcessor struct { oc *opentsdb.Client im *vm.Importer otsdbcc int - isSilent bool isVerbose bool } @@ -26,7 +25,7 @@ type queryObj struct { StartTime int64 } -func newOtsdbProcessor(oc *opentsdb.Client, im *vm.Importer, otsdbcc int, silent, verbose bool) *otsdbProcessor { +func newOtsdbProcessor(oc *opentsdb.Client, im *vm.Importer, otsdbcc int, verbose bool) *otsdbProcessor { if otsdbcc < 1 { otsdbcc = 1 } @@ -34,7 +33,6 @@ func newOtsdbProcessor(oc *opentsdb.Client, im *vm.Importer, otsdbcc int, silent oc: oc, im: im, otsdbcc: otsdbcc, - isSilent: silent, isVerbose: verbose, } } @@ -55,7 +53,7 @@ func (op *otsdbProcessor) run() error { } question := fmt.Sprintf("Found %d metrics to import. Continue?", len(metrics)) - if !op.isSilent && !prompt(question) { + if !prompt(question) { return nil } op.im.ResetStats() diff --git a/app/vmctl/prometheus.go b/app/vmctl/prometheus.go index 4b18bff27a..eba010070a 100644 --- a/app/vmctl/prometheus.go +++ b/app/vmctl/prometheus.go @@ -5,11 +5,12 @@ import ( "log" "sync" + "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm" - "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/prometheus/tsdb/chunkenc" ) type prometheusProcessor struct { @@ -24,9 +25,12 @@ type prometheusProcessor struct { // and defines number of concurrently // running snapshot block readers cc int + + // isVerbose enables verbose output + isVerbose bool } -func (pp *prometheusProcessor) run(silent, verbose bool) error { +func (pp *prometheusProcessor) run() error { blocks, err := pp.cl.Explore() if err != nil { return fmt.Errorf("explore failed: %s", err) @@ -35,12 +39,11 @@ func (pp *prometheusProcessor) run(silent, verbose bool) error { return fmt.Errorf("found no blocks to import") } question := fmt.Sprintf("Found %d blocks to import. Continue?", len(blocks)) - if !silent && !prompt(question) { + if !prompt(question) { return nil } bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks)) - if err := barpool.Start(); err != nil { return err } @@ -72,7 +75,7 @@ func (pp *prometheusProcessor) run(silent, verbose bool) error { return fmt.Errorf("prometheus error: %s", promErr) case vmErr := <-pp.im.Errors(): close(blockReadersCh) - return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose)) + return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose)) case blockReadersCh <- br: } } @@ -85,7 +88,7 @@ func (pp *prometheusProcessor) run(silent, verbose bool) error { // drain import errors channel for vmErr := range pp.im.Errors() { if vmErr.Err != nil { - return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose)) + return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose)) } } for err := range errCh { diff --git a/app/vmctl/prometheus_test.go b/app/vmctl/prometheus_test.go index dbf74eda5d..13281d0259 100644 --- a/app/vmctl/prometheus_test.go +++ b/app/vmctl/prometheus_test.go @@ -25,6 +25,9 @@ const ( // This test simulates close process if user abort it func Test_prometheusProcessor_run(t *testing.T) { t.Skip() + + defer func() { isSilent = false }() + type fields struct { cfg prometheus.Config vmCfg vm.Config @@ -117,11 +120,12 @@ func Test_prometheusProcessor_run(t *testing.T) { t.Run(tt.name, func(t *testing.T) { client := tt.fields.cl(tt.fields.cfg) importer := tt.fields.im(tt.fields.vmCfg) - + isSilent = tt.args.silent pp := &prometheusProcessor{ - cl: client, - im: importer, - cc: tt.fields.cc, + cl: client, + im: importer, + cc: tt.fields.cc, + isVerbose: tt.args.verbose, } // we should answer on prompt @@ -157,7 +161,7 @@ func Test_prometheusProcessor_run(t *testing.T) { go tt.fields.closer(importer) } - if err := pp.run(tt.args.silent, tt.args.verbose); (err != nil) != tt.wantErr { + if err := pp.run(); (err != nil) != tt.wantErr { t.Errorf("run() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/app/vmctl/remote_read_test.go b/app/vmctl/remote_read_test.go index 7593313179..c5a69a61ce 100644 --- a/app/vmctl/remote_read_test.go +++ b/app/vmctl/remote_read_test.go @@ -6,14 +6,21 @@ import ( "testing" "time" + "github.com/prometheus/prometheus/prompb" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/testdata/servers_integration_test" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm" - "github.com/prometheus/prometheus/prompb" ) func TestRemoteRead(t *testing.T) { + barpool.Disable(true) + defer func() { + barpool.Disable(false) + }() + defer func() { isSilent = false }() var testCases = []struct { name string @@ -31,7 +38,7 @@ func TestRemoteRead(t *testing.T) { { name: "step minute on minute time range", remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"}, - vmCfg: vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true}, + vmCfg: vm.Config{Addr: "", Concurrency: 1}, start: "2022-11-26T11:23:05+02:00", end: "2022-11-26T11:24:05+02:00", numOfSamples: 2, @@ -62,7 +69,7 @@ func TestRemoteRead(t *testing.T) { { name: "step month on month time range", remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"}, - vmCfg: vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true, + vmCfg: vm.Config{Addr: "", Concurrency: 1, Transport: http.DefaultTransport.(*http.Transport)}, start: "2022-09-26T11:23:05+02:00", end: "2022-11-26T11:24:05+02:00", @@ -157,7 +164,6 @@ func TestRemoteRead(t *testing.T) { chunk: tt.chunk, }, cc: 1, - isSilent: true, isVerbose: false, } @@ -170,6 +176,11 @@ func TestRemoteRead(t *testing.T) { } func TestSteamRemoteRead(t *testing.T) { + barpool.Disable(true) + defer func() { + barpool.Disable(false) + }() + defer func() { isSilent = false }() var testCases = []struct { name string @@ -187,7 +198,7 @@ func TestSteamRemoteRead(t *testing.T) { { name: "step minute on minute time range", remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true}, - vmCfg: vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true}, + vmCfg: vm.Config{Addr: "", Concurrency: 1}, start: "2022-11-26T11:23:05+02:00", end: "2022-11-26T11:24:05+02:00", numOfSamples: 2, @@ -218,7 +229,7 @@ func TestSteamRemoteRead(t *testing.T) { { name: "step month on month time range", remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true}, - vmCfg: vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true}, + vmCfg: vm.Config{Addr: "", Concurrency: 1}, start: "2022-09-26T11:23:05+02:00", end: "2022-11-26T11:24:05+02:00", numOfSamples: 2, @@ -312,7 +323,6 @@ func TestSteamRemoteRead(t *testing.T) { chunk: tt.chunk, }, cc: 1, - isSilent: true, isVerbose: false, } diff --git a/app/vmctl/remoteread.go b/app/vmctl/remoteread.go index 8632369c6f..09ac7f0525 100644 --- a/app/vmctl/remoteread.go +++ b/app/vmctl/remoteread.go @@ -7,8 +7,6 @@ import ( "sync" "time" - "github.com/cheggaaa/pb/v3" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper" @@ -22,7 +20,6 @@ type remoteReadProcessor struct { src *remoteread.Client cc int - isSilent bool isVerbose bool } @@ -50,21 +47,17 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error { question := fmt.Sprintf("Selected time range %q - %q will be split into %d ranges according to %q step. Continue?", rrp.filter.timeStart.String(), rrp.filter.timeEnd.String(), len(ranges), rrp.filter.chunk) - if !rrp.isSilent && !prompt(question) { + if !prompt(question) { return nil } - var bar *pb.ProgressBar - if !rrp.isSilent { - bar = barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing ranges"), len(ranges)) - if err := barpool.Start(); err != nil { - return err - } + bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing ranges"), len(ranges)) + if err := barpool.Start(); err != nil { + return err } + defer func() { - if !rrp.isSilent { - barpool.Stop() - } + barpool.Stop() log.Println("Import finished!") log.Print(rrp.dst.Stats()) }() @@ -82,9 +75,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error { errCh <- fmt.Errorf("request failed for: %s", err) return } - if bar != nil { - bar.Increment() - } + bar.Increment() } }() } diff --git a/app/vmctl/utils.go b/app/vmctl/utils.go index c53b95554e..3fa384d63d 100644 --- a/app/vmctl/utils.go +++ b/app/vmctl/utils.go @@ -12,7 +12,13 @@ import ( const barTpl = `{{ blue "%s:" }} {{ counters . }} {{ bar . "[" "โ" (cycle . "โ") "โ" "]" }} {{ percent . }}` +// isSilent should be inited in main +var isSilent bool + func prompt(question string) bool { + if isSilent { + return true + } isTerminal := terminal.IsTerminal(int(os.Stdout.Fd())) if !isTerminal { return true diff --git a/app/vmctl/vm/vm.go b/app/vmctl/vm/vm.go index 539976770f..03183e8675 100644 --- a/app/vmctl/vm/vm.go +++ b/app/vmctl/vm/vm.go @@ -16,7 +16,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter" "github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal" - "github.com/cheggaaa/pb/v3" ) // Config contains list of params to configure @@ -55,8 +54,6 @@ type Config struct { // RateLimit defines a data transfer speed in bytes per second. // Is applied to each worker (see Concurrency) independently. RateLimit int64 - // Whether to disable progress bar per VM worker - DisableProgressBar bool } // Importer performs insertion of timeseries @@ -159,12 +156,10 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) { im.wg.Add(int(cfg.Concurrency)) for i := 0; i < int(cfg.Concurrency); i++ { - var bar *pb.ProgressBar - if !cfg.DisableProgressBar { - pbPrefix := fmt.Sprintf(`{{ green "VM worker %d:" }}`, i) - bar = barpool.AddWithTemplate(pbPrefix+pbTpl, 0) - } - go func(bar *pb.ProgressBar) { + pbPrefix := fmt.Sprintf(`{{ green "VM worker %d:" }}`, i) + bar := barpool.AddWithTemplate(pbPrefix+pbTpl, 0) + + go func(bar barpool.Bar) { defer im.wg.Done() im.startWorker(ctx, bar, cfg.BatchSize, cfg.SignificantFigures, cfg.RoundDigits) }(bar) @@ -217,7 +212,7 @@ func (im *Importer) Close() { }) } -func (im *Importer) startWorker(ctx context.Context, bar *pb.ProgressBar, batchSize, significantFigures, roundDigits int) { +func (im *Importer) startWorker(ctx context.Context, bar barpool.Bar, batchSize, significantFigures, roundDigits int) { var batch []*TimeSeries var dataPoints int var waitForBatch time.Time @@ -252,9 +247,7 @@ func (im *Importer) startWorker(ctx context.Context, bar *pb.ProgressBar, batchS batch = append(batch, ts) dataPoints += len(ts.Values) - if bar != nil { - bar.Add(len(ts.Values)) - } + bar.Add(len(ts.Values)) if dataPoints < batchSize { continue diff --git a/app/vmctl/vm_native.go b/app/vmctl/vm_native.go index bbdbe5f631..dea0fdc752 100644 --- a/app/vmctl/vm_native.go +++ b/app/vmctl/vm_native.go @@ -9,8 +9,6 @@ import ( "sync" "time" - "github.com/cheggaaa/pb/v3" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter" @@ -33,7 +31,6 @@ type vmNativeProcessor struct { rateLimit int64 interCluster bool cc int - isSilent bool isNative bool disablePerMetricRequests bool @@ -82,13 +79,13 @@ func (p *vmNativeProcessor) run(ctx context.Context) error { return fmt.Errorf("failed to get tenants: %w", err) } question := fmt.Sprintf("The following tenants were discovered: %s.\n Continue?", tenants) - if !p.isSilent && !prompt(question) { + if !prompt(question) { return nil } } for _, tenantID := range tenants { - err := p.runBackfilling(ctx, tenantID, ranges, p.isSilent) + err := p.runBackfilling(ctx, tenantID, ranges) if err != nil { return fmt.Errorf("migration failed: %s", err) } @@ -100,7 +97,7 @@ func (p *vmNativeProcessor) run(ctx context.Context) error { return nil } -func (p *vmNativeProcessor) do(ctx context.Context, f native.Filter, srcURL, dstURL string, bar *pb.ProgressBar) error { +func (p *vmNativeProcessor) do(ctx context.Context, f native.Filter, srcURL, dstURL string, bar barpool.Bar) error { retryableFunc := func() error { return p.runSingle(ctx, f, srcURL, dstURL, bar) } attempts, err := p.backoff.Retry(ctx, retryableFunc) @@ -114,15 +111,18 @@ func (p *vmNativeProcessor) do(ctx context.Context, f native.Filter, srcURL, dst return nil } -func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcURL, dstURL string, bar *pb.ProgressBar) error { +func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcURL, dstURL string, bar barpool.Bar) error { reader, err := p.src.ExportPipe(ctx, srcURL, f) if err != nil { return fmt.Errorf("failed to init export pipe: %w", err) } - if p.disablePerMetricRequests && bar != nil { - fmt.Printf("Continue import process with filter %s:\n", f.String()) - reader = bar.NewProxyReader(reader) + if p.disablePerMetricRequests { + pr := bar.NewProxyReader(reader) + if pr != nil { + reader = bar.NewProxyReader(reader) + fmt.Printf("Continue import process with filter %s:\n", f.String()) + } } pr, pw := io.Pipe() @@ -155,7 +155,7 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU return <-importCh } -func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, ranges [][]time.Time, silent bool) error { +func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, ranges [][]time.Time) error { exportAddr := nativeExportAddr importAddr := nativeImportAddr if p.isNative { @@ -194,7 +194,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, "": ranges, } if !p.disablePerMetricRequests { - metrics, err = p.explore(ctx, p.src, tenantID, ranges, silent) + metrics, err = p.explore(ctx, p.src, tenantID, ranges) if err != nil { return fmt.Errorf("failed to explore metric names: %s", err) } @@ -216,26 +216,24 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, // do not prompt for intercluster because there could be many tenants, // and we don't want to interrupt the process when moving to the next tenant. question := foundSeriesMsg + ". Continue?" - if !silent && !prompt(question) { + if !prompt(question) { return nil } } else { log.Print(foundSeriesMsg) } - var bar *pb.ProgressBar barPrefix := "Requests to make" if p.interCluster { barPrefix = fmt.Sprintf("Requests to make for tenant %s", tenantID) } - if !silent { - bar = barpool.NewSingleProgress(fmt.Sprintf(nativeWithBackoffTpl, barPrefix), requestsToMake) - if p.disablePerMetricRequests { - bar = barpool.NewSingleProgress(nativeSingleProcessTpl, 0) - } - bar.Start() - defer bar.Finish() + + bar := barpool.NewSingleProgress(fmt.Sprintf(nativeWithBackoffTpl, barPrefix), requestsToMake) + if p.disablePerMetricRequests { + bar = barpool.NewSingleProgress(nativeSingleProcessTpl, 0) } + bar.Start() + defer bar.Finish() filterCh := make(chan native.Filter) errCh := make(chan error, p.cc) @@ -251,9 +249,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, errCh <- err return } - if bar != nil { - bar.Increment() - } + bar.Increment() } else { if err := p.runSingle(ctx, f, srcURL, dstURL, bar); err != nil { errCh <- err @@ -298,15 +294,12 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, return nil } -func (p *vmNativeProcessor) explore(ctx context.Context, src *native.Client, tenantID string, ranges [][]time.Time, silent bool) (map[string][][]time.Time, error) { +func (p *vmNativeProcessor) explore(ctx context.Context, src *native.Client, tenantID string, ranges [][]time.Time) (map[string][][]time.Time, error) { log.Printf("Exploring metrics...") - var bar *pb.ProgressBar - if !silent { - bar = barpool.NewSingleProgress(fmt.Sprintf(nativeWithBackoffTpl, "Explore requests to make"), len(ranges)) - bar.Start() - defer bar.Finish() - } + bar := barpool.NewSingleProgress(fmt.Sprintf(nativeWithBackoffTpl, "Explore requests to make"), len(ranges)) + bar.Start() + defer bar.Finish() metrics := make(map[string][][]time.Time) for _, r := range ranges { @@ -317,9 +310,7 @@ func (p *vmNativeProcessor) explore(ctx context.Context, src *native.Client, ten for i := range ms { metrics[ms[i]] = append(metrics[ms[i]], r) } - if bar != nil { - bar.Increment() - } + bar.Increment() } return metrics, nil } diff --git a/app/vmctl/vm_native_test.go b/app/vmctl/vm_native_test.go index 78c35612c5..1f24dad560 100644 --- a/app/vmctl/vm_native_test.go +++ b/app/vmctl/vm_native_test.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool" "log" "net/http" "os" @@ -37,6 +38,12 @@ func Test_vmNativeProcessor_run(t *testing.T) { } }() + barpool.Disable(true) + defer func() { + barpool.Disable(false) + }() + defer func() { isSilent = false }() + type fields struct { filter native.Filter dst *native.Client @@ -218,6 +225,7 @@ func Test_vmNativeProcessor_run(t *testing.T) { HTTPClient: &http.Client{Transport: &http.Transport{DisableKeepAlives: false}}, } + isSilent = tt.args.silent p := &vmNativeProcessor{ filter: tt.fields.filter, dst: tt.fields.dst, @@ -227,7 +235,6 @@ func Test_vmNativeProcessor_run(t *testing.T) { rateLimit: tt.fields.rateLimit, interCluster: tt.fields.interCluster, cc: tt.fields.cc, - isSilent: tt.args.silent, isNative: true, } diff --git a/app/vminsert/main.go b/app/vminsert/main.go index 181d00b839..487d4aacde 100644 --- a/app/vminsert/main.go +++ b/app/vminsert/main.go @@ -73,8 +73,8 @@ var ( "See also -opentsdbHTTPListenAddr.useProxyProtocol") opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+ "at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt") - configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg") - reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...") + configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings.") + reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.") maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented") maxLabelValueLen = flag.Int("maxLabelValueLen", 1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented") ) diff --git a/app/vmui/packages/vmui/package-lock.json b/app/vmui/packages/vmui/package-lock.json index 7499f667d7..d895e367aa 100644 --- a/app/vmui/packages/vmui/package-lock.json +++ b/app/vmui/packages/vmui/package-lock.json @@ -11,7 +11,6 @@ "@types/lodash.debounce": "^4.0.6", "@types/lodash.get": "^4.4.6", "@types/lodash.throttle": "^4.1.6", - "@types/marked": "^5.0.0", "@types/node": "^20.4.0", "@types/qs": "^6.9.7", "@types/react-input-mask": "^3.0.2", @@ -22,7 +21,8 @@ "lodash.debounce": "^4.0.8", "lodash.get": "^4.4.2", "lodash.throttle": "^4.1.1", - "marked": "^5.1.0", + "marked": "^12.0.2", + "marked-emoji": "^1.4.0", "preact": "^10.7.1", "qs": "^6.10.3", "react-input-mask": "^2.0.4", @@ -4272,11 +4272,6 @@ "@types/lodash": "*" } }, - "node_modules/@types/marked": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/@types/marked/-/marked-5.0.2.tgz", - "integrity": "sha512-OucS4KMHhFzhz27KxmWg7J+kIYqyqoW5kdIEI319hqARQQUTqhao3M/F+uFnDXD0Rg72iDDZxZNxq5gvctmLlg==" - }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -13598,14 +13593,22 @@ } }, "node_modules/marked": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/marked/-/marked-5.1.2.tgz", - "integrity": "sha512-ahRPGXJpjMjwSOlBoTMZAK7ATXkli5qCPxZ21TG44rx1KEo44bii4ekgTDQPNRQ4Kh7JMb9Ub1PVk1NxRSsorg==", + "version": "12.0.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-12.0.2.tgz", + "integrity": "sha512-qXUm7e/YKFoqFPYPa3Ukg9xlI5cyAtGmyEIzMfW//m6kXwCy2Ps9DYf5ioijFKQ8qyuscrHoY04iJGctu2Kg0Q==", "bin": { "marked": "bin/marked.js" }, "engines": { - "node": ">= 16" + "node": ">= 18" + } + }, + "node_modules/marked-emoji": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/marked-emoji/-/marked-emoji-1.4.0.tgz", + "integrity": "sha512-/2TJfGzXpiBBq+X3akHHbTrAjZPJDwR+7FV6SyQLECnQEfaoVkrpKZJzHhPTAq3Sl/A1l2frMT0u6b38VBBlNg==", + "peerDependencies": { + "marked": ">=4 <13" } }, "node_modules/mdn-data": { diff --git a/app/vmui/packages/vmui/package.json b/app/vmui/packages/vmui/package.json index 6682b1052a..5af4c67b60 100644 --- a/app/vmui/packages/vmui/package.json +++ b/app/vmui/packages/vmui/package.json @@ -7,7 +7,6 @@ "@types/lodash.debounce": "^4.0.6", "@types/lodash.get": "^4.4.6", "@types/lodash.throttle": "^4.1.6", - "@types/marked": "^5.0.0", "@types/node": "^20.4.0", "@types/qs": "^6.9.7", "@types/react-input-mask": "^3.0.2", @@ -18,7 +17,8 @@ "lodash.debounce": "^4.0.8", "lodash.get": "^4.4.2", "lodash.throttle": "^4.1.1", - "marked": "^5.1.0", + "marked": "^12.0.2", + "marked-emoji": "^1.4.0", "preact": "^10.7.1", "qs": "^6.10.3", "react-input-mask": "^2.0.4", diff --git a/app/vmui/packages/vmui/src/AppLogs.tsx b/app/vmui/packages/vmui/src/AppLogs.tsx index 698abf94e7..d041679e00 100644 --- a/app/vmui/packages/vmui/src/AppLogs.tsx +++ b/app/vmui/packages/vmui/src/AppLogs.tsx @@ -4,6 +4,7 @@ import AppContextProvider from "./contexts/AppContextProvider"; import ThemeProvider from "./components/Main/ThemeProvider/ThemeProvider"; import ExploreLogs from "./pages/ExploreLogs/ExploreLogs"; import LogsLayout from "./layouts/LogsLayout/LogsLayout"; +import "./constants/markedPlugins"; const AppLogs: FC = () => { const [loadedTheme, setLoadedTheme] = useState(false); diff --git a/app/vmui/packages/vmui/src/constants/emojis.ts b/app/vmui/packages/vmui/src/constants/emojis.ts new file mode 100644 index 0000000000..9f189d734c --- /dev/null +++ b/app/vmui/packages/vmui/src/constants/emojis.ts @@ -0,0 +1,1915 @@ +export default { + "100": "๐ฏ", + "1234": "๐ข", + "grinning": "๐", + "smiley": "๐", + "smile": "๐", + "grin": "๐", + "laughing": "๐", + "satisfied": "๐", + "sweat_smile": "๐ ", + "rofl": "๐คฃ", + "joy": "๐", + "slightly_smiling_face": "๐", + "upside_down_face": "๐", + "melting_face": "๐ซ ", + "wink": "๐", + "blush": "๐", + "innocent": "๐", + "smiling_face_with_three_hearts": "๐ฅฐ", + "heart_eyes": "๐", + "star_struck": "๐คฉ", + "kissing_heart": "๐", + "kissing": "๐", + "relaxed": "โบ๏ธ", + "kissing_closed_eyes": "๐", + "kissing_smiling_eyes": "๐", + "smiling_face_with_tear": "๐ฅฒ", + "yum": "๐", + "stuck_out_tongue": "๐", + "stuck_out_tongue_winking_eye": "๐", + "zany_face": "๐คช", + "stuck_out_tongue_closed_eyes": "๐", + "money_mouth_face": "๐ค", + "hugs": "๐ค", + "hand_over_mouth": "๐คญ", + "face_with_open_eyes_and_hand_over_mouth": "๐ซข", + "face_with_peeking_eye": "๐ซฃ", + "shushing_face": "๐คซ", + "thinking": "๐ค", + "saluting_face": "๐ซก", + "zipper_mouth_face": "๐ค", + "raised_eyebrow": "๐คจ", + "neutral_face": "๐", + "expressionless": "๐", + "no_mouth": "๐ถ", + "dotted_line_face": "๐ซฅ", + "face_in_clouds": "๐ถโ๐ซ๏ธ", + "smirk": "๐", + "unamused": "๐", + "roll_eyes": "๐", + "grimacing": "๐ฌ", + "face_exhaling": "๐ฎโ๐จ", + "lying_face": "๐คฅ", + "shaking_face": "๐ซจ", + "relieved": "๐", + "pensive": "๐", + "sleepy": "๐ช", + "drooling_face": "๐คค", + "sleeping": "๐ด", + "mask": "๐ท", + "face_with_thermometer": "๐ค", + "face_with_head_bandage": "๐ค", + "nauseated_face": "๐คข", + "vomiting_face": "๐คฎ", + "sneezing_face": "๐คง", + "hot_face": "๐ฅต", + "cold_face": "๐ฅถ", + "woozy_face": "๐ฅด", + "dizzy_face": "๐ต", + "face_with_spiral_eyes": "๐ตโ๐ซ", + "exploding_head": "๐คฏ", + "cowboy_hat_face": "๐ค ", + "partying_face": "๐ฅณ", + "disguised_face": "๐ฅธ", + "sunglasses": "๐", + "nerd_face": "๐ค", + "monocle_face": "๐ง", + "confused": "๐", + "face_with_diagonal_mouth": "๐ซค", + "worried": "๐", + "slightly_frowning_face": "๐", + "frowning_face": "โน๏ธ", + "open_mouth": "๐ฎ", + "hushed": "๐ฏ", + "astonished": "๐ฒ", + "flushed": "๐ณ", + "pleading_face": "๐ฅบ", + "face_holding_back_tears": "๐ฅน", + "frowning": "๐ฆ", + "anguished": "๐ง", + "fearful": "๐จ", + "cold_sweat": "๐ฐ", + "disappointed_relieved": "๐ฅ", + "cry": "๐ข", + "sob": "๐ญ", + "scream": "๐ฑ", + "confounded": "๐", + "persevere": "๐ฃ", + "disappointed": "๐", + "sweat": "๐", + "weary": "๐ฉ", + "tired_face": "๐ซ", + "yawning_face": "๐ฅฑ", + "triumph": "๐ค", + "rage": "๐ก", + "pout": "๐ก", + "angry": "๐ ", + "cursing_face": "๐คฌ", + "smiling_imp": "๐", + "imp": "๐ฟ", + "skull": "๐", + "skull_and_crossbones": "โ ๏ธ", + "hankey": "๐ฉ", + "poop": "๐ฉ", + "shit": "๐ฉ", + "clown_face": "๐คก", + "japanese_ogre": "๐น", + "japanese_goblin": "๐บ", + "ghost": "๐ป", + "alien": "๐ฝ", + "space_invader": "๐พ", + "robot": "๐ค", + "smiley_cat": "๐บ", + "smile_cat": "๐ธ", + "joy_cat": "๐น", + "heart_eyes_cat": "๐ป", + "smirk_cat": "๐ผ", + "kissing_cat": "๐ฝ", + "scream_cat": "๐", + "crying_cat_face": "๐ฟ", + "pouting_cat": "๐พ", + "see_no_evil": "๐", + "hear_no_evil": "๐", + "speak_no_evil": "๐", + "love_letter": "๐", + "cupid": "๐", + "gift_heart": "๐", + "sparkling_heart": "๐", + "heartpulse": "๐", + "heartbeat": "๐", + "revolving_hearts": "๐", + "two_hearts": "๐", + "heart_decoration": "๐", + "heavy_heart_exclamation": "โฃ๏ธ", + "broken_heart": "๐", + "heart_on_fire": "โค๏ธโ๐ฅ", + "mending_heart": "โค๏ธโ๐ฉน", + "heart": "โค๏ธ", + "pink_heart": "๐ฉท", + "orange_heart": "๐งก", + "yellow_heart": "๐", + "green_heart": "๐", + "blue_heart": "๐", + "light_blue_heart": "๐ฉต", + "purple_heart": "๐", + "brown_heart": "๐ค", + "black_heart": "๐ค", + "grey_heart": "๐ฉถ", + "white_heart": "๐ค", + "kiss": "๐", + "anger": "๐ข", + "boom": "๐ฅ", + "collision": "๐ฅ", + "dizzy": "๐ซ", + "sweat_drops": "๐ฆ", + "dash": "๐จ", + "hole": "๐ณ๏ธ", + "speech_balloon": "๐ฌ", + "eye_speech_bubble": "๐๏ธโ๐จ๏ธ", + "left_speech_bubble": "๐จ๏ธ", + "right_anger_bubble": "๐ฏ๏ธ", + "thought_balloon": "๐ญ", + "zzz": "๐ค", + "wave": "๐", + "raised_back_of_hand": "๐ค", + "raised_hand_with_fingers_splayed": "๐๏ธ", + "hand": "โ", + "raised_hand": "โ", + "vulcan_salute": "๐", + "rightwards_hand": "๐ซฑ", + "leftwards_hand": "๐ซฒ", + "palm_down_hand": "๐ซณ", + "palm_up_hand": "๐ซด", + "leftwards_pushing_hand": "๐ซท", + "rightwards_pushing_hand": "๐ซธ", + "ok_hand": "๐", + "pinched_fingers": "๐ค", + "pinching_hand": "๐ค", + "v": "โ๏ธ", + "crossed_fingers": "๐ค", + "hand_with_index_finger_and_thumb_crossed": "๐ซฐ", + "love_you_gesture": "๐ค", + "metal": "๐ค", + "call_me_hand": "๐ค", + "point_left": "๐", + "point_right": "๐", + "point_up_2": "๐", + "middle_finger": "๐", + "fu": "๐", + "point_down": "๐", + "point_up": "โ๏ธ", + "index_pointing_at_the_viewer": "๐ซต", + "+1": "๐", + "thumbsup": "๐", + "-1": "๐", + "thumbsdown": "๐", + "fist_raised": "โ", + "fist": "โ", + "fist_oncoming": "๐", + "facepunch": "๐", + "punch": "๐", + "fist_left": "๐ค", + "fist_right": "๐ค", + "clap": "๐", + "raised_hands": "๐", + "heart_hands": "๐ซถ", + "open_hands": "๐", + "palms_up_together": "๐คฒ", + "handshake": "๐ค", + "pray": "๐", + "writing_hand": "โ๏ธ", + "nail_care": "๐ ", + "selfie": "๐คณ", + "muscle": "๐ช", + "mechanical_arm": "๐ฆพ", + "mechanical_leg": "๐ฆฟ", + "leg": "๐ฆต", + "foot": "๐ฆถ", + "ear": "๐", + "ear_with_hearing_aid": "๐ฆป", + "nose": "๐", + "brain": "๐ง ", + "anatomical_heart": "๐ซ", + "lungs": "๐ซ", + "tooth": "๐ฆท", + "bone": "๐ฆด", + "eyes": "๐", + "eye": "๐๏ธ", + "tongue": "๐ ", + "lips": "๐", + "biting_lip": "๐ซฆ", + "baby": "๐ถ", + "child": "๐ง", + "boy": "๐ฆ", + "girl": "๐ง", + "adult": "๐ง", + "blond_haired_person": "๐ฑ", + "man": "๐จ", + "bearded_person": "๐ง", + "man_beard": "๐งโโ๏ธ", + "woman_beard": "๐งโโ๏ธ", + "red_haired_man": "๐จโ๐ฆฐ", + "curly_haired_man": "๐จโ๐ฆฑ", + "white_haired_man": "๐จโ๐ฆณ", + "bald_man": "๐จโ๐ฆฒ", + "woman": "๐ฉ", + "red_haired_woman": "๐ฉโ๐ฆฐ", + "person_red_hair": "๐งโ๐ฆฐ", + "curly_haired_woman": "๐ฉโ๐ฆฑ", + "person_curly_hair": "๐งโ๐ฆฑ", + "white_haired_woman": "๐ฉโ๐ฆณ", + "person_white_hair": "๐งโ๐ฆณ", + "bald_woman": "๐ฉโ๐ฆฒ", + "person_bald": "๐งโ๐ฆฒ", + "blond_haired_woman": "๐ฑโโ๏ธ", + "blonde_woman": "๐ฑโโ๏ธ", + "blond_haired_man": "๐ฑโโ๏ธ", + "older_adult": "๐ง", + "older_man": "๐ด", + "older_woman": "๐ต", + "frowning_person": "๐", + "frowning_man": "๐โโ๏ธ", + "frowning_woman": "๐โโ๏ธ", + "pouting_face": "๐", + "pouting_man": "๐โโ๏ธ", + "pouting_woman": "๐โโ๏ธ", + "no_good": "๐ ", + "no_good_man": "๐ โโ๏ธ", + "ng_man": "๐ โโ๏ธ", + "no_good_woman": "๐ โโ๏ธ", + "ng_woman": "๐ โโ๏ธ", + "ok_person": "๐", + "ok_man": "๐โโ๏ธ", + "ok_woman": "๐โโ๏ธ", + "tipping_hand_person": "๐", + "information_desk_person": "๐", + "tipping_hand_man": "๐โโ๏ธ", + "sassy_man": "๐โโ๏ธ", + "tipping_hand_woman": "๐โโ๏ธ", + "sassy_woman": "๐โโ๏ธ", + "raising_hand": "๐", + "raising_hand_man": "๐โโ๏ธ", + "raising_hand_woman": "๐โโ๏ธ", + "deaf_person": "๐ง", + "deaf_man": "๐งโโ๏ธ", + "deaf_woman": "๐งโโ๏ธ", + "bow": "๐", + "bowing_man": "๐โโ๏ธ", + "bowing_woman": "๐โโ๏ธ", + "facepalm": "๐คฆ", + "man_facepalming": "๐คฆโโ๏ธ", + "woman_facepalming": "๐คฆโโ๏ธ", + "shrug": "๐คท", + "man_shrugging": "๐คทโโ๏ธ", + "woman_shrugging": "๐คทโโ๏ธ", + "health_worker": "๐งโโ๏ธ", + "man_health_worker": "๐จโโ๏ธ", + "woman_health_worker": "๐ฉโโ๏ธ", + "student": "๐งโ๐", + "man_student": "๐จโ๐", + "woman_student": "๐ฉโ๐", + "teacher": "๐งโ๐ซ", + "man_teacher": "๐จโ๐ซ", + "woman_teacher": "๐ฉโ๐ซ", + "judge": "๐งโโ๏ธ", + "man_judge": "๐จโโ๏ธ", + "woman_judge": "๐ฉโโ๏ธ", + "farmer": "๐งโ๐พ", + "man_farmer": "๐จโ๐พ", + "woman_farmer": "๐ฉโ๐พ", + "cook": "๐งโ๐ณ", + "man_cook": "๐จโ๐ณ", + "woman_cook": "๐ฉโ๐ณ", + "mechanic": "๐งโ๐ง", + "man_mechanic": "๐จโ๐ง", + "woman_mechanic": "๐ฉโ๐ง", + "factory_worker": "๐งโ๐ญ", + "man_factory_worker": "๐จโ๐ญ", + "woman_factory_worker": "๐ฉโ๐ญ", + "office_worker": "๐งโ๐ผ", + "man_office_worker": "๐จโ๐ผ", + "woman_office_worker": "๐ฉโ๐ผ", + "scientist": "๐งโ๐ฌ", + "man_scientist": "๐จโ๐ฌ", + "woman_scientist": "๐ฉโ๐ฌ", + "technologist": "๐งโ๐ป", + "man_technologist": "๐จโ๐ป", + "woman_technologist": "๐ฉโ๐ป", + "singer": "๐งโ๐ค", + "man_singer": "๐จโ๐ค", + "woman_singer": "๐ฉโ๐ค", + "artist": "๐งโ๐จ", + "man_artist": "๐จโ๐จ", + "woman_artist": "๐ฉโ๐จ", + "pilot": "๐งโโ๏ธ", + "man_pilot": "๐จโโ๏ธ", + "woman_pilot": "๐ฉโโ๏ธ", + "astronaut": "๐งโ๐", + "man_astronaut": "๐จโ๐", + "woman_astronaut": "๐ฉโ๐", + "firefighter": "๐งโ๐", + "man_firefighter": "๐จโ๐", + "woman_firefighter": "๐ฉโ๐", + "police_officer": "๐ฎ", + "cop": "๐ฎ", + "policeman": "๐ฎโโ๏ธ", + "policewoman": "๐ฎโโ๏ธ", + "detective": "๐ต๏ธ", + "male_detective": "๐ต๏ธโโ๏ธ", + "female_detective": "๐ต๏ธโโ๏ธ", + "guard": "๐", + "guardsman": "๐โโ๏ธ", + "guardswoman": "๐โโ๏ธ", + "ninja": "๐ฅท", + "construction_worker": "๐ท", + "construction_worker_man": "๐ทโโ๏ธ", + "construction_worker_woman": "๐ทโโ๏ธ", + "person_with_crown": "๐ซ ", + "prince": "๐คด", + "princess": "๐ธ", + "person_with_turban": "๐ณ", + "man_with_turban": "๐ณโโ๏ธ", + "woman_with_turban": "๐ณโโ๏ธ", + "man_with_gua_pi_mao": "๐ฒ", + "woman_with_headscarf": "๐ง", + "person_in_tuxedo": "๐คต", + "man_in_tuxedo": "๐คตโโ๏ธ", + "woman_in_tuxedo": "๐คตโโ๏ธ", + "person_with_veil": "๐ฐ", + "man_with_veil": "๐ฐโโ๏ธ", + "woman_with_veil": "๐ฐโโ๏ธ", + "bride_with_veil": "๐ฐโโ๏ธ", + "pregnant_woman": "๐คฐ", + "pregnant_man": "๐ซ", + "pregnant_person": "๐ซ", + "breast_feeding": "๐คฑ", + "woman_feeding_baby": "๐ฉโ๐ผ", + "man_feeding_baby": "๐จโ๐ผ", + "person_feeding_baby": "๐งโ๐ผ", + "angel": "๐ผ", + "santa": "๐ ", + "mrs_claus": "๐คถ", + "mx_claus": "๐งโ๐", + "superhero": "๐ฆธ", + "superhero_man": "๐ฆธโโ๏ธ", + "superhero_woman": "๐ฆธโโ๏ธ", + "supervillain": "๐ฆน", + "supervillain_man": "๐ฆนโโ๏ธ", + "supervillain_woman": "๐ฆนโโ๏ธ", + "mage": "๐ง", + "mage_man": "๐งโโ๏ธ", + "mage_woman": "๐งโโ๏ธ", + "fairy": "๐ง", + "fairy_man": "๐งโโ๏ธ", + "fairy_woman": "๐งโโ๏ธ", + "vampire": "๐ง", + "vampire_man": "๐งโโ๏ธ", + "vampire_woman": "๐งโโ๏ธ", + "merperson": "๐ง", + "merman": "๐งโโ๏ธ", + "mermaid": "๐งโโ๏ธ", + "elf": "๐ง", + "elf_man": "๐งโโ๏ธ", + "elf_woman": "๐งโโ๏ธ", + "genie": "๐ง", + "genie_man": "๐งโโ๏ธ", + "genie_woman": "๐งโโ๏ธ", + "zombie": "๐ง", + "zombie_man": "๐งโโ๏ธ", + "zombie_woman": "๐งโโ๏ธ", + "troll": "๐ง", + "massage": "๐", + "massage_man": "๐โโ๏ธ", + "massage_woman": "๐โโ๏ธ", + "haircut": "๐", + "haircut_man": "๐โโ๏ธ", + "haircut_woman": "๐โโ๏ธ", + "walking": "๐ถ", + "walking_man": "๐ถโโ๏ธ", + "walking_woman": "๐ถโโ๏ธ", + "standing_person": "๐ง", + "standing_man": "๐งโโ๏ธ", + "standing_woman": "๐งโโ๏ธ", + "kneeling_person": "๐ง", + "kneeling_man": "๐งโโ๏ธ", + "kneeling_woman": "๐งโโ๏ธ", + "person_with_probing_cane": "๐งโ๐ฆฏ", + "man_with_probing_cane": "๐จโ๐ฆฏ", + "woman_with_probing_cane": "๐ฉโ๐ฆฏ", + "person_in_motorized_wheelchair": "๐งโ๐ฆผ", + "man_in_motorized_wheelchair": "๐จโ๐ฆผ", + "woman_in_motorized_wheelchair": "๐ฉโ๐ฆผ", + "person_in_manual_wheelchair": "๐งโ๐ฆฝ", + "man_in_manual_wheelchair": "๐จโ๐ฆฝ", + "woman_in_manual_wheelchair": "๐ฉโ๐ฆฝ", + "runner": "๐", + "running": "๐", + "running_man": "๐โโ๏ธ", + "running_woman": "๐โโ๏ธ", + "woman_dancing": "๐", + "dancer": "๐", + "man_dancing": "๐บ", + "business_suit_levitating": "๐ด๏ธ", + "dancers": "๐ฏ", + "dancing_men": "๐ฏโโ๏ธ", + "dancing_women": "๐ฏโโ๏ธ", + "sauna_person": "๐ง", + "sauna_man": "๐งโโ๏ธ", + "sauna_woman": "๐งโโ๏ธ", + "climbing": "๐ง", + "climbing_man": "๐งโโ๏ธ", + "climbing_woman": "๐งโโ๏ธ", + "person_fencing": "๐คบ", + "horse_racing": "๐", + "skier": "โท๏ธ", + "snowboarder": "๐", + "golfing": "๐๏ธ", + "golfing_man": "๐๏ธโโ๏ธ", + "golfing_woman": "๐๏ธโโ๏ธ", + "surfer": "๐", + "surfing_man": "๐โโ๏ธ", + "surfing_woman": "๐โโ๏ธ", + "rowboat": "๐ฃ", + "rowing_man": "๐ฃโโ๏ธ", + "rowing_woman": "๐ฃโโ๏ธ", + "swimmer": "๐", + "swimming_man": "๐โโ๏ธ", + "swimming_woman": "๐โโ๏ธ", + "bouncing_ball_person": "โน๏ธ", + "bouncing_ball_man": "โน๏ธโโ๏ธ", + "basketball_man": "โน๏ธโโ๏ธ", + "bouncing_ball_woman": "โน๏ธโโ๏ธ", + "basketball_woman": "โน๏ธโโ๏ธ", + "weight_lifting": "๐๏ธ", + "weight_lifting_man": "๐๏ธโโ๏ธ", + "weight_lifting_woman": "๐๏ธโโ๏ธ", + "bicyclist": "๐ด", + "biking_man": "๐ดโโ๏ธ", + "biking_woman": "๐ดโโ๏ธ", + "mountain_bicyclist": "๐ต", + "mountain_biking_man": "๐ตโโ๏ธ", + "mountain_biking_woman": "๐ตโโ๏ธ", + "cartwheeling": "๐คธ", + "man_cartwheeling": "๐คธโโ๏ธ", + "woman_cartwheeling": "๐คธโโ๏ธ", + "wrestling": "๐คผ", + "men_wrestling": "๐คผโโ๏ธ", + "women_wrestling": "๐คผโโ๏ธ", + "water_polo": "๐คฝ", + "man_playing_water_polo": "๐คฝโโ๏ธ", + "woman_playing_water_polo": "๐คฝโโ๏ธ", + "handball_person": "๐คพ", + "man_playing_handball": "๐คพโโ๏ธ", + "woman_playing_handball": "๐คพโโ๏ธ", + "juggling_person": "๐คน", + "man_juggling": "๐คนโโ๏ธ", + "woman_juggling": "๐คนโโ๏ธ", + "lotus_position": "๐ง", + "lotus_position_man": "๐งโโ๏ธ", + "lotus_position_woman": "๐งโโ๏ธ", + "bath": "๐", + "sleeping_bed": "๐", + "people_holding_hands": "๐งโ๐คโ๐ง", + "two_women_holding_hands": "๐ญ", + "couple": "๐ซ", + "two_men_holding_hands": "๐ฌ", + "couplekiss": "๐", + "couplekiss_man_woman": "๐ฉโโค๏ธโ๐โ๐จ", + "couplekiss_man_man": "๐จโโค๏ธโ๐โ๐จ", + "couplekiss_woman_woman": "๐ฉโโค๏ธโ๐โ๐ฉ", + "couple_with_heart": "๐", + "couple_with_heart_woman_man": "๐ฉโโค๏ธโ๐จ", + "couple_with_heart_man_man": "๐จโโค๏ธโ๐จ", + "couple_with_heart_woman_woman": "๐ฉโโค๏ธโ๐ฉ", + "family": "๐ช", + "family_man_woman_boy": "๐จโ๐ฉโ๐ฆ", + "family_man_woman_girl": "๐จโ๐ฉโ๐ง", + "family_man_woman_girl_boy": "๐จโ๐ฉโ๐งโ๐ฆ", + "family_man_woman_boy_boy": "๐จโ๐ฉโ๐ฆโ๐ฆ", + "family_man_woman_girl_girl": "๐จโ๐ฉโ๐งโ๐ง", + "family_man_man_boy": "๐จโ๐จโ๐ฆ", + "family_man_man_girl": "๐จโ๐จโ๐ง", + "family_man_man_girl_boy": "๐จโ๐จโ๐งโ๐ฆ", + "family_man_man_boy_boy": "๐จโ๐จโ๐ฆโ๐ฆ", + "family_man_man_girl_girl": "๐จโ๐จโ๐งโ๐ง", + "family_woman_woman_boy": "๐ฉโ๐ฉโ๐ฆ", + "family_woman_woman_girl": "๐ฉโ๐ฉโ๐ง", + "family_woman_woman_girl_boy": "๐ฉโ๐ฉโ๐งโ๐ฆ", + "family_woman_woman_boy_boy": "๐ฉโ๐ฉโ๐ฆโ๐ฆ", + "family_woman_woman_girl_girl": "๐ฉโ๐ฉโ๐งโ๐ง", + "family_man_boy": "๐จโ๐ฆ", + "family_man_boy_boy": "๐จโ๐ฆโ๐ฆ", + "family_man_girl": "๐จโ๐ง", + "family_man_girl_boy": "๐จโ๐งโ๐ฆ", + "family_man_girl_girl": "๐จโ๐งโ๐ง", + "family_woman_boy": "๐ฉโ๐ฆ", + "family_woman_boy_boy": "๐ฉโ๐ฆโ๐ฆ", + "family_woman_girl": "๐ฉโ๐ง", + "family_woman_girl_boy": "๐ฉโ๐งโ๐ฆ", + "family_woman_girl_girl": "๐ฉโ๐งโ๐ง", + "speaking_head": "๐ฃ๏ธ", + "bust_in_silhouette": "๐ค", + "busts_in_silhouette": "๐ฅ", + "people_hugging": "๐ซ", + "footprints": "๐ฃ", + "monkey_face": "๐ต", + "monkey": "๐", + "gorilla": "๐ฆ", + "orangutan": "๐ฆง", + "dog": "๐ถ", + "dog2": "๐", + "guide_dog": "๐ฆฎ", + "service_dog": "๐โ๐ฆบ", + "poodle": "๐ฉ", + "wolf": "๐บ", + "fox_face": "๐ฆ", + "raccoon": "๐ฆ", + "cat": "๐ฑ", + "cat2": "๐", + "black_cat": "๐โโฌ", + "lion": "๐ฆ", + "tiger": "๐ฏ", + "tiger2": "๐ ", + "leopard": "๐", + "horse": "๐ด", + "moose": "๐ซ", + "donkey": "๐ซ", + "racehorse": "๐", + "unicorn": "๐ฆ", + "zebra": "๐ฆ", + "deer": "๐ฆ", + "bison": "๐ฆฌ", + "cow": "๐ฎ", + "ox": "๐", + "water_buffalo": "๐", + "cow2": "๐", + "pig": "๐ท", + "pig2": "๐", + "boar": "๐", + "pig_nose": "๐ฝ", + "ram": "๐", + "sheep": "๐", + "goat": "๐", + "dromedary_camel": "๐ช", + "camel": "๐ซ", + "llama": "๐ฆ", + "giraffe": "๐ฆ", + "elephant": "๐", + "mammoth": "๐ฆฃ", + "rhinoceros": "๐ฆ", + "hippopotamus": "๐ฆ", + "mouse": "๐ญ", + "mouse2": "๐", + "rat": "๐", + "hamster": "๐น", + "rabbit": "๐ฐ", + "rabbit2": "๐", + "chipmunk": "๐ฟ๏ธ", + "beaver": "๐ฆซ", + "hedgehog": "๐ฆ", + "bat": "๐ฆ", + "bear": "๐ป", + "polar_bear": "๐ปโโ๏ธ", + "koala": "๐จ", + "panda_face": "๐ผ", + "sloth": "๐ฆฅ", + "otter": "๐ฆฆ", + "skunk": "๐ฆจ", + "kangaroo": "๐ฆ", + "badger": "๐ฆก", + "feet": "๐พ", + "paw_prints": "๐พ", + "turkey": "๐ฆ", + "chicken": "๐", + "rooster": "๐", + "hatching_chick": "๐ฃ", + "baby_chick": "๐ค", + "hatched_chick": "๐ฅ", + "bird": "๐ฆ", + "penguin": "๐ง", + "dove": "๐๏ธ", + "eagle": "๐ฆ ", + "duck": "๐ฆ", + "swan": "๐ฆข", + "owl": "๐ฆ", + "dodo": "๐ฆค", + "feather": "๐ชถ", + "flamingo": "๐ฆฉ", + "peacock": "๐ฆ", + "parrot": "๐ฆ", + "wing": "๐ชฝ", + "black_bird": "๐ฆโโฌ", + "goose": "๐ชฟ", + "frog": "๐ธ", + "crocodile": "๐", + "turtle": "๐ข", + "lizard": "๐ฆ", + "snake": "๐", + "dragon_face": "๐ฒ", + "dragon": "๐", + "sauropod": "๐ฆ", + "t-rex": "๐ฆ", + "whale": "๐ณ", + "whale2": "๐", + "dolphin": "๐ฌ", + "flipper": "๐ฌ", + "seal": "๐ฆญ", + "fish": "๐", + "tropical_fish": "๐ ", + "blowfish": "๐ก", + "shark": "๐ฆ", + "octopus": "๐", + "shell": "๐", + "coral": "๐ชธ", + "jellyfish": "๐ชผ", + "snail": "๐", + "butterfly": "๐ฆ", + "bug": "๐", + "ant": "๐", + "bee": "๐", + "honeybee": "๐", + "beetle": "๐ชฒ", + "lady_beetle": "๐", + "cricket": "๐ฆ", + "cockroach": "๐ชณ", + "spider": "๐ท๏ธ", + "spider_web": "๐ธ๏ธ", + "scorpion": "๐ฆ", + "mosquito": "๐ฆ", + "fly": "๐ชฐ", + "worm": "๐ชฑ", + "microbe": "๐ฆ ", + "bouquet": "๐", + "cherry_blossom": "๐ธ", + "white_flower": "๐ฎ", + "lotus": "๐ชท", + "rosette": "๐ต๏ธ", + "rose": "๐น", + "wilted_flower": "๐ฅ", + "hibiscus": "๐บ", + "sunflower": "๐ป", + "blossom": "๐ผ", + "tulip": "๐ท", + "hyacinth": "๐ชป", + "seedling": "๐ฑ", + "potted_plant": "๐ชด", + "evergreen_tree": "๐ฒ", + "deciduous_tree": "๐ณ", + "palm_tree": "๐ด", + "cactus": "๐ต", + "ear_of_rice": "๐พ", + "herb": "๐ฟ", + "shamrock": "โ๏ธ", + "four_leaf_clover": "๐", + "maple_leaf": "๐", + "fallen_leaf": "๐", + "leaves": "๐", + "empty_nest": "๐ชน", + "nest_with_eggs": "๐ชบ", + "mushroom": "๐", + "grapes": "๐", + "melon": "๐", + "watermelon": "๐", + "tangerine": "๐", + "orange": "๐", + "mandarin": "๐", + "lemon": "๐", + "banana": "๐", + "pineapple": "๐", + "mango": "๐ฅญ", + "apple": "๐", + "green_apple": "๐", + "pear": "๐", + "peach": "๐", + "cherries": "๐", + "strawberry": "๐", + "blueberries": "๐ซ", + "kiwi_fruit": "๐ฅ", + "tomato": "๐ ", + "olive": "๐ซ", + "coconut": "๐ฅฅ", + "avocado": "๐ฅ", + "eggplant": "๐", + "potato": "๐ฅ", + "carrot": "๐ฅ", + "corn": "๐ฝ", + "hot_pepper": "๐ถ๏ธ", + "bell_pepper": "๐ซ", + "cucumber": "๐ฅ", + "leafy_green": "๐ฅฌ", + "broccoli": "๐ฅฆ", + "garlic": "๐ง", + "onion": "๐ง ", + "peanuts": "๐ฅ", + "beans": "๐ซ", + "chestnut": "๐ฐ", + "ginger_root": "๐ซ", + "pea_pod": "๐ซ", + "bread": "๐", + "croissant": "๐ฅ", + "baguette_bread": "๐ฅ", + "flatbread": "๐ซ", + "pretzel": "๐ฅจ", + "bagel": "๐ฅฏ", + "pancakes": "๐ฅ", + "waffle": "๐ง", + "cheese": "๐ง", + "meat_on_bone": "๐", + "poultry_leg": "๐", + "cut_of_meat": "๐ฅฉ", + "bacon": "๐ฅ", + "hamburger": "๐", + "fries": "๐", + "pizza": "๐", + "hotdog": "๐ญ", + "sandwich": "๐ฅช", + "taco": "๐ฎ", + "burrito": "๐ฏ", + "tamale": "๐ซ", + "stuffed_flatbread": "๐ฅ", + "falafel": "๐ง", + "egg": "๐ฅ", + "fried_egg": "๐ณ", + "shallow_pan_of_food": "๐ฅ", + "stew": "๐ฒ", + "fondue": "๐ซ", + "bowl_with_spoon": "๐ฅฃ", + "green_salad": "๐ฅ", + "popcorn": "๐ฟ", + "butter": "๐ง", + "salt": "๐ง", + "canned_food": "๐ฅซ", + "bento": "๐ฑ", + "rice_cracker": "๐", + "rice_ball": "๐", + "rice": "๐", + "curry": "๐", + "ramen": "๐", + "spaghetti": "๐", + "sweet_potato": "๐ ", + "oden": "๐ข", + "sushi": "๐ฃ", + "fried_shrimp": "๐ค", + "fish_cake": "๐ฅ", + "moon_cake": "๐ฅฎ", + "dango": "๐ก", + "dumpling": "๐ฅ", + "fortune_cookie": "๐ฅ ", + "takeout_box": "๐ฅก", + "crab": "๐ฆ", + "lobster": "๐ฆ", + "shrimp": "๐ฆ", + "squid": "๐ฆ", + "oyster": "๐ฆช", + "icecream": "๐ฆ", + "shaved_ice": "๐ง", + "ice_cream": "๐จ", + "doughnut": "๐ฉ", + "cookie": "๐ช", + "birthday": "๐", + "cake": "๐ฐ", + "cupcake": "๐ง", + "pie": "๐ฅง", + "chocolate_bar": "๐ซ", + "candy": "๐ฌ", + "lollipop": "๐ญ", + "custard": "๐ฎ", + "honey_pot": "๐ฏ", + "baby_bottle": "๐ผ", + "milk_glass": "๐ฅ", + "coffee": "โ", + "teapot": "๐ซ", + "tea": "๐ต", + "sake": "๐ถ", + "champagne": "๐พ", + "wine_glass": "๐ท", + "cocktail": "๐ธ", + "tropical_drink": "๐น", + "beer": "๐บ", + "beers": "๐ป", + "clinking_glasses": "๐ฅ", + "tumbler_glass": "๐ฅ", + "pouring_liquid": "๐ซ", + "cup_with_straw": "๐ฅค", + "bubble_tea": "๐ง", + "beverage_box": "๐ง", + "mate": "๐ง", + "ice_cube": "๐ง", + "chopsticks": "๐ฅข", + "plate_with_cutlery": "๐ฝ๏ธ", + "fork_and_knife": "๐ด", + "spoon": "๐ฅ", + "hocho": "๐ช", + "knife": "๐ช", + "jar": "๐ซ", + "amphora": "๐บ", + "earth_africa": "๐", + "earth_americas": "๐", + "earth_asia": "๐", + "globe_with_meridians": "๐", + "world_map": "๐บ๏ธ", + "japan": "๐พ", + "compass": "๐งญ", + "mountain_snow": "๐๏ธ", + "mountain": "โฐ๏ธ", + "volcano": "๐", + "mount_fuji": "๐ป", + "camping": "๐๏ธ", + "beach_umbrella": "๐๏ธ", + "desert": "๐๏ธ", + "desert_island": "๐๏ธ", + "national_park": "๐๏ธ", + "stadium": "๐๏ธ", + "classical_building": "๐๏ธ", + "building_construction": "๐๏ธ", + "bricks": "๐งฑ", + "rock": "๐ชจ", + "wood": "๐ชต", + "hut": "๐", + "houses": "๐๏ธ", + "derelict_house": "๐๏ธ", + "house": "๐ ", + "house_with_garden": "๐ก", + "office": "๐ข", + "post_office": "๐ฃ", + "european_post_office": "๐ค", + "hospital": "๐ฅ", + "bank": "๐ฆ", + "hotel": "๐จ", + "love_hotel": "๐ฉ", + "convenience_store": "๐ช", + "school": "๐ซ", + "department_store": "๐ฌ", + "factory": "๐ญ", + "japanese_castle": "๐ฏ", + "european_castle": "๐ฐ", + "wedding": "๐", + "tokyo_tower": "๐ผ", + "statue_of_liberty": "๐ฝ", + "church": "โช", + "mosque": "๐", + "hindu_temple": "๐", + "synagogue": "๐", + "shinto_shrine": "โฉ๏ธ", + "kaaba": "๐", + "fountain": "โฒ", + "tent": "โบ", + "foggy": "๐", + "night_with_stars": "๐", + "cityscape": "๐๏ธ", + "sunrise_over_mountains": "๐", + "sunrise": "๐ ", + "city_sunset": "๐", + "city_sunrise": "๐", + "bridge_at_night": "๐", + "hotsprings": "โจ๏ธ", + "carousel_horse": "๐ ", + "playground_slide": "๐", + "ferris_wheel": "๐ก", + "roller_coaster": "๐ข", + "barber": "๐", + "circus_tent": "๐ช", + "steam_locomotive": "๐", + "railway_car": "๐", + "bullettrain_side": "๐", + "bullettrain_front": "๐ ", + "train2": "๐", + "metro": "๐", + "light_rail": "๐", + "station": "๐", + "tram": "๐", + "monorail": "๐", + "mountain_railway": "๐", + "train": "๐", + "bus": "๐", + "oncoming_bus": "๐", + "trolleybus": "๐", + "minibus": "๐", + "ambulance": "๐", + "fire_engine": "๐", + "police_car": "๐", + "oncoming_police_car": "๐", + "taxi": "๐", + "oncoming_taxi": "๐", + "car": "๐", + "red_car": "๐", + "oncoming_automobile": "๐", + "blue_car": "๐", + "pickup_truck": "๐ป", + "truck": "๐", + "articulated_lorry": "๐", + "tractor": "๐", + "racing_car": "๐๏ธ", + "motorcycle": "๐๏ธ", + "motor_scooter": "๐ต", + "manual_wheelchair": "๐ฆฝ", + "motorized_wheelchair": "๐ฆผ", + "auto_rickshaw": "๐บ", + "bike": "๐ฒ", + "kick_scooter": "๐ด", + "skateboard": "๐น", + "roller_skate": "๐ผ", + "busstop": "๐", + "motorway": "๐ฃ๏ธ", + "railway_track": "๐ค๏ธ", + "oil_drum": "๐ข๏ธ", + "fuelpump": "โฝ", + "wheel": "๐", + "rotating_light": "๐จ", + "traffic_light": "๐ฅ", + "vertical_traffic_light": "๐ฆ", + "stop_sign": "๐", + "construction": "๐ง", + "anchor": "โ", + "ring_buoy": "๐", + "boat": "โต", + "sailboat": "โต", + "canoe": "๐ถ", + "speedboat": "๐ค", + "passenger_ship": "๐ณ๏ธ", + "ferry": "โด๏ธ", + "motor_boat": "๐ฅ๏ธ", + "ship": "๐ข", + "airplane": "โ๏ธ", + "small_airplane": "๐ฉ๏ธ", + "flight_departure": "๐ซ", + "flight_arrival": "๐ฌ", + "parachute": "๐ช", + "seat": "๐บ", + "helicopter": "๐", + "suspension_railway": "๐", + "mountain_cableway": "๐ ", + "aerial_tramway": "๐ก", + "artificial_satellite": "๐ฐ๏ธ", + "rocket": "๐", + "flying_saucer": "๐ธ", + "bellhop_bell": "๐๏ธ", + "luggage": "๐งณ", + "hourglass": "โ", + "hourglass_flowing_sand": "โณ", + "watch": "โ", + "alarm_clock": "โฐ", + "stopwatch": "โฑ๏ธ", + "timer_clock": "โฒ๏ธ", + "mantelpiece_clock": "๐ฐ๏ธ", + "clock12": "๐", + "clock1230": "๐ง", + "clock1": "๐", + "clock130": "๐", + "clock2": "๐", + "clock230": "๐", + "clock3": "๐", + "clock330": "๐", + "clock4": "๐", + "clock430": "๐", + "clock5": "๐", + "clock530": "๐ ", + "clock6": "๐", + "clock630": "๐ก", + "clock7": "๐", + "clock730": "๐ข", + "clock8": "๐", + "clock830": "๐ฃ", + "clock9": "๐", + "clock930": "๐ค", + "clock10": "๐", + "clock1030": "๐ฅ", + "clock11": "๐", + "clock1130": "๐ฆ", + "new_moon": "๐", + "waxing_crescent_moon": "๐", + "first_quarter_moon": "๐", + "moon": "๐", + "waxing_gibbous_moon": "๐", + "full_moon": "๐", + "waning_gibbous_moon": "๐", + "last_quarter_moon": "๐", + "waning_crescent_moon": "๐", + "crescent_moon": "๐", + "new_moon_with_face": "๐", + "first_quarter_moon_with_face": "๐", + "last_quarter_moon_with_face": "๐", + "thermometer": "๐ก๏ธ", + "sunny": "โ๏ธ", + "full_moon_with_face": "๐", + "sun_with_face": "๐", + "ringed_planet": "๐ช", + "star": "โญ", + "star2": "๐", + "stars": "๐ ", + "milky_way": "๐", + "cloud": "โ๏ธ", + "partly_sunny": "โ ", + "cloud_with_lightning_and_rain": "โ๏ธ", + "sun_behind_small_cloud": "๐ค๏ธ", + "sun_behind_large_cloud": "๐ฅ๏ธ", + "sun_behind_rain_cloud": "๐ฆ๏ธ", + "cloud_with_rain": "๐ง๏ธ", + "cloud_with_snow": "๐จ๏ธ", + "cloud_with_lightning": "๐ฉ๏ธ", + "tornado": "๐ช๏ธ", + "fog": "๐ซ๏ธ", + "wind_face": "๐ฌ๏ธ", + "cyclone": "๐", + "rainbow": "๐", + "closed_umbrella": "๐", + "open_umbrella": "โ๏ธ", + "umbrella": "โ", + "parasol_on_ground": "โฑ๏ธ", + "zap": "โก", + "snowflake": "โ๏ธ", + "snowman_with_snow": "โ๏ธ", + "snowman": "โ", + "comet": "โ๏ธ", + "fire": "๐ฅ", + "droplet": "๐ง", + "ocean": "๐", + "jack_o_lantern": "๐", + "christmas_tree": "๐", + "fireworks": "๐", + "sparkler": "๐", + "firecracker": "๐งจ", + "sparkles": "โจ", + "balloon": "๐", + "tada": "๐", + "confetti_ball": "๐", + "tanabata_tree": "๐", + "bamboo": "๐", + "dolls": "๐", + "flags": "๐", + "wind_chime": "๐", + "rice_scene": "๐", + "red_envelope": "๐งง", + "ribbon": "๐", + "gift": "๐", + "reminder_ribbon": "๐๏ธ", + "tickets": "๐๏ธ", + "ticket": "๐ซ", + "medal_military": "๐๏ธ", + "trophy": "๐", + "medal_sports": "๐ ", + "1st_place_medal": "๐ฅ", + "2nd_place_medal": "๐ฅ", + "3rd_place_medal": "๐ฅ", + "soccer": "โฝ", + "baseball": "โพ", + "softball": "๐ฅ", + "basketball": "๐", + "volleyball": "๐", + "football": "๐", + "rugby_football": "๐", + "tennis": "๐พ", + "flying_disc": "๐ฅ", + "bowling": "๐ณ", + "cricket_game": "๐", + "field_hockey": "๐", + "ice_hockey": "๐", + "lacrosse": "๐ฅ", + "ping_pong": "๐", + "badminton": "๐ธ", + "boxing_glove": "๐ฅ", + "martial_arts_uniform": "๐ฅ", + "goal_net": "๐ฅ ", + "golf": "โณ", + "ice_skate": "โธ๏ธ", + "fishing_pole_and_fish": "๐ฃ", + "diving_mask": "๐คฟ", + "running_shirt_with_sash": "๐ฝ", + "ski": "๐ฟ", + "sled": "๐ท", + "curling_stone": "๐ฅ", + "dart": "๐ฏ", + "yo_yo": "๐ช", + "kite": "๐ช", + "gun": "๐ซ", + "8ball": "๐ฑ", + "crystal_ball": "๐ฎ", + "magic_wand": "๐ช", + "video_game": "๐ฎ", + "joystick": "๐น๏ธ", + "slot_machine": "๐ฐ", + "game_die": "๐ฒ", + "jigsaw": "๐งฉ", + "teddy_bear": "๐งธ", + "pinata": "๐ช ", + "mirror_ball": "๐ชฉ", + "nesting_dolls": "๐ช", + "spades": "โ ๏ธ", + "hearts": "โฅ๏ธ", + "diamonds": "โฆ๏ธ", + "clubs": "โฃ๏ธ", + "chess_pawn": "โ๏ธ", + "black_joker": "๐", + "mahjong": "๐", + "flower_playing_cards": "๐ด", + "performing_arts": "๐ญ", + "framed_picture": "๐ผ๏ธ", + "art": "๐จ", + "thread": "๐งต", + "sewing_needle": "๐ชก", + "yarn": "๐งถ", + "knot": "๐ชข", + "eyeglasses": "๐", + "dark_sunglasses": "๐ถ๏ธ", + "goggles": "๐ฅฝ", + "lab_coat": "๐ฅผ", + "safety_vest": "๐ฆบ", + "necktie": "๐", + "shirt": "๐", + "tshirt": "๐", + "jeans": "๐", + "scarf": "๐งฃ", + "gloves": "๐งค", + "coat": "๐งฅ", + "socks": "๐งฆ", + "dress": "๐", + "kimono": "๐", + "sari": "๐ฅป", + "one_piece_swimsuit": "๐ฉฑ", + "swim_brief": "๐ฉฒ", + "shorts": "๐ฉณ", + "bikini": "๐", + "womans_clothes": "๐", + "folding_hand_fan": "๐ชญ", + "purse": "๐", + "handbag": "๐", + "pouch": "๐", + "shopping": "๐๏ธ", + "school_satchel": "๐", + "thong_sandal": "๐ฉด", + "mans_shoe": "๐", + "shoe": "๐", + "athletic_shoe": "๐", + "hiking_boot": "๐ฅพ", + "flat_shoe": "๐ฅฟ", + "high_heel": "๐ ", + "sandal": "๐ก", + "ballet_shoes": "๐ฉฐ", + "boot": "๐ข", + "hair_pick": "๐ชฎ", + "crown": "๐", + "womans_hat": "๐", + "tophat": "๐ฉ", + "mortar_board": "๐", + "billed_cap": "๐งข", + "military_helmet": "๐ช", + "rescue_worker_helmet": "โ๏ธ", + "prayer_beads": "๐ฟ", + "lipstick": "๐", + "ring": "๐", + "gem": "๐", + "mute": "๐", + "speaker": "๐", + "sound": "๐", + "loud_sound": "๐", + "loudspeaker": "๐ข", + "mega": "๐ฃ", + "postal_horn": "๐ฏ", + "bell": "๐", + "no_bell": "๐", + "musical_score": "๐ผ", + "musical_note": "๐ต", + "notes": "๐ถ", + "studio_microphone": "๐๏ธ", + "level_slider": "๐๏ธ", + "control_knobs": "๐๏ธ", + "microphone": "๐ค", + "headphones": "๐ง", + "radio": "๐ป", + "saxophone": "๐ท", + "accordion": "๐ช", + "guitar": "๐ธ", + "musical_keyboard": "๐น", + "trumpet": "๐บ", + "violin": "๐ป", + "banjo": "๐ช", + "drum": "๐ฅ", + "long_drum": "๐ช", + "maracas": "๐ช", + "flute": "๐ช", + "iphone": "๐ฑ", + "calling": "๐ฒ", + "phone": "โ๏ธ", + "telephone": "โ๏ธ", + "telephone_receiver": "๐", + "pager": "๐", + "fax": "๐ ", + "battery": "๐", + "low_battery": "๐ชซ", + "electric_plug": "๐", + "computer": "๐ป", + "desktop_computer": "๐ฅ๏ธ", + "printer": "๐จ๏ธ", + "keyboard": "โจ๏ธ", + "computer_mouse": "๐ฑ๏ธ", + "trackball": "๐ฒ๏ธ", + "minidisc": "๐ฝ", + "floppy_disk": "๐พ", + "cd": "๐ฟ", + "dvd": "๐", + "abacus": "๐งฎ", + "movie_camera": "๐ฅ", + "film_strip": "๐๏ธ", + "film_projector": "๐ฝ๏ธ", + "clapper": "๐ฌ", + "tv": "๐บ", + "camera": "๐ท", + "camera_flash": "๐ธ", + "video_camera": "๐น", + "vhs": "๐ผ", + "mag": "๐", + "mag_right": "๐", + "candle": "๐ฏ๏ธ", + "bulb": "๐ก", + "flashlight": "๐ฆ", + "izakaya_lantern": "๐ฎ", + "lantern": "๐ฎ", + "diya_lamp": "๐ช", + "notebook_with_decorative_cover": "๐", + "closed_book": "๐", + "book": "๐", + "open_book": "๐", + "green_book": "๐", + "blue_book": "๐", + "orange_book": "๐", + "books": "๐", + "notebook": "๐", + "ledger": "๐", + "page_with_curl": "๐", + "scroll": "๐", + "page_facing_up": "๐", + "newspaper": "๐ฐ", + "newspaper_roll": "๐๏ธ", + "bookmark_tabs": "๐", + "bookmark": "๐", + "label": "๐ท๏ธ", + "moneybag": "๐ฐ", + "coin": "๐ช", + "yen": "๐ด", + "dollar": "๐ต", + "euro": "๐ถ", + "pound": "๐ท", + "money_with_wings": "๐ธ", + "credit_card": "๐ณ", + "receipt": "๐งพ", + "chart": "๐น", + "envelope": "โ๏ธ", + "email": "๐ง", + "e-mail": "๐ง", + "incoming_envelope": "๐จ", + "envelope_with_arrow": "๐ฉ", + "outbox_tray": "๐ค", + "inbox_tray": "๐ฅ", + "package": "๐ฆ", + "mailbox": "๐ซ", + "mailbox_closed": "๐ช", + "mailbox_with_mail": "๐ฌ", + "mailbox_with_no_mail": "๐ญ", + "postbox": "๐ฎ", + "ballot_box": "๐ณ๏ธ", + "pencil2": "โ๏ธ", + "black_nib": "โ๏ธ", + "fountain_pen": "๐๏ธ", + "pen": "๐๏ธ", + "paintbrush": "๐๏ธ", + "crayon": "๐๏ธ", + "memo": "๐", + "pencil": "๐", + "briefcase": "๐ผ", + "file_folder": "๐", + "open_file_folder": "๐", + "card_index_dividers": "๐๏ธ", + "date": "๐ ", + "calendar": "๐", + "spiral_notepad": "๐๏ธ", + "spiral_calendar": "๐๏ธ", + "card_index": "๐", + "chart_with_upwards_trend": "๐", + "chart_with_downwards_trend": "๐", + "bar_chart": "๐", + "clipboard": "๐", + "pushpin": "๐", + "round_pushpin": "๐", + "paperclip": "๐", + "paperclips": "๐๏ธ", + "straight_ruler": "๐", + "triangular_ruler": "๐", + "scissors": "โ๏ธ", + "card_file_box": "๐๏ธ", + "file_cabinet": "๐๏ธ", + "wastebasket": "๐๏ธ", + "lock": "๐", + "unlock": "๐", + "lock_with_ink_pen": "๐", + "closed_lock_with_key": "๐", + "key": "๐", + "old_key": "๐๏ธ", + "hammer": "๐จ", + "axe": "๐ช", + "pick": "โ๏ธ", + "hammer_and_pick": "โ๏ธ", + "hammer_and_wrench": "๐ ๏ธ", + "dagger": "๐ก๏ธ", + "crossed_swords": "โ๏ธ", + "bomb": "๐ฃ", + "boomerang": "๐ช", + "bow_and_arrow": "๐น", + "shield": "๐ก๏ธ", + "carpentry_saw": "๐ช", + "wrench": "๐ง", + "screwdriver": "๐ช", + "nut_and_bolt": "๐ฉ", + "gear": "โ๏ธ", + "clamp": "๐๏ธ", + "balance_scale": "โ๏ธ", + "probing_cane": "๐ฆฏ", + "link": "๐", + "chains": "โ๏ธ", + "hook": "๐ช", + "toolbox": "๐งฐ", + "magnet": "๐งฒ", + "ladder": "๐ช", + "alembic": "โ๏ธ", + "test_tube": "๐งช", + "petri_dish": "๐งซ", + "dna": "๐งฌ", + "microscope": "๐ฌ", + "telescope": "๐ญ", + "satellite": "๐ก", + "syringe": "๐", + "drop_of_blood": "๐ฉธ", + "pill": "๐", + "adhesive_bandage": "๐ฉน", + "crutch": "๐ฉผ", + "stethoscope": "๐ฉบ", + "x_ray": "๐ฉป", + "door": "๐ช", + "elevator": "๐", + "mirror": "๐ช", + "window": "๐ช", + "bed": "๐๏ธ", + "couch_and_lamp": "๐๏ธ", + "chair": "๐ช", + "toilet": "๐ฝ", + "plunger": "๐ช ", + "shower": "๐ฟ", + "bathtub": "๐", + "mouse_trap": "๐ชค", + "razor": "๐ช", + "lotion_bottle": "๐งด", + "safety_pin": "๐งท", + "broom": "๐งน", + "basket": "๐งบ", + "roll_of_paper": "๐งป", + "bucket": "๐ชฃ", + "soap": "๐งผ", + "bubbles": "๐ซง", + "toothbrush": "๐ชฅ", + "sponge": "๐งฝ", + "fire_extinguisher": "๐งฏ", + "shopping_cart": "๐", + "smoking": "๐ฌ", + "coffin": "โฐ๏ธ", + "headstone": "๐ชฆ", + "funeral_urn": "โฑ๏ธ", + "nazar_amulet": "๐งฟ", + "hamsa": "๐ชฌ", + "moyai": "๐ฟ", + "placard": "๐ชง", + "identification_card": "๐ชช", + "atm": "๐ง", + "put_litter_in_its_place": "๐ฎ", + "potable_water": "๐ฐ", + "wheelchair": "โฟ", + "mens": "๐น", + "womens": "๐บ", + "restroom": "๐ป", + "baby_symbol": "๐ผ", + "wc": "๐พ", + "passport_control": "๐", + "customs": "๐", + "baggage_claim": "๐", + "left_luggage": "๐ ", + "warning": "โ ๏ธ", + "children_crossing": "๐ธ", + "no_entry": "โ", + "no_entry_sign": "๐ซ", + "no_bicycles": "๐ณ", + "no_smoking": "๐ญ", + "do_not_litter": "๐ฏ", + "non-potable_water": "๐ฑ", + "no_pedestrians": "๐ท", + "no_mobile_phones": "๐ต", + "underage": "๐", + "radioactive": "โข๏ธ", + "biohazard": "โฃ๏ธ", + "arrow_up": "โฌ๏ธ", + "arrow_upper_right": "โ๏ธ", + "arrow_right": "โก๏ธ", + "arrow_lower_right": "โ๏ธ", + "arrow_down": "โฌ๏ธ", + "arrow_lower_left": "โ๏ธ", + "arrow_left": "โฌ ๏ธ", + "arrow_upper_left": "โ๏ธ", + "arrow_up_down": "โ๏ธ", + "left_right_arrow": "โ๏ธ", + "leftwards_arrow_with_hook": "โฉ๏ธ", + "arrow_right_hook": "โช๏ธ", + "arrow_heading_up": "โคด๏ธ", + "arrow_heading_down": "โคต๏ธ", + "arrows_clockwise": "๐", + "arrows_counterclockwise": "๐", + "back": "๐", + "end": "๐", + "on": "๐", + "soon": "๐", + "top": "๐", + "place_of_worship": "๐", + "atom_symbol": "โ๏ธ", + "om": "๐๏ธ", + "star_of_david": "โก๏ธ", + "wheel_of_dharma": "โธ๏ธ", + "yin_yang": "โฏ๏ธ", + "latin_cross": "โ๏ธ", + "orthodox_cross": "โฆ๏ธ", + "star_and_crescent": "โช๏ธ", + "peace_symbol": "โฎ๏ธ", + "menorah": "๐", + "six_pointed_star": "๐ฏ", + "khanda": "๐ชฏ", + "aries": "โ", + "taurus": "โ", + "gemini": "โ", + "cancer": "โ", + "leo": "โ", + "virgo": "โ", + "libra": "โ", + "scorpius": "โ", + "sagittarius": "โ", + "capricorn": "โ", + "aquarius": "โ", + "pisces": "โ", + "ophiuchus": "โ", + "twisted_rightwards_arrows": "๐", + "repeat": "๐", + "repeat_one": "๐", + "arrow_forward": "โถ๏ธ", + "fast_forward": "โฉ", + "next_track_button": "โญ๏ธ", + "play_or_pause_button": "โฏ๏ธ", + "arrow_backward": "โ๏ธ", + "rewind": "โช", + "previous_track_button": "โฎ๏ธ", + "arrow_up_small": "๐ผ", + "arrow_double_up": "โซ", + "arrow_down_small": "๐ฝ", + "arrow_double_down": "โฌ", + "pause_button": "โธ๏ธ", + "stop_button": "โน๏ธ", + "record_button": "โบ๏ธ", + "eject_button": "โ๏ธ", + "cinema": "๐ฆ", + "low_brightness": "๐ ", + "high_brightness": "๐", + "signal_strength": "๐ถ", + "wireless": "๐", + "vibration_mode": "๐ณ", + "mobile_phone_off": "๐ด", + "female_sign": "โ๏ธ", + "male_sign": "โ๏ธ", + "transgender_symbol": "โง๏ธ", + "heavy_multiplication_x": "โ๏ธ", + "heavy_plus_sign": "โ", + "heavy_minus_sign": "โ", + "heavy_division_sign": "โ", + "heavy_equals_sign": "๐ฐ", + "infinity": "โพ๏ธ", + "bangbang": "โผ๏ธ", + "interrobang": "โ๏ธ", + "question": "โ", + "grey_question": "โ", + "grey_exclamation": "โ", + "exclamation": "โ", + "heavy_exclamation_mark": "โ", + "wavy_dash": "ใฐ๏ธ", + "currency_exchange": "๐ฑ", + "heavy_dollar_sign": "๐ฒ", + "medical_symbol": "โ๏ธ", + "recycle": "โป๏ธ", + "fleur_de_lis": "โ๏ธ", + "trident": "๐ฑ", + "name_badge": "๐", + "beginner": "๐ฐ", + "o": "โญ", + "white_check_mark": "โ ", + "ballot_box_with_check": "โ๏ธ", + "heavy_check_mark": "โ๏ธ", + "x": "โ", + "negative_squared_cross_mark": "โ", + "curly_loop": "โฐ", + "loop": "โฟ", + "part_alternation_mark": "ใฝ๏ธ", + "eight_spoked_asterisk": "โณ๏ธ", + "eight_pointed_black_star": "โด๏ธ", + "sparkle": "โ๏ธ", + "copyright": "ยฉ๏ธ", + "registered": "ยฎ๏ธ", + "tm": "โข๏ธ", + "hash": "#๏ธโฃ", + "asterisk": "*๏ธโฃ", + "zero": "0๏ธโฃ", + "one": "1๏ธโฃ", + "two": "2๏ธโฃ", + "three": "3๏ธโฃ", + "four": "4๏ธโฃ", + "five": "5๏ธโฃ", + "six": "6๏ธโฃ", + "seven": "7๏ธโฃ", + "eight": "8๏ธโฃ", + "nine": "9๏ธโฃ", + "keycap_ten": "๐", + "capital_abcd": "๐ ", + "abcd": "๐ก", + "symbols": "๐ฃ", + "abc": "๐ค", + "a": "๐ ฐ๏ธ", + "ab": "๐", + "b": "๐ ฑ๏ธ", + "cl": "๐", + "cool": "๐", + "free": "๐", + "information_source": "โน๏ธ", + "id": "๐", + "m": "โ๏ธ", + "new": "๐", + "ng": "๐", + "o2": "๐ พ๏ธ", + "ok": "๐", + "parking": "๐ ฟ๏ธ", + "sos": "๐", + "up": "๐", + "vs": "๐", + "koko": "๐", + "sa": "๐๏ธ", + "u6708": "๐ท๏ธ", + "u6709": "๐ถ", + "u6307": "๐ฏ", + "ideograph_advantage": "๐", + "u5272": "๐น", + "u7121": "๐", + "u7981": "๐ฒ", + "accept": "๐", + "u7533": "๐ธ", + "u5408": "๐ด", + "u7a7a": "๐ณ", + "congratulations": "ใ๏ธ", + "secret": "ใ๏ธ", + "u55b6": "๐บ", + "u6e80": "๐ต", + "red_circle": "๐ด", + "orange_circle": "๐ ", + "yellow_circle": "๐ก", + "green_circle": "๐ข", + "large_blue_circle": "๐ต", + "purple_circle": "๐ฃ", + "brown_circle": "๐ค", + "black_circle": "โซ", + "white_circle": "โช", + "red_square": "๐ฅ", + "orange_square": "๐ง", + "yellow_square": "๐จ", + "green_square": "๐ฉ", + "blue_square": "๐ฆ", + "purple_square": "๐ช", + "brown_square": "๐ซ", + "black_large_square": "โฌ", + "white_large_square": "โฌ", + "black_medium_square": "โผ๏ธ", + "white_medium_square": "โป๏ธ", + "black_medium_small_square": "โพ", + "white_medium_small_square": "โฝ", + "black_small_square": "โช๏ธ", + "white_small_square": "โซ๏ธ", + "large_orange_diamond": "๐ถ", + "large_blue_diamond": "๐ท", + "small_orange_diamond": "๐ธ", + "small_blue_diamond": "๐น", + "small_red_triangle": "๐บ", + "small_red_triangle_down": "๐ป", + "diamond_shape_with_a_dot_inside": "๐ ", + "radio_button": "๐", + "white_square_button": "๐ณ", + "black_square_button": "๐ฒ", + "checkered_flag": "๐", + "triangular_flag_on_post": "๐ฉ", + "crossed_flags": "๐", + "black_flag": "๐ด", + "white_flag": "๐ณ๏ธ", + "rainbow_flag": "๐ณ๏ธโ๐", + "transgender_flag": "๐ณ๏ธโโง๏ธ", + "pirate_flag": "๐ดโโ ๏ธ", + "ascension_island": "๐ฆ๐จ", + "andorra": "๐ฆ๐ฉ", + "united_arab_emirates": "๐ฆ๐ช", + "afghanistan": "๐ฆ๐ซ", + "antigua_barbuda": "๐ฆ๐ฌ", + "anguilla": "๐ฆ๐ฎ", + "albania": "๐ฆ๐ฑ", + "armenia": "๐ฆ๐ฒ", + "angola": "๐ฆ๐ด", + "antarctica": "๐ฆ๐ถ", + "argentina": "๐ฆ๐ท", + "american_samoa": "๐ฆ๐ธ", + "austria": "๐ฆ๐น", + "australia": "๐ฆ๐บ", + "aruba": "๐ฆ๐ผ", + "aland_islands": "๐ฆ๐ฝ", + "azerbaijan": "๐ฆ๐ฟ", + "bosnia_herzegovina": "๐ง๐ฆ", + "barbados": "๐ง๐ง", + "bangladesh": "๐ง๐ฉ", + "belgium": "๐ง๐ช", + "burkina_faso": "๐ง๐ซ", + "bulgaria": "๐ง๐ฌ", + "bahrain": "๐ง๐ญ", + "burundi": "๐ง๐ฎ", + "benin": "๐ง๐ฏ", + "st_barthelemy": "๐ง๐ฑ", + "bermuda": "๐ง๐ฒ", + "brunei": "๐ง๐ณ", + "bolivia": "๐ง๐ด", + "caribbean_netherlands": "๐ง๐ถ", + "brazil": "๐ง๐ท", + "bahamas": "๐ง๐ธ", + "bhutan": "๐ง๐น", + "bouvet_island": "๐ง๐ป", + "botswana": "๐ง๐ผ", + "belarus": "โฌ๏ธ๐ฅโฌ", + "belize": "๐ง๐ฟ", + "canada": "๐จ๐ฆ", + "cocos_islands": "๐จ๐จ", + "congo_kinshasa": "๐จ๐ฉ", + "central_african_republic": "๐จ๐ซ", + "congo_brazzaville": "๐จ๐ฌ", + "switzerland": "๐จ๐ญ", + "cote_divoire": "๐จ๐ฎ", + "cook_islands": "๐จ๐ฐ", + "chile": "๐จ๐ฑ", + "cameroon": "๐จ๐ฒ", + "cn": "๐จ๐ณ", + "colombia": "๐จ๐ด", + "clipperton_island": "๐จ๐ต", + "costa_rica": "๐จ๐ท", + "cuba": "๐จ๐บ", + "cape_verde": "๐จ๐ป", + "curacao": "๐จ๐ผ", + "christmas_island": "๐จ๐ฝ", + "cyprus": "๐จ๐พ", + "czech_republic": "๐จ๐ฟ", + "de": "๐ฉ๐ช", + "diego_garcia": "๐ฉ๐ฌ", + "djibouti": "๐ฉ๐ฏ", + "denmark": "๐ฉ๐ฐ", + "dominica": "๐ฉ๐ฒ", + "dominican_republic": "๐ฉ๐ด", + "algeria": "๐ฉ๐ฟ", + "ceuta_melilla": "๐ช๐ฆ", + "ecuador": "๐ช๐จ", + "estonia": "๐ช๐ช", + "egypt": "๐ช๐ฌ", + "western_sahara": "๐ช๐ญ", + "eritrea": "๐ช๐ท", + "es": "๐ช๐ธ", + "ethiopia": "๐ช๐น", + "eu": "๐ช๐บ", + "european_union": "๐ช๐บ", + "finland": "๐ซ๐ฎ", + "fiji": "๐ซ๐ฏ", + "falkland_islands": "๐ซ๐ฐ", + "micronesia": "๐ซ๐ฒ", + "faroe_islands": "๐ซ๐ด", + "fr": "๐ซ๐ท", + "gabon": "๐ฌ๐ฆ", + "gb": "๐ฌ๐ง", + "uk": "๐ฌ๐ง", + "grenada": "๐ฌ๐ฉ", + "georgia": "๐ฌ๐ช", + "french_guiana": "๐ฌ๐ซ", + "guernsey": "๐ฌ๐ฌ", + "ghana": "๐ฌ๐ญ", + "gibraltar": "๐ฌ๐ฎ", + "greenland": "๐ฌ๐ฑ", + "gambia": "๐ฌ๐ฒ", + "guinea": "๐ฌ๐ณ", + "guadeloupe": "๐ฌ๐ต", + "equatorial_guinea": "๐ฌ๐ถ", + "greece": "๐ฌ๐ท", + "south_georgia_south_sandwich_islands": "๐ฌ๐ธ", + "guatemala": "๐ฌ๐น", + "guam": "๐ฌ๐บ", + "guinea_bissau": "๐ฌ๐ผ", + "guyana": "๐ฌ๐พ", + "hong_kong": "๐ญ๐ฐ", + "heard_mcdonald_islands": "๐ญ๐ฒ", + "honduras": "๐ญ๐ณ", + "croatia": "๐ญ๐ท", + "haiti": "๐ญ๐น", + "hungary": "๐ญ๐บ", + "canary_islands": "๐ฎ๐จ", + "indonesia": "๐ฎ๐ฉ", + "ireland": "๐ฎ๐ช", + "israel": "๐ฎ๐ฑ", + "isle_of_man": "๐ฎ๐ฒ", + "india": "๐ฎ๐ณ", + "british_indian_ocean_territory": "๐ฎ๐ด", + "iraq": "๐ฎ๐ถ", + "iran": "๐ฎ๐ท", + "iceland": "๐ฎ๐ธ", + "it": "๐ฎ๐น", + "jersey": "๐ฏ๐ช", + "jamaica": "๐ฏ๐ฒ", + "jordan": "๐ฏ๐ด", + "jp": "๐ฏ๐ต", + "kenya": "๐ฐ๐ช", + "kyrgyzstan": "๐ฐ๐ฌ", + "cambodia": "๐ฐ๐ญ", + "kiribati": "๐ฐ๐ฎ", + "comoros": "๐ฐ๐ฒ", + "st_kitts_nevis": "๐ฐ๐ณ", + "north_korea": "๐ฐ๐ต", + "kr": "๐ฐ๐ท", + "kuwait": "๐ฐ๐ผ", + "cayman_islands": "๐ฐ๐พ", + "kazakhstan": "๐ฐ๐ฟ", + "laos": "๐ฑ๐ฆ", + "lebanon": "๐ฑ๐ง", + "st_lucia": "๐ฑ๐จ", + "liechtenstein": "๐ฑ๐ฎ", + "sri_lanka": "๐ฑ๐ฐ", + "liberia": "๐ฑ๐ท", + "lesotho": "๐ฑ๐ธ", + "lithuania": "๐ฑ๐น", + "luxembourg": "๐ฑ๐บ", + "latvia": "๐ฑ๐ป", + "libya": "๐ฑ๐พ", + "morocco": "๐ฒ๐ฆ", + "monaco": "๐ฒ๐จ", + "moldova": "๐ฒ๐ฉ", + "montenegro": "๐ฒ๐ช", + "st_martin": "๐ฒ๐ซ", + "madagascar": "๐ฒ๐ฌ", + "marshall_islands": "๐ฒ๐ญ", + "macedonia": "๐ฒ๐ฐ", + "mali": "๐ฒ๐ฑ", + "myanmar": "๐ฒ๐ฒ", + "mongolia": "๐ฒ๐ณ", + "macau": "๐ฒ๐ด", + "northern_mariana_islands": "๐ฒ๐ต", + "martinique": "๐ฒ๐ถ", + "mauritania": "๐ฒ๐ท", + "montserrat": "๐ฒ๐ธ", + "malta": "๐ฒ๐น", + "mauritius": "๐ฒ๐บ", + "maldives": "๐ฒ๐ป", + "malawi": "๐ฒ๐ผ", + "mexico": "๐ฒ๐ฝ", + "malaysia": "๐ฒ๐พ", + "mozambique": "๐ฒ๐ฟ", + "namibia": "๐ณ๐ฆ", + "new_caledonia": "๐ณ๐จ", + "niger": "๐ณ๐ช", + "norfolk_island": "๐ณ๐ซ", + "nigeria": "๐ณ๐ฌ", + "nicaragua": "๐ณ๐ฎ", + "netherlands": "๐ณ๐ฑ", + "norway": "๐ณ๐ด", + "nepal": "๐ณ๐ต", + "nauru": "๐ณ๐ท", + "niue": "๐ณ๐บ", + "new_zealand": "๐ณ๐ฟ", + "oman": "๐ด๐ฒ", + "panama": "๐ต๐ฆ", + "peru": "๐ต๐ช", + "french_polynesia": "๐ต๐ซ", + "papua_new_guinea": "๐ต๐ฌ", + "philippines": "๐ต๐ญ", + "pakistan": "๐ต๐ฐ", + "poland": "๐ต๐ฑ", + "st_pierre_miquelon": "๐ต๐ฒ", + "pitcairn_islands": "๐ต๐ณ", + "puerto_rico": "๐ต๐ท", + "palestinian_territories": "๐ต๐ธ", + "portugal": "๐ต๐น", + "palau": "๐ต๐ผ", + "paraguay": "๐ต๐พ", + "qatar": "๐ถ๐ฆ", + "reunion": "๐ท๐ช", + "romania": "๐ท๐ด", + "serbia": "๐ท๐ธ", + "ru": "๐ท๐บ", + "rwanda": "๐ท๐ผ", + "saudi_arabia": "๐ธ๐ฆ", + "solomon_islands": "๐ธ๐ง", + "seychelles": "๐ธ๐จ", + "sudan": "๐ธ๐ฉ", + "sweden": "๐ธ๐ช", + "singapore": "๐ธ๐ฌ", + "st_helena": "๐ธ๐ญ", + "slovenia": "๐ธ๐ฎ", + "svalbard_jan_mayen": "๐ธ๐ฏ", + "slovakia": "๐ธ๐ฐ", + "sierra_leone": "๐ธ๐ฑ", + "san_marino": "๐ธ๐ฒ", + "senegal": "๐ธ๐ณ", + "somalia": "๐ธ๐ด", + "suriname": "๐ธ๐ท", + "south_sudan": "๐ธ๐ธ", + "sao_tome_principe": "๐ธ๐น", + "el_salvador": "๐ธ๐ป", + "sint_maarten": "๐ธ๐ฝ", + "syria": "๐ธ๐พ", + "swaziland": "๐ธ๐ฟ", + "tristan_da_cunha": "๐น๐ฆ", + "turks_caicos_islands": "๐น๐จ", + "chad": "๐น๐ฉ", + "french_southern_territories": "๐น๐ซ", + "togo": "๐น๐ฌ", + "thailand": "๐น๐ญ", + "tajikistan": "๐น๐ฏ", + "tokelau": "๐น๐ฐ", + "timor_leste": "๐น๐ฑ", + "turkmenistan": "๐น๐ฒ", + "tunisia": "๐น๐ณ", + "tonga": "๐น๐ด", + "tr": "๐น๐ท", + "trinidad_tobago": "๐น๐น", + "tuvalu": "๐น๐ป", + "taiwan": "๐น๐ผ", + "tanzania": "๐น๐ฟ", + "ukraine": "๐บ๐ฆ", + "uganda": "๐บ๐ฌ", + "us_outlying_islands": "๐บ๐ฒ", + "united_nations": "๐บ๐ณ", + "us": "๐บ๐ธ", + "uruguay": "๐บ๐พ", + "uzbekistan": "๐บ๐ฟ", + "vatican_city": "๐ป๐ฆ", + "st_vincent_grenadines": "๐ป๐จ", + "venezuela": "๐ป๐ช", + "british_virgin_islands": "๐ป๐ฌ", + "us_virgin_islands": "๐ป๐ฎ", + "vietnam": "๐ป๐ณ", + "vanuatu": "๐ป๐บ", + "wallis_futuna": "๐ผ๐ซ", + "samoa": "๐ผ๐ธ", + "kosovo": "๐ฝ๐ฐ", + "yemen": "๐พ๐ช", + "mayotte": "๐พ๐น", + "south_africa": "๐ฟ๐ฆ", + "zambia": "๐ฟ๐ฒ", + "zimbabwe": "๐ฟ๐ผ", + "england": "๐ด๓ ง๓ ข๓ ฅ๓ ฎ๓ ง๓ ฟ", + "scotland": "๐ด๓ ง๓ ข๓ ณ๓ ฃ๓ ด๓ ฟ", + "wales": "๐ด๓ ง๓ ข๓ ท๓ ฌ๓ ณ๓ ฟ" +}; diff --git a/app/vmui/packages/vmui/src/constants/markedPlugins.ts b/app/vmui/packages/vmui/src/constants/markedPlugins.ts new file mode 100644 index 0000000000..2a7d046c7c --- /dev/null +++ b/app/vmui/packages/vmui/src/constants/markedPlugins.ts @@ -0,0 +1,5 @@ +import { markedEmoji } from "marked-emoji"; +import { marked } from "marked"; +import emojis from "./emojis"; + +marked.use(markedEmoji({ emojis, renderer: (token) => token.emoji })); diff --git a/app/vmui/packages/vmui/src/hooks/useGetMetricsQL.tsx b/app/vmui/packages/vmui/src/hooks/useGetMetricsQL.tsx index fb930c4baf..c41e4be92b 100644 --- a/app/vmui/packages/vmui/src/hooks/useGetMetricsQL.tsx +++ b/app/vmui/packages/vmui/src/hooks/useGetMetricsQL.tsx @@ -54,7 +54,7 @@ const useGetMetricsQL = () => { const processMarkdown = (text: string) => { const div = document.createElement("div"); - div.innerHTML = marked(text); + div.innerHTML = marked(text) as string; const groups = div.querySelectorAll(`${CATEGORY_TAG}, ${FUNCTION_TAG}`); return processGroups(groups); }; diff --git a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx index 8696531e45..9dbebedb6b 100644 --- a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx +++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx @@ -26,6 +26,7 @@ const ExploreLogs: FC = () => { const { logs, isLoading, error, fetchLogs } = useFetchLogs(serverUrl, query, limit); const [queryError, setQueryError] = useState<ErrorTypes | string>(""); const [loaded, isLoaded] = useState(false); + const [markdownParsing, setMarkdownParsing] = useState(getFromStorage("LOGS_MARKDOWN") === "true"); const handleRunQuery = () => { if (!query) { @@ -51,6 +52,11 @@ const ExploreLogs: FC = () => { saveToStorage("LOGS_LIMIT", `${limit}`); }; + const handleChangeMarkdownParsing = (val: boolean) => { + saveToStorage("LOGS_MARKDOWN", `${val}`); + setMarkdownParsing(val); + }; + useEffect(() => { if (query) handleRunQuery(); }, [period]); @@ -65,15 +71,18 @@ const ExploreLogs: FC = () => { query={query} error={queryError} limit={limit} + markdownParsing={markdownParsing} onChange={setQuery} onChangeLimit={handleChangeLimit} onRun={handleRunQuery} + onChangeMarkdownParsing={handleChangeMarkdownParsing} /> {isLoading && <Spinner />} {error && <Alert variant="error">{error}</Alert>} <ExploreLogsBody data={logs} loaded={loaded} + markdownParsing={markdownParsing} /> </div> ); diff --git a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx index 7538a0e148..3a66a35039 100644 --- a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx +++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx @@ -15,10 +15,12 @@ import useBoolean from "../../../hooks/useBoolean"; import TableLogs from "./TableLogs"; import GroupLogs from "../GroupLogs/GroupLogs"; import { DATE_TIME_FORMAT } from "../../../constants/date"; +import { marked } from "marked"; export interface ExploreLogBodyProps { data: Logs[]; loaded?: boolean; + markdownParsing: boolean; } enum DisplayType { @@ -33,7 +35,7 @@ const tabs = [ { label: "JSON", value: DisplayType.json, icon: <CodeIcon/> }, ]; -const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, loaded }) => { +const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, loaded, markdownParsing }) => { const { isMobile } = useDeviceDetect(); const { timezone } = useTimeState(); const { setSearchParamsFromKeys } = useSearchParamsFromObject(); @@ -46,11 +48,12 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, loaded }) => { ...item, _vmui_time: item._time ? dayjs(item._time).tz().format(`${DATE_TIME_FORMAT}.SSS`) : "", _vmui_data: JSON.stringify(item, null, 2), + _vmui_markdown: marked(item._msg.replace(/```/g, "\n```\n")) as string, })) as Logs[], [data, timezone]); const columns = useMemo(() => { if (!logs?.length) return []; - const hideColumns = ["_vmui_data", "_vmui_time"]; + const hideColumns = ["_vmui_data", "_vmui_time", "_vmui_markdown"]; const keys = new Set<string>(); for (const item of logs) { for (const key in item) { @@ -125,6 +128,7 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, loaded }) => { <GroupLogs logs={logs} columns={columns} + markdownParsing={markdownParsing} /> )} {activeTab === DisplayType.json && ( diff --git a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/ExploreLogsHeader.tsx b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/ExploreLogsHeader.tsx index 5a7c54973c..aed3af35b5 100644 --- a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/ExploreLogsHeader.tsx +++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/ExploreLogsHeader.tsx @@ -6,17 +6,29 @@ import useDeviceDetect from "../../../hooks/useDeviceDetect"; import Button from "../../../components/Main/Button/Button"; import QueryEditor from "../../../components/Configurators/QueryEditor/QueryEditor"; import TextField from "../../../components/Main/TextField/TextField"; +import Switch from "../../../components/Main/Switch/Switch"; export interface ExploreLogHeaderProps { query: string; limit: number; error?: string; + markdownParsing: boolean; onChange: (val: string) => void; onChangeLimit: (val: number) => void; onRun: () => void; + onChangeMarkdownParsing: (val: boolean) => void; } -const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({ query, limit, error, onChange, onChangeLimit, onRun }) => { +const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({ + query, + limit, + error, + markdownParsing, + onChange, + onChangeLimit, + onRun, + onChangeMarkdownParsing, +}) => { const { isMobile } = useDeviceDetect(); const [errorLimit, setErrorLimit] = useState(""); @@ -66,6 +78,14 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({ query, limit, error, onC /> </div> <div className="vm-explore-logs-header-bottom"> + <div className="vm-explore-logs-header-bottom-contols"> + <Switch + label={"Markdown parsing"} + value={markdownParsing} + onChange={onChangeMarkdownParsing} + fullWidth={isMobile} + /> + </div> <div className="vm-explore-logs-header-bottom-helpful"> <a className="vm-link vm-link_with-icon" diff --git a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/style.scss b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/style.scss index 49a439863c..1bc7ab0b69 100644 --- a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/style.scss +++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/style.scss @@ -25,6 +25,10 @@ justify-content: normal; } + &-contols { + flex-grow: 1; + } + &__execute { display: grid; } diff --git a/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx b/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx index 739d0c0331..82983a14fc 100644 --- a/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx +++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx @@ -11,9 +11,10 @@ import GroupLogsItem from "./GroupLogsItem"; interface TableLogsProps { logs: Logs[]; columns: string[]; + markdownParsing: boolean; } -const GroupLogs: FC<TableLogsProps> = ({ logs }) => { +const GroupLogs: FC<TableLogsProps> = ({ logs, markdownParsing }) => { const copyToClipboard = useCopyToClipboard(); const [copied, setCopied] = useState<string | null>(null); @@ -77,6 +78,7 @@ const GroupLogs: FC<TableLogsProps> = ({ logs }) => { <GroupLogsItem key={`${value._msg}${value._time}`} log={value} + markdownParsing={markdownParsing} /> ))} </div> diff --git a/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx b/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx index 7208de9328..64539db0e1 100644 --- a/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx +++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx @@ -10,15 +10,16 @@ import classNames from "classnames"; interface Props { log: Logs; + markdownParsing: boolean; } -const GroupLogsItem: FC<Props> = ({ log }) => { +const GroupLogsItem: FC<Props> = ({ log, markdownParsing }) => { const { value: isOpenFields, toggle: toggleOpenFields, } = useBoolean(false); - const excludeKeys = ["_stream", "_msg", "_time", "_vmui_time", "_vmui_data"]; + const excludeKeys = ["_stream", "_msg", "_time", "_vmui_time", "_vmui_data", "_vmui_markdown"]; const fields = Object.entries(log).filter(([key]) => !excludeKeys.includes(key)); const hasFields = fields.length > 0; @@ -71,6 +72,7 @@ const GroupLogsItem: FC<Props> = ({ log }) => { "vm-group-logs-row-content__msg": true, "vm-group-logs-row-content__msg_missing": !log._msg })} + dangerouslySetInnerHTML={markdownParsing && log._vmui_markdown ? { __html: log._vmui_markdown } : undefined} > {log._msg || "message missing"} </div> diff --git a/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss b/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss index 0710a3f8e9..71ee06179c 100644 --- a/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss +++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss @@ -96,6 +96,65 @@ font-style: italic; text-align: center; } + + /* styles for markdown */ + p, pre, code { + white-space: pre-wrap; + word-wrap: break-word; + word-break: normal; + } + + code:not(pre code), pre { + display: inline-block; + background: $color-hover-black; + border: 1px solid $color-hover-black; + border-radius: $border-radius-small; + tab-size: 4; + font-variant-ligatures: none; + margin: calc($padding-small/4) 0; + } + + p { + font-family: $font-family-global; + line-height: 1.4; + } + + pre { + padding: $padding-small; + } + + code { + font-size: $font-size-small; + padding: calc($padding-small / 4) calc($padding-small / 2); + } + + a { + color: $color-primary; + cursor: pointer; + + &:hover { + text-decoration: underline; + } + } + + strong { + font-weight: bold; + } + + em { + font-style: italic; + } + + blockquote { + border-left: 4px solid $color-hover-black; + margin: calc($padding-small/2) $padding-small; + padding: calc($padding-small/2) $padding-small; + } + + ul, ol { + list-style-position: inside; + } + /* end styles for markdown */ } } diff --git a/app/vmui/packages/vmui/src/pages/PredefinedPanels/PredefinedPanel/PredefinedPanel.tsx b/app/vmui/packages/vmui/src/pages/PredefinedPanels/PredefinedPanel/PredefinedPanel.tsx index 268d51a934..3c2dce8e11 100644 --- a/app/vmui/packages/vmui/src/pages/PredefinedPanels/PredefinedPanel/PredefinedPanel.tsx +++ b/app/vmui/packages/vmui/src/pages/PredefinedPanels/PredefinedPanel/PredefinedPanel.tsx @@ -89,7 +89,7 @@ const PredefinedPanel: FC<PredefinedPanelsProps> = ({ <> <div> <span>Description:</span> - <div dangerouslySetInnerHTML={{ __html: marked.parse(description) }}/> + <div dangerouslySetInnerHTML={{ __html: marked(description) as string }}/> </div> <hr/> </> diff --git a/app/vmui/packages/vmui/src/utils/storage.ts b/app/vmui/packages/vmui/src/utils/storage.ts index 995cf77725..7c5ee7109f 100644 --- a/app/vmui/packages/vmui/src/utils/storage.ts +++ b/app/vmui/packages/vmui/src/utils/storage.ts @@ -7,6 +7,7 @@ export type StorageKeys = "AUTOCOMPLETE" | "DISABLED_DEFAULT_TIMEZONE" | "THEME" | "LOGS_LIMIT" + | "LOGS_MARKDOWN" | "EXPLORE_METRICS_TIPS" | "QUERY_HISTORY" | "QUERY_FAVORITES" diff --git a/dashboards/vm/vmagent.json b/dashboards/vm/vmagent.json index d9e5fdc982..c78ef1cabe 100644 --- a/dashboards/vm/vmagent.json +++ b/dashboards/vm/vmagent.json @@ -86,7 +86,7 @@ } ] }, - "description": "Overview for VictoriaMetrics vmagent v1.80.0 or higher", + "description": "Overview for VictoriaMetrics vmagent v1.102.0 or higher", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -1336,7 +1336,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1344,8 +1345,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1440,7 +1440,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1448,8 +1449,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1627,8 +1627,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1636,10 +1635,9 @@ "h": 8, "w": 12, "x": 0, - "y": 3 + "y": 11 }, "id": 109, - "links": [], "options": { "legend": { "calcs": [ @@ -1743,8 +1741,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1752,10 +1749,9 @@ "h": 8, "w": 12, "x": 12, - "y": 3 + "y": 11 }, "id": 111, - "links": [], "options": { "legend": { "calcs": [ @@ -1856,8 +1852,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [ { @@ -1878,10 +1873,9 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 19 }, "id": 81, - "links": [], "options": { "legend": { "calcs": [ @@ -1993,8 +1987,7 @@ } ] }, - "unit": "bps", - "unitScale": true + "unit": "bps" }, "overrides": [ { @@ -2015,7 +2008,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 19 }, "id": 7, "options": { @@ -2125,8 +2118,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2134,10 +2126,9 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 27 }, "id": 83, - "links": [], "options": { "legend": { "calcs": [ @@ -2232,8 +2223,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2241,10 +2231,9 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 27 }, "id": 39, - "links": [], "options": { "legend": { "calcs": [ @@ -2340,8 +2329,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2349,10 +2337,9 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 35 }, "id": 135, - "links": [], "options": { "legend": { "calcs": [ @@ -2447,8 +2434,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2456,10 +2442,9 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 35 }, "id": 41, - "links": [], "options": { "legend": { "calcs": [ @@ -2534,6 +2519,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2547,6 +2533,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2584,7 +2571,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 12 }, "id": 92, "options": { @@ -2636,6 +2623,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2649,6 +2637,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2686,7 +2675,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 12 }, "id": 95, "options": { @@ -2738,6 +2727,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2752,6 +2742,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2791,7 +2782,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 20 }, "id": 98, "options": { @@ -2843,6 +2834,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2857,6 +2849,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2896,7 +2889,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 20 }, "id": 99, "options": { @@ -3000,10 +2993,9 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 28 }, "id": 79, - "links": [], "options": { "legend": { "calcs": [ @@ -3105,7 +3097,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 28 }, "id": 18, "links": [ @@ -3215,10 +3207,9 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 36 }, "id": 127, - "links": [], "options": { "legend": { "calcs": [ @@ -3318,7 +3309,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 36 }, "id": 50, "options": { @@ -3418,7 +3409,7 @@ "h": 7, "w": 24, "x": 0, - "y": 36 + "y": 44 }, "id": 129, "options": { @@ -3565,8 +3556,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3574,7 +3564,7 @@ "h": 7, "w": 12, "x": 0, - "y": 37 + "y": 13 }, "id": 48, "options": { @@ -3672,8 +3662,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3681,7 +3670,7 @@ "h": 7, "w": 12, "x": 12, - "y": 37 + "y": 13 }, "id": 76, "options": { @@ -3777,8 +3766,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3786,7 +3774,7 @@ "h": 7, "w": 12, "x": 0, - "y": 44 + "y": 20 }, "id": 132, "options": { @@ -3884,8 +3872,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3893,7 +3880,7 @@ "h": 7, "w": 12, "x": 12, - "y": 44 + "y": 20 }, "id": 133, "options": { @@ -3990,8 +3977,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3999,7 +3985,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 27 }, "id": 20, "options": { @@ -4095,8 +4081,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4104,7 +4089,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 27 }, "id": 126, "options": { @@ -4199,8 +4184,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [] }, @@ -4208,7 +4192,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 35 }, "id": 46, "options": { @@ -4303,8 +4287,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4312,7 +4295,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 35 }, "id": 31, "options": { @@ -4437,6 +4420,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4450,6 +4434,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4489,10 +4474,9 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 14 }, "id": 73, - "links": [], "options": { "legend": { "calcs": [ @@ -4553,6 +4537,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4566,6 +4551,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4605,10 +4591,9 @@ "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 14 }, "id": 131, - "links": [], "options": { "legend": { "calcs": [ @@ -4656,6 +4641,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4669,6 +4655,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4708,10 +4695,9 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 22 }, "id": 130, - "links": [], "options": { "legend": { "calcs": [ @@ -4772,6 +4758,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4785,6 +4772,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4824,10 +4812,9 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 22 }, "id": 77, - "links": [], "options": { "legend": { "calcs": [ @@ -4885,6 +4872,620 @@ "title": "Ingestion", "type": "row" }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 136, + "panels": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 137, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Dedup flush duration (0.99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Amount of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_ignored_samples_total[5m]) > 0", + "instant": false, + "legendFormat": "{{reason}}: {{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Ignored samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": -5, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "instant": false, + "legendFormat": "aggregate: {{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "dedup: {{instance}} ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Flush timeouts", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the lag between average samples timestamp and aggregation interval.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign of data delay before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Aggregated samples lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in bytes.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 140, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor items count", + "type": "timeseries" + } + ], + "title": "Streaming aggregation", + "type": "row" + }, { "collapsed": true, "datasource": { @@ -4895,7 +5496,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 39 }, "id": 58, "panels": [ @@ -4911,6 +5512,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4924,6 +5526,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4964,7 +5567,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 16 }, "id": 60, "options": { @@ -5014,6 +5617,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5027,6 +5631,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5066,7 +5671,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 16 }, "id": 66, "options": { @@ -5116,6 +5721,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5129,6 +5735,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5168,7 +5775,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 24 }, "id": 61, "options": { @@ -5218,6 +5825,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5231,6 +5839,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5270,7 +5879,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 24 }, "id": 65, "options": { @@ -5371,7 +5980,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 32 }, "id": 88, "options": { @@ -5469,7 +6078,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 32 }, "id": 84, "options": { @@ -5570,7 +6179,7 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 40 }, "id": 90, "options": { @@ -5623,7 +6232,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 40 }, "id": 113, "panels": [ @@ -5636,7 +6245,7 @@ "h": 2, "w": 24, "x": 0, - "y": 87 + "y": 17 }, "id": 115, "options": { @@ -5648,7 +6257,7 @@ "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", "mode": "markdown" }, - "pluginVersion": "9.2.6", + "pluginVersion": "10.4.2", "transparent": true, "type": "text" }, @@ -5664,6 +6273,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5677,6 +6287,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5712,10 +6323,9 @@ "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 19 }, "id": 119, - "links": [], "options": { "legend": { "calcs": [ @@ -5767,6 +6377,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5780,6 +6391,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5815,10 +6427,9 @@ "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 19 }, "id": 117, - "links": [], "options": { "legend": { "calcs": [ @@ -5868,6 +6479,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5881,6 +6493,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5920,7 +6533,7 @@ "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 27 }, "id": 125, "links": [ @@ -5976,6 +6589,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5989,6 +6603,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6040,7 +6655,7 @@ "h": 8, "w": 12, "x": 12, - "y": 97 + "y": 27 }, "id": 123, "options": { @@ -6104,6 +6719,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -6117,6 +6733,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6168,10 +6785,9 @@ "h": 8, "w": 12, "x": 0, - "y": 105 + "y": 35 }, "id": 121, - "links": [], "options": { "legend": { "calcs": [ diff --git a/dashboards/vmagent.json b/dashboards/vmagent.json index 80c6bddb8b..06e0b0422d 100644 --- a/dashboards/vmagent.json +++ b/dashboards/vmagent.json @@ -85,7 +85,7 @@ } ] }, - "description": "Overview for VictoriaMetrics vmagent v1.80.0 or higher", + "description": "Overview for VictoriaMetrics vmagent v1.102.0 or higher", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -1335,7 +1335,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1343,8 +1344,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1439,7 +1439,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1447,8 +1448,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1626,8 +1626,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1635,10 +1634,9 @@ "h": 8, "w": 12, "x": 0, - "y": 3 + "y": 11 }, "id": 109, - "links": [], "options": { "legend": { "calcs": [ @@ -1742,8 +1740,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1751,10 +1748,9 @@ "h": 8, "w": 12, "x": 12, - "y": 3 + "y": 11 }, "id": 111, - "links": [], "options": { "legend": { "calcs": [ @@ -1855,8 +1851,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [ { @@ -1877,10 +1872,9 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 19 }, "id": 81, - "links": [], "options": { "legend": { "calcs": [ @@ -1992,8 +1986,7 @@ } ] }, - "unit": "bps", - "unitScale": true + "unit": "bps" }, "overrides": [ { @@ -2014,7 +2007,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 19 }, "id": 7, "options": { @@ -2124,8 +2117,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2133,10 +2125,9 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 27 }, "id": 83, - "links": [], "options": { "legend": { "calcs": [ @@ -2231,8 +2222,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2240,10 +2230,9 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 27 }, "id": 39, - "links": [], "options": { "legend": { "calcs": [ @@ -2339,8 +2328,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2348,10 +2336,9 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 35 }, "id": 135, - "links": [], "options": { "legend": { "calcs": [ @@ -2446,8 +2433,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2455,10 +2441,9 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 35 }, "id": 41, - "links": [], "options": { "legend": { "calcs": [ @@ -2533,6 +2518,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2546,6 +2532,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2583,7 +2570,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 12 }, "id": 92, "options": { @@ -2635,6 +2622,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2648,6 +2636,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2685,7 +2674,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 12 }, "id": 95, "options": { @@ -2737,6 +2726,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2751,6 +2741,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2790,7 +2781,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 20 }, "id": 98, "options": { @@ -2842,6 +2833,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2856,6 +2848,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2895,7 +2888,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 20 }, "id": 99, "options": { @@ -2999,10 +2992,9 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 28 }, "id": 79, - "links": [], "options": { "legend": { "calcs": [ @@ -3104,7 +3096,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 28 }, "id": 18, "links": [ @@ -3214,10 +3206,9 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 36 }, "id": 127, - "links": [], "options": { "legend": { "calcs": [ @@ -3317,7 +3308,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 36 }, "id": 50, "options": { @@ -3417,7 +3408,7 @@ "h": 7, "w": 24, "x": 0, - "y": 36 + "y": 44 }, "id": 129, "options": { @@ -3564,8 +3555,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3573,7 +3563,7 @@ "h": 7, "w": 12, "x": 0, - "y": 37 + "y": 13 }, "id": 48, "options": { @@ -3671,8 +3661,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3680,7 +3669,7 @@ "h": 7, "w": 12, "x": 12, - "y": 37 + "y": 13 }, "id": 76, "options": { @@ -3776,8 +3765,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3785,7 +3773,7 @@ "h": 7, "w": 12, "x": 0, - "y": 44 + "y": 20 }, "id": 132, "options": { @@ -3883,8 +3871,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3892,7 +3879,7 @@ "h": 7, "w": 12, "x": 12, - "y": 44 + "y": 20 }, "id": 133, "options": { @@ -3989,8 +3976,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3998,7 +3984,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 27 }, "id": 20, "options": { @@ -4094,8 +4080,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4103,7 +4088,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 27 }, "id": 126, "options": { @@ -4198,8 +4183,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [] }, @@ -4207,7 +4191,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 35 }, "id": 46, "options": { @@ -4302,8 +4286,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4311,7 +4294,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 35 }, "id": 31, "options": { @@ -4436,6 +4419,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4449,6 +4433,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4488,10 +4473,9 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 14 }, "id": 73, - "links": [], "options": { "legend": { "calcs": [ @@ -4552,6 +4536,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4565,6 +4550,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4604,10 +4590,9 @@ "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 14 }, "id": 131, - "links": [], "options": { "legend": { "calcs": [ @@ -4655,6 +4640,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4668,6 +4654,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4707,10 +4694,9 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 22 }, "id": 130, - "links": [], "options": { "legend": { "calcs": [ @@ -4771,6 +4757,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4784,6 +4771,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4823,10 +4811,9 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 22 }, "id": 77, - "links": [], "options": { "legend": { "calcs": [ @@ -4884,6 +4871,620 @@ "title": "Ingestion", "type": "row" }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 136, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 137, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Dedup flush duration (0.99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Amount of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_ignored_samples_total[5m]) > 0", + "instant": false, + "legendFormat": "{{reason}}: {{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Ignored samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": -5, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "instant": false, + "legendFormat": "aggregate: {{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "dedup: {{instance}} ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Flush timeouts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the lag between average samples timestamp and aggregation interval.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign of data delay before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Aggregated samples lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in bytes.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 140, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor items count", + "type": "timeseries" + } + ], + "title": "Streaming aggregation", + "type": "row" + }, { "collapsed": true, "datasource": { @@ -4894,7 +5495,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 39 }, "id": 58, "panels": [ @@ -4910,6 +5511,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4923,6 +5525,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4963,7 +5566,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 16 }, "id": 60, "options": { @@ -5013,6 +5616,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5026,6 +5630,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5065,7 +5670,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 16 }, "id": 66, "options": { @@ -5115,6 +5720,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5128,6 +5734,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5167,7 +5774,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 24 }, "id": 61, "options": { @@ -5217,6 +5824,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5230,6 +5838,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5269,7 +5878,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 24 }, "id": 65, "options": { @@ -5370,7 +5979,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 32 }, "id": 88, "options": { @@ -5468,7 +6077,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 32 }, "id": 84, "options": { @@ -5569,7 +6178,7 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 40 }, "id": 90, "options": { @@ -5622,7 +6231,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 40 }, "id": 113, "panels": [ @@ -5635,7 +6244,7 @@ "h": 2, "w": 24, "x": 0, - "y": 87 + "y": 17 }, "id": 115, "options": { @@ -5647,7 +6256,7 @@ "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", "mode": "markdown" }, - "pluginVersion": "9.2.6", + "pluginVersion": "10.4.2", "transparent": true, "type": "text" }, @@ -5663,6 +6272,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5676,6 +6286,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5711,10 +6322,9 @@ "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 19 }, "id": 119, - "links": [], "options": { "legend": { "calcs": [ @@ -5766,6 +6376,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5779,6 +6390,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5814,10 +6426,9 @@ "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 19 }, "id": 117, - "links": [], "options": { "legend": { "calcs": [ @@ -5867,6 +6478,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5880,6 +6492,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5919,7 +6532,7 @@ "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 27 }, "id": 125, "links": [ @@ -5975,6 +6588,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5988,6 +6602,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6039,7 +6654,7 @@ "h": 8, "w": 12, "x": 12, - "y": 97 + "y": 27 }, "id": 123, "options": { @@ -6103,6 +6718,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -6116,6 +6732,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6167,10 +6784,9 @@ "h": 8, "w": 12, "x": 0, - "y": 105 + "y": 35 }, "id": 121, - "links": [], "options": { "legend": { "calcs": [ diff --git a/deployment/docker/alerts-vmagent.yml b/deployment/docker/alerts-vmagent.yml index fc11b87d08..15d522a7fc 100644 --- a/deployment/docker/alerts-vmagent.yml +++ b/deployment/docker/alerts-vmagent.yml @@ -135,3 +135,23 @@ groups: summary: "Configuration reload failed for vmagent instance {{ $labels.instance }}" description: "Configuration hot-reload failed for vmagent on instance {{ $labels.instance }}. Check vmagent's logs for detailed error message." + + - alert: StreamAggrFlushTimeout + expr: | + increase(vm_streamaggr_flush_timeouts_total[5m]) > 0 + labels: + severity: warning + annotations: + summary: "Streaming aggregation at \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't be finished within the configured aggregation interval." + description: "Stream aggregation process can't keep up with the load and might produce incorrect aggregation results. Check logs for more details. + Possible solutions: increase aggregation interval; aggregate smaller number of series; reduce samples' ingestion rate to stream aggregation." + + - alert: StreamAggrDedupFlushTimeout + expr: | + increase(vm_streamaggr_dedup_flush_timeouts_total[5m]) > 0 + labels: + severity: warning + annotations: + summary: "Deduplication \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't be finished within configured deduplication interval." + description: "Deduplication process can't keep up with the load and might produce incorrect results. Check docs https://docs.victoriametrics.com/stream-aggregation/#deduplication and logs for more details. + Possible solutions: increase deduplication interval; deduplicate smaller number of series; reduce samples' ingestion rate." diff --git a/deployment/docker/alerts-vmalert.yml b/deployment/docker/alerts-vmalert.yml index bae602201b..af7895da33 100644 --- a/deployment/docker/alerts-vmalert.yml +++ b/deployment/docker/alerts-vmalert.yml @@ -40,7 +40,7 @@ groups: Check vmalert's logs for detailed error message." - alert: RecordingRulesNoData - expr: sum(vmalert_recording_rules_last_evaluation_samples) without(recording, id) < 1 + expr: sum(vmalert_recording_rules_last_evaluation_samples) without(id) < 1 for: 30m labels: severity: info diff --git a/deployment/docker/docker-compose-cluster.yml b/deployment/docker/docker-compose-cluster.yml index 6194de2523..1672f516bf 100644 --- a/deployment/docker/docker-compose-cluster.yml +++ b/deployment/docker/docker-compose-cluster.yml @@ -1,4 +1,3 @@ -version: '3.5' services: # Metrics collector. # It scrapes targets defined in --promscrape.config diff --git a/deployment/docker/docker-compose-victorialogs.yml b/deployment/docker/docker-compose-victorialogs.yml index 484a802659..e2e676481a 100644 --- a/deployment/docker/docker-compose-victorialogs.yml +++ b/deployment/docker/docker-compose-victorialogs.yml @@ -1,4 +1,3 @@ -version: "3.5" services: # Grafana instance configured with VictoriaLogs as datasource grafana: diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index e444bd654b..8be7e65017 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.5" services: # Metrics collector. # It scrapes targets defined in --promscrape.config diff --git a/deployment/docker/vm-datasource/docker-compose-cluster.yml b/deployment/docker/vm-datasource/docker-compose-cluster.yml index 948c6d12c5..5c8d806df9 100644 --- a/deployment/docker/vm-datasource/docker-compose-cluster.yml +++ b/deployment/docker/vm-datasource/docker-compose-cluster.yml @@ -1,4 +1,3 @@ -version: "3.5" services: grafana: container_name: grafana diff --git a/deployment/docker/vm-datasource/docker-compose.yml b/deployment/docker/vm-datasource/docker-compose.yml index bfbf25bc6c..994d096307 100644 --- a/deployment/docker/vm-datasource/docker-compose.yml +++ b/deployment/docker/vm-datasource/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.5" services: grafana: container_name: grafana diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f9093c22e7..0f617fc915 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -30,6 +30,16 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). ## tip +**Update note 1: the `--vm-disable-progress-bar` command-line flag at `vmctl` was deprecated. Use `--disable-progress-bar` instead.** + +* FEATURE: [alerts-vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmagent.yml): add new alerting rules `StreamAggrFlushTimeout` and `StreamAggrDedupFlushTimeout` to notify about issues during stream aggregation. +* FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add row `Streaming aggregation` with panels related to [streaming aggregation](https://docs.victoriametrics.com/stream-aggregation/) process. +* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): add `idleConnTimeout` flag set to 50s by default. It should reduce the probability of `broken pipe` or `connection reset by peer` errors in vmauth logs. +* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): add auto request retry for trivial network errors, such as `broken pipe` and `connection reset` for requests to the configured backends. + +* BUGFIX: all VictoriaMetrics components: prioritize `-configAuthKey` and `-reloadAuthKey` over `-httpAuth.*` settings. This change aligns behavior of mentioned flags with other auth flags like `-metricsAuthKey`, `-flagsAuthKey`, `-pprofAuthKey`. Check [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6329). +* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): add `--disable-progress-bar` global command-line flag. It can be used for disabling dynamic progress bar for all migration modes. `--vm-disable-progress-bar` command-line flag is deprecated and will be removed in the future releases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6367). + ## [v1.102.0-rc1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.0-rc1) Released at 2024-06-07 diff --git a/docs/README.md b/docs/README.md index 74470eb62f..799eeea8b7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2765,7 +2765,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -cacheExpireDuration duration Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s) -configAuthKey value - Authorization key for accessing /config page. It must be passed via authKey query arg + Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings. Flag value can be read from the given file when using -configAuthKey=file:///abs/path/to/file or -configAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -configAuthKey=http://host/path or -configAuthKey=https://host/path -csvTrimTimestamp duration Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms) @@ -3079,7 +3079,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -relabelConfig string Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal -reloadAuthKey value - Auth key for /-/reload http endpoint. It must be passed as authKey=... + Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings. Flag value can be read from the given file when using -reloadAuthKey=file:///abs/path/to/file or -reloadAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -reloadAuthKey=http://host/path or -reloadAuthKey=https://host/path -retentionFilter array Retention filter in the format 'filter:retention'. For example, '{env="dev"}:3d' configures the retention for time series with env="dev" label to 3 days. See https://docs.victoriametrics.com/#retention-filters for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise/ diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index c0c3bdec2c..ed4c30afe7 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2773,7 +2773,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -cacheExpireDuration duration Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s) -configAuthKey value - Authorization key for accessing /config page. It must be passed via authKey query arg + Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings. Flag value can be read from the given file when using -configAuthKey=file:///abs/path/to/file or -configAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -configAuthKey=http://host/path or -configAuthKey=https://host/path -csvTrimTimestamp duration Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms) @@ -3087,7 +3087,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -relabelConfig string Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal -reloadAuthKey value - Auth key for /-/reload http endpoint. It must be passed as authKey=... + Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings. Flag value can be read from the given file when using -reloadAuthKey=file:///abs/path/to/file or -reloadAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -reloadAuthKey=http://host/path or -reloadAuthKey=https://host/path -retentionFilter array Retention filter in the format 'filter:retention'. For example, '{env="dev"}:3d' configures the retention for time series with env="dev" label to 3 days. See https://docs.victoriametrics.com/#retention-filters for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise/ diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index 7e248b5065..27686bea0f 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -20,6 +20,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip * FEATURE: disallow unescaped `!` char in [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) queries, since it permits writing incorrect query, which may look like correct one. For example, `foo!:bar` instead of `foo:!bar`. +* FEATURE: [web UI](https://docs.victoriametrics.com/VictoriaLogs/querying/#web-ui): add markdown support to the `Group` view. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6292). ## [v0.18.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.18.0-victorialogs) diff --git a/docs/VictoriaLogs/QuickStart.md b/docs/VictoriaLogs/QuickStart.md index c4a8eb0303..9db128086a 100644 --- a/docs/VictoriaLogs/QuickStart.md +++ b/docs/VictoriaLogs/QuickStart.md @@ -9,6 +9,8 @@ menu: title: Quick Start aliases: - /VictoriaLogs/QuickStart.html +- /victorialogs/quick-start.html +- /victorialogs/quick-start/ --- # VictoriaLogs Quick Start diff --git a/docs/vmagent.md b/docs/vmagent.md index 6fe169f619..872514117f 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1637,7 +1637,7 @@ See the docs at https://docs.victoriametrics.com/vmagent/ . -cacheExpireDuration duration Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s) -configAuthKey value - Authorization key for accessing /config page. It must be passed via authKey query arg + Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings. Flag value can be read from the given file when using -configAuthKey=file:///abs/path/to/file or -configAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -configAuthKey=http://host/path or -configAuthKey=https://host/path -csvTrimTimestamp duration Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms) @@ -2006,7 +2006,7 @@ See the docs at https://docs.victoriametrics.com/vmagent/ . Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -reloadAuthKey value - Auth key for /-/reload http endpoint. It must be passed as authKey=... + Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings. Flag value can be read from the given file when using -reloadAuthKey=file:///abs/path/to/file or -reloadAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -reloadAuthKey=http://host/path or -reloadAuthKey=https://host/path -remoteWrite.aws.accessKey array Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set diff --git a/docs/vmalert-tool.md b/docs/vmalert-tool.md index baad0fcadf..07551e61a2 100644 --- a/docs/vmalert-tool.md +++ b/docs/vmalert-tool.md @@ -6,6 +6,8 @@ menu: parent: 'victoriametrics' weight: 12 title: vmalert-tool +aliases: + - /vmalert-tool.html --- # vmalert-tool diff --git a/docs/vmalert.md b/docs/vmalert.md index b3719d78d1..19fbe9e5d1 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -1263,7 +1263,7 @@ The shortlist of configuration flags is the following: Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -reloadAuthKey value - Auth key for /-/reload http endpoint. It must be passed as authKey=... + Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings. Flag value can be read from the given file when using -reloadAuthKey=file:///abs/path/to/file or -reloadAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -reloadAuthKey=http://host/path or -reloadAuthKey=https://host/path -remoteRead.basicAuth.password string Optional basic auth password for -remoteRead.url diff --git a/docs/vmauth.md b/docs/vmauth.md index 18477558f9..7956b694a4 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -1206,6 +1206,8 @@ See the docs at https://docs.victoriametrics.com/vmauth/ . Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing Supports array of values separated by comma or specified via multiple flags. Empty values are set to false. + -idleConnTimeout duration + Defines a duration for idle (keep-alive connections) to exist. Consider setting this value less than "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors. (default 50s) -internStringCacheExpireDuration duration The expiry duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s) -internStringDisableCache @@ -1287,7 +1289,7 @@ See the docs at https://docs.victoriametrics.com/vmauth/ . Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -reloadAuthKey value - Auth key for /-/reload http endpoint. It must be passed as authKey=... + Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings. Flag value can be read from the given file when using -reloadAuthKey=file:///abs/path/to/file or -reloadAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -reloadAuthKey=http://host/path or -reloadAuthKey=https://host/path -responseTimeout duration The timeout for receiving a response from backend (default 5m0s) diff --git a/lib/bytesutil/internstring.go b/lib/bytesutil/internstring.go index e62445ec3f..9c19c7840f 100644 --- a/lib/bytesutil/internstring.go +++ b/lib/bytesutil/internstring.go @@ -19,6 +19,121 @@ var ( "See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache") ) +type internStringMap struct { + mu sync.Mutex + mutable map[string]string + mutableReads uint64 + + readonly atomic.Pointer[map[string]internStringMapEntry] + + nextCleanupTime atomic.Uint64 +} + +type internStringMapEntry struct { + deadline uint64 + s string +} + +func newInternStringMap() *internStringMap { + ism := &internStringMap{ + mutable: make(map[string]string), + } + readonly := make(map[string]internStringMapEntry) + ism.readonly.Store(&readonly) + ism.nextCleanupTime.Store(fasttime.UnixTimestamp() + 61) + return ism +} + +func (m *internStringMap) getReadonly() map[string]internStringMapEntry { + return *m.readonly.Load() +} + +func (m *internStringMap) intern(s string) string { + if *disableCache || len(s) > *internStringMaxLen { + return strings.Clone(s) + } + currentTime := fasttime.UnixTimestamp() + if currentTime >= m.nextCleanupTime.Load() { + m.nextCleanupTime.Store(currentTime + 61) + m.cleanup() + } + + readonly := m.getReadonly() + e, ok := readonly[s] + if ok { + // Fast path - the string has been found in readonly map + return e.s + } + + // Slower path - search for the string in mutable map + m.mu.Lock() + sInterned, ok := m.mutable[s] + if !ok { + // Verify whether the s has been already registered by concurrent goroutines in m.readonly + readonly = m.getReadonly() + e, ok = readonly[s] + if !ok { + // Slowest path - register the string in mutable map. + // Make a new copy for s in order to remove references from possible bigger string s refers to. + sInterned = strings.Clone(s) + m.mutable[sInterned] = sInterned + } else { + sInterned = e.s + } + } + m.mutableReads++ + if m.mutableReads > uint64(len(readonly)) { + m.migrateMutableToReadonlyLocked() + m.mutableReads = 0 + } + m.mu.Unlock() + + return sInterned +} + +func (m *internStringMap) migrateMutableToReadonlyLocked() { + readonly := m.getReadonly() + readonlyCopy := make(map[string]internStringMapEntry, len(readonly)+len(m.mutable)) + for k, e := range readonly { + readonlyCopy[k] = e + } + deadline := fasttime.UnixTimestamp() + uint64(cacheExpireDuration.Seconds()+0.5) + for k, s := range m.mutable { + readonlyCopy[k] = internStringMapEntry{ + s: s, + deadline: deadline, + } + } + m.mutable = make(map[string]string) + m.readonly.Store(&readonlyCopy) +} + +func (m *internStringMap) cleanup() { + m.mu.Lock() + defer m.mu.Unlock() + + readonly := m.getReadonly() + currentTime := fasttime.UnixTimestamp() + needCleanup := false + for _, e := range readonly { + if e.deadline <= currentTime { + needCleanup = true + break + } + } + if !needCleanup { + return + } + + readonlyCopy := make(map[string]internStringMapEntry, len(readonly)) + for k, e := range readonly { + if e.deadline > currentTime { + readonlyCopy[k] = e + } + } + m.readonly.Store(&readonlyCopy) +} + func isSkipCache(s string) bool { return *disableCache || len(s) > *internStringMaxLen } @@ -33,52 +148,7 @@ func InternBytes(b []byte) string { // // This may be needed for reducing the amounts of allocated memory. func InternString(s string) string { - if isSkipCache(s) { - // Make a new copy for s in order to remove references from possible bigger string s refers to. - // This also protects from cases when s points to unsafe string - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3227 - return strings.Clone(s) - } - - ct := fasttime.UnixTimestamp() - if v, ok := internStringsMap.Load(s); ok { - e := v.(*ismEntry) - if e.lastAccessTime.Load()+10 < ct { - // Reduce the frequency of e.lastAccessTime update to once per 10 seconds - // in order to improve the fast path speed on systems with many CPU cores. - e.lastAccessTime.Store(ct) - } - return e.s - } - // Make a new copy for s in order to remove references from possible bigger string s refers to. - sCopy := strings.Clone(s) - e := &ismEntry{ - s: sCopy, - } - e.lastAccessTime.Store(ct) - internStringsMap.Store(sCopy, e) - - if needCleanup(&internStringsMapLastCleanupTime, ct) { - // Perform a global cleanup for internStringsMap by removing items, which weren't accessed during the last 5 minutes. - m := &internStringsMap - deadline := ct - uint64(cacheExpireDuration.Seconds()) - m.Range(func(k, v interface{}) bool { - e := v.(*ismEntry) - if e.lastAccessTime.Load() < deadline { - m.Delete(k) - } - return true - }) - } - - return sCopy + return ism.intern(s) } -type ismEntry struct { - lastAccessTime atomic.Uint64 - s string -} - -var ( - internStringsMap sync.Map - internStringsMapLastCleanupTime atomic.Uint64 -) +var ism = newInternStringMap() diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go index d3c75d85fb..4c9c768e25 100644 --- a/lib/httpserver/httpserver.go +++ b/lib/httpserver/httpserver.go @@ -396,6 +396,18 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4128 fmt.Fprintf(w, "User-agent: *\nDisallow: /\n") return + case "/config", "/-/reload": + // only some components (vmagent, vmalert, etc.) support these handlers + // these components are responsible for CheckAuthFlag call + // see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6329 + w = &responseWriterWithAbort{ + ResponseWriter: w, + } + if !rh(w, r) { + Errorf(w, r, "unsupported path requested: %q", r.URL.Path) + unsupportedRequestErrors.Inc() + } + return default: if strings.HasPrefix(r.URL.Path, "/debug/pprof/") { pprofRequests.Inc() diff --git a/lib/protoparser/opentelemetry/firehose/http.go b/lib/protoparser/opentelemetry/firehose/http.go index a222e72796..d17b14a0d6 100644 --- a/lib/protoparser/opentelemetry/firehose/http.go +++ b/lib/protoparser/opentelemetry/firehose/http.go @@ -2,6 +2,7 @@ package firehose import ( "fmt" + "html" "net/http" "time" ) @@ -12,11 +13,12 @@ import ( func WriteSuccessResponse(w http.ResponseWriter, r *http.Request) { requestID := r.Header.Get("X-Amz-Firehose-Request-Id") if requestID == "" { - // This isn't a AWS firehose request - just return an empty response in this case. + // This isn't an AWS firehose request - just return an empty response in this case. w.WriteHeader(http.StatusOK) return } + requestID = html.EscapeString(requestID) body := fmt.Sprintf(`{"requestId":%q,"timestamp":%d}`, requestID, time.Now().UnixMilli()) h := w.Header() diff --git a/lib/streamaggr/dedup.go b/lib/streamaggr/dedup.go index 399e9a3dc8..d04980dba6 100644 --- a/lib/streamaggr/dedup.go +++ b/lib/streamaggr/dedup.go @@ -2,10 +2,13 @@ package streamaggr import ( "sync" + "sync/atomic" "unsafe" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/cespare/xxhash/v2" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil" ) const dedupAggrShardsCount = 128 @@ -24,7 +27,12 @@ type dedupAggrShard struct { type dedupAggrShardNopad struct { mu sync.Mutex - m map[string]dedupAggrSample + m map[string]*dedupAggrSample + + samplesBuf []dedupAggrSample + + sizeBytes atomic.Uint64 + itemsCount atomic.Uint64 } type dedupAggrSample struct { @@ -42,7 +50,7 @@ func newDedupAggr() *dedupAggr { func (da *dedupAggr) sizeBytes() uint64 { n := uint64(unsafe.Sizeof(*da)) for i := range da.shards { - n += da.shards[i].sizeBytes() + n += da.shards[i].sizeBytes.Load() } return n } @@ -50,28 +58,11 @@ func (da *dedupAggr) sizeBytes() uint64 { func (da *dedupAggr) itemsCount() uint64 { n := uint64(0) for i := range da.shards { - n += da.shards[i].itemsCount() + n += da.shards[i].itemsCount.Load() } return n } -func (das *dedupAggrShard) sizeBytes() uint64 { - das.mu.Lock() - n := uint64(unsafe.Sizeof(*das)) - for k, s := range das.m { - n += uint64(len(k)) + uint64(unsafe.Sizeof(k)+unsafe.Sizeof(s)) - } - das.mu.Unlock() - return n -} - -func (das *dedupAggrShard) itemsCount() uint64 { - das.mu.Lock() - n := uint64(len(das.m)) - das.mu.Unlock() - return n -} - func (da *dedupAggr) pushSamples(samples []pushSample) { pss := getPerShardSamples() shards := pss.shards @@ -113,7 +104,7 @@ func (ctx *dedupFlushCtx) reset() { ctx.samples = ctx.samples[:0] } -func (da *dedupAggr) flush(f func(samples []pushSample), resetState bool) { +func (da *dedupAggr) flush(f func(samples []pushSample)) { var wg sync.WaitGroup for i := range da.shards { flushConcurrencyCh <- struct{}{} @@ -125,7 +116,7 @@ func (da *dedupAggr) flush(f func(samples []pushSample), resetState bool) { }() ctx := getDedupFlushCtx() - shard.flush(ctx, f, resetState) + shard.flush(ctx, f) putDedupFlushCtx(ctx) }(&da.shards[i]) } @@ -169,36 +160,43 @@ func (das *dedupAggrShard) pushSamples(samples []pushSample) { m := das.m if m == nil { - m = make(map[string]dedupAggrSample, len(samples)) + m = make(map[string]*dedupAggrSample, len(samples)) das.m = m } + samplesBuf := das.samplesBuf for _, sample := range samples { s, ok := m[sample.key] if !ok { + samplesBuf = slicesutil.SetLength(samplesBuf, len(samplesBuf)+1) + s = &samplesBuf[len(samplesBuf)-1] + s.value = sample.value + s.timestamp = sample.timestamp + key := bytesutil.InternString(sample.key) - m[key] = dedupAggrSample{ - value: sample.value, - timestamp: sample.timestamp, - } + m[key] = s + + das.itemsCount.Add(1) + das.sizeBytes.Add(uint64(len(key)) + uint64(unsafe.Sizeof(key)+unsafe.Sizeof(s)+unsafe.Sizeof(*s))) continue } // Update the existing value according to logic described at https://docs.victoriametrics.com/#deduplication if sample.timestamp > s.timestamp || (sample.timestamp == s.timestamp && sample.value > s.value) { - key := bytesutil.InternString(sample.key) - m[key] = dedupAggrSample{ - value: sample.value, - timestamp: sample.timestamp, - } + s.value = sample.value + s.timestamp = sample.timestamp } } + das.samplesBuf = samplesBuf } -func (das *dedupAggrShard) flush(ctx *dedupFlushCtx, f func(samples []pushSample), resetState bool) { +func (das *dedupAggrShard) flush(ctx *dedupFlushCtx, f func(samples []pushSample)) { das.mu.Lock() m := das.m - if resetState && len(m) > 0 { - das.m = make(map[string]dedupAggrSample, len(m)) + if len(m) > 0 { + das.m = make(map[string]*dedupAggrSample, len(m)) + das.sizeBytes.Store(0) + das.itemsCount.Store(0) + das.samplesBuf = das.samplesBuf[:0] } das.mu.Unlock() @@ -216,7 +214,7 @@ func (das *dedupAggrShard) flush(ctx *dedupFlushCtx, f func(samples []pushSample }) // Limit the number of samples per each flush in order to limit memory usage. - if len(dstSamples) >= 100_000 { + if len(dstSamples) >= 10_000 { f(dstSamples) clear(dstSamples) dstSamples = dstSamples[:0] diff --git a/lib/streamaggr/dedup_test.go b/lib/streamaggr/dedup_test.go index f0b4cc9517..91ce09e136 100644 --- a/lib/streamaggr/dedup_test.go +++ b/lib/streamaggr/dedup_test.go @@ -23,8 +23,8 @@ func TestDedupAggrSerial(t *testing.T) { da.pushSamples(samples) } - if n := da.sizeBytes(); n > 4_200_000 { - t.Fatalf("too big dedupAggr state before flush: %d bytes; it shouldn't exceed 4_200_000 bytes", n) + if n := da.sizeBytes(); n > 5_000_000 { + t.Fatalf("too big dedupAggr state before flush: %d bytes; it shouldn't exceed 5_000_000 bytes", n) } if n := da.itemsCount(); n != seriesCount { t.Fatalf("unexpected itemsCount; got %d; want %d", n, seriesCount) @@ -39,7 +39,7 @@ func TestDedupAggrSerial(t *testing.T) { } mu.Unlock() } - da.flush(flushSamples, true) + da.flush(flushSamples) if !reflect.DeepEqual(expectedSamplesMap, flushedSamplesMap) { t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", flushedSamplesMap, expectedSamplesMap) diff --git a/lib/streamaggr/dedup_timing_test.go b/lib/streamaggr/dedup_timing_test.go index abb1d04517..2b6bab25c9 100644 --- a/lib/streamaggr/dedup_timing_test.go +++ b/lib/streamaggr/dedup_timing_test.go @@ -4,7 +4,6 @@ import ( "fmt" "sync/atomic" "testing" - "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" ) @@ -17,22 +16,8 @@ func BenchmarkDedupAggr(b *testing.B) { } } -func BenchmarkDedupAggrFlushSerial(b *testing.B) { - as := newTotalAggrState(time.Hour, true, true) - benchSamples := newBenchSamples(100_000) - da := newDedupAggr() - da.pushSamples(benchSamples) - - b.ResetTimer() - b.ReportAllocs() - b.SetBytes(int64(len(benchSamples))) - for i := 0; i < b.N; i++ { - da.flush(as.pushSamples, false) - } -} - func benchmarkDedupAggr(b *testing.B, samplesPerPush int) { - const loops = 100 + const loops = 2 benchSamples := newBenchSamples(samplesPerPush) da := newDedupAggr() diff --git a/lib/streamaggr/deduplicator.go b/lib/streamaggr/deduplicator.go index 3ec25f5fa3..0212bfd9ed 100644 --- a/lib/streamaggr/deduplicator.go +++ b/lib/streamaggr/deduplicator.go @@ -166,7 +166,7 @@ func (d *Deduplicator) flush(pushFunc PushFunc, dedupInterval time.Duration) { ctx.labels = labels ctx.samples = samples putDeduplicatorFlushCtx(ctx) - }, true) + }) duration := time.Since(startTime) d.dedupFlushDuration.Update(duration.Seconds()) diff --git a/lib/streamaggr/streamaggr.go b/lib/streamaggr/streamaggr.go index 92fdba57a8..34ad118cf3 100644 --- a/lib/streamaggr/streamaggr.go +++ b/lib/streamaggr/streamaggr.go @@ -747,7 +747,7 @@ func (a *aggregator) dedupFlush(dedupInterval time.Duration) { startTime := time.Now() - a.da.flush(a.pushSamples, true) + a.da.flush(a.pushSamples) d := time.Since(startTime) a.dedupFlushDuration.Update(d.Seconds())