diff --git a/go.mod b/go.mod index 7c87efac4..64633537d 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/VictoriaMetrics/VictoriaMetrics require ( github.com/VictoriaMetrics/fastcache v1.5.1 - github.com/VictoriaMetrics/metrics v1.6.2 + github.com/VictoriaMetrics/metrics v1.7.0 github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18 github.com/golang/snappy v0.0.1 github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index a699aa0bb..c4b089288 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,8 @@ github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/VictoriaMetrics/fastcache v1.5.1 h1:qHgHjyoNFV7jgucU8QZUuU4gcdhfs8QW1kw68OD2Lag= github.com/VictoriaMetrics/fastcache v1.5.1/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= -github.com/VictoriaMetrics/metrics v1.6.2 h1:VMe8c8ZBPgNVZkPoT06LsoU2nb+8e7iPaOWbVRNhxjo= -github.com/VictoriaMetrics/metrics v1.6.2/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ= +github.com/VictoriaMetrics/metrics v1.7.0 h1:+bdBpPEMOSgOwoQFf4KHqgeAy6xiXn/uzlrKx2YSCT8= +github.com/VictoriaMetrics/metrics v1.7.0/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= diff --git a/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go b/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go new file mode 100644 index 000000000..f8b606731 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go @@ -0,0 +1,64 @@ +package metrics + +import ( + "fmt" + "io" + "runtime" + + "github.com/valyala/histogram" +) + +func writeGoMetrics(w io.Writer) { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + fmt.Fprintf(w, "go_memstats_alloc_bytes %d\n", ms.Alloc) + fmt.Fprintf(w, "go_memstats_alloc_bytes_total %d\n", ms.TotalAlloc) + fmt.Fprintf(w, "go_memstats_buck_hash_sys_bytes %d\n", ms.BuckHashSys) + fmt.Fprintf(w, "go_memstats_frees_total %d\n", ms.Frees) + fmt.Fprintf(w, "go_memstats_gc_cpu_fraction %g\n", ms.GCCPUFraction) + fmt.Fprintf(w, "go_memstats_gc_sys_bytes %d\n", ms.GCSys) + fmt.Fprintf(w, "go_memstats_heap_alloc_bytes %d\n", ms.HeapAlloc) + fmt.Fprintf(w, "go_memstats_heap_idle_bytes %d\n", ms.HeapIdle) + fmt.Fprintf(w, "go_memstats_heap_inuse_bytes %d\n", ms.HeapInuse) + fmt.Fprintf(w, "go_memstats_heap_objects %d\n", ms.HeapObjects) + fmt.Fprintf(w, "go_memstats_heap_released_bytes %d\n", ms.HeapReleased) + fmt.Fprintf(w, "go_memstats_heap_sys_bytes %d\n", ms.HeapSys) + fmt.Fprintf(w, "go_memstats_last_gc_time_seconds %g\n", float64(ms.LastGC)/1e9) + fmt.Fprintf(w, "go_memstats_lookups_total %d\n", ms.Lookups) + fmt.Fprintf(w, "go_memstats_mallocs_total %d\n", ms.Mallocs) + fmt.Fprintf(w, "go_memstats_mcache_inuse_bytes %d\n", ms.MCacheInuse) + fmt.Fprintf(w, "go_memstats_mcache_sys_bytes %d\n", ms.MCacheSys) + fmt.Fprintf(w, "go_memstats_mspan_inuse_bytes %d\n", ms.MSpanInuse) + fmt.Fprintf(w, "go_memstats_mspan_sys_bytes %d\n", ms.MSpanSys) + fmt.Fprintf(w, "go_memstats_next_gc_bytes %d\n", ms.NextGC) + fmt.Fprintf(w, "go_memstats_other_sys_bytes %d\n", ms.OtherSys) + fmt.Fprintf(w, "go_memstats_stack_inuse_bytes %d\n", ms.StackInuse) + fmt.Fprintf(w, "go_memstats_stack_sys_bytes %d\n", ms.StackSys) + fmt.Fprintf(w, "go_memstats_sys_bytes %d\n", ms.Sys) + + fmt.Fprintf(w, "go_cgo_calls_count %d\n", runtime.NumCgoCall()) + fmt.Fprintf(w, "go_cpu_count %d\n", runtime.NumCPU()) + + gcPauses := histogram.NewFast() + for _, pauseNs := range ms.PauseNs[:] { + gcPauses.Update(float64(pauseNs) / 1e9) + } + phis := []float64{0, 0.25, 0.5, 0.75, 1} + quantiles := make([]float64, 0, len(phis)) + for i, q := range gcPauses.Quantiles(quantiles[:0], phis) { + fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %g`+"\n", phis[i], q) + } + fmt.Fprintf(w, `go_gc_duration_seconds_sum %g`+"\n", float64(ms.PauseTotalNs)/1e9) + fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC) + fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC) + + fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0)) + fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine()) + numThread, _ := runtime.ThreadCreateProfile(nil) + fmt.Fprintf(w, `go_threads %d`+"\n", numThread) + + // Export build details. + fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version()) + fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n", + runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT()) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/metrics.go b/vendor/github.com/VictoriaMetrics/metrics/metrics.go index c28a08ad5..8fc8f6e0e 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/metrics.go +++ b/vendor/github.com/VictoriaMetrics/metrics/metrics.go @@ -13,11 +13,7 @@ package metrics import ( - "fmt" "io" - "runtime" - - "github.com/valyala/histogram" ) type namedMetric struct { @@ -33,8 +29,8 @@ var defaultSet = NewSet() // WritePrometheus writes all the registered metrics in Prometheus format to w. // -// If exposeProcessMetrics is true, then various `go_*` metrics are exposed -// for the current process. +// If exposeProcessMetrics is true, then various `go_*` and `process_*` metrics +// are exposed for the current process. // // The WritePrometheus func is usually called inside "/metrics" handler: // @@ -45,61 +41,7 @@ var defaultSet = NewSet() func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { defaultSet.WritePrometheus(w) if exposeProcessMetrics { + writeGoMetrics(w) writeProcessMetrics(w) } } - -func writeProcessMetrics(w io.Writer) { - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - fmt.Fprintf(w, `go_memstats_alloc_bytes %d`+"\n", ms.Alloc) - fmt.Fprintf(w, `go_memstats_alloc_bytes_total %d`+"\n", ms.TotalAlloc) - fmt.Fprintf(w, `go_memstats_buck_hash_sys_bytes %d`+"\n", ms.BuckHashSys) - fmt.Fprintf(w, `go_memstats_frees_total %d`+"\n", ms.Frees) - fmt.Fprintf(w, `go_memstats_gc_cpu_fraction %f`+"\n", ms.GCCPUFraction) - fmt.Fprintf(w, `go_memstats_gc_sys_bytes %d`+"\n", ms.GCSys) - fmt.Fprintf(w, `go_memstats_heap_alloc_bytes %d`+"\n", ms.HeapAlloc) - fmt.Fprintf(w, `go_memstats_heap_idle_bytes %d`+"\n", ms.HeapIdle) - fmt.Fprintf(w, `go_memstats_heap_inuse_bytes %d`+"\n", ms.HeapInuse) - fmt.Fprintf(w, `go_memstats_heap_objects %d`+"\n", ms.HeapObjects) - fmt.Fprintf(w, `go_memstats_heap_released_bytes %d`+"\n", ms.HeapReleased) - fmt.Fprintf(w, `go_memstats_heap_sys_bytes %d`+"\n", ms.HeapSys) - fmt.Fprintf(w, `go_memstats_last_gc_time_seconds %f`+"\n", float64(ms.LastGC)/1e9) - fmt.Fprintf(w, `go_memstats_lookups_total %d`+"\n", ms.Lookups) - fmt.Fprintf(w, `go_memstats_mallocs_total %d`+"\n", ms.Mallocs) - fmt.Fprintf(w, `go_memstats_mcache_inuse_bytes %d`+"\n", ms.MCacheInuse) - fmt.Fprintf(w, `go_memstats_mcache_sys_bytes %d`+"\n", ms.MCacheSys) - fmt.Fprintf(w, `go_memstats_mspan_inuse_bytes %d`+"\n", ms.MSpanInuse) - fmt.Fprintf(w, `go_memstats_mspan_sys_bytes %d`+"\n", ms.MSpanSys) - fmt.Fprintf(w, `go_memstats_next_gc_bytes %d`+"\n", ms.NextGC) - fmt.Fprintf(w, `go_memstats_other_sys_bytes %d`+"\n", ms.OtherSys) - fmt.Fprintf(w, `go_memstats_stack_inuse_bytes %d`+"\n", ms.StackInuse) - fmt.Fprintf(w, `go_memstats_stack_sys_bytes %d`+"\n", ms.StackSys) - fmt.Fprintf(w, `go_memstats_sys_bytes %d`+"\n", ms.Sys) - - fmt.Fprintf(w, `go_cgo_calls_count %d`+"\n", runtime.NumCgoCall()) - fmt.Fprintf(w, `go_cpu_count %d`+"\n", runtime.NumCPU()) - - gcPauses := histogram.NewFast() - for _, pauseNs := range ms.PauseNs[:] { - gcPauses.Update(float64(pauseNs) / 1e9) - } - phis := []float64{0, 0.25, 0.5, 0.75, 1} - quantiles := make([]float64, 0, len(phis)) - for i, q := range gcPauses.Quantiles(quantiles[:0], phis) { - fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %f`+"\n", phis[i], q) - } - fmt.Fprintf(w, `go_gc_duration_seconds_sum %f`+"\n", float64(ms.PauseTotalNs)/1e9) - fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC) - fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC) - - fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0)) - fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine()) - numThread, _ := runtime.ThreadCreateProfile(nil) - fmt.Fprintf(w, `go_threads %d`+"\n", numThread) - - // Export build details. - fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version()) - fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n", - runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT()) -} diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go new file mode 100644 index 000000000..3f4744a81 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go @@ -0,0 +1,79 @@ +package metrics + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "time" +) + +const statFilepath = "/proc/self/stat" + +// See https://github.com/prometheus/procfs/blob/a4ac0826abceb44c40fc71daed2b301db498b93e/proc_stat.go#L40 . +const userHZ = 100 + +// See http://man7.org/linux/man-pages/man5/proc.5.html +type procStat struct { + State byte + Ppid int + Pgrp int + Session int + TtyNr int + Tpgid int + Flags uint + Minflt uint + Cminflt uint + Majflt uint + Cmajflt uint + Utime uint + Stime uint + Cutime int + Cstime int + Priority int + Nice int + NumThreads int + ItrealValue int + Starttime uint64 + Vsize uint + Rss int +} + +func writeProcessMetrics(w io.Writer) { + data, err := ioutil.ReadFile(statFilepath) + if err != nil { + log.Printf("ERROR: cannot open %s: %s", statFilepath, err) + return + } + // Search for the end of command. + n := bytes.LastIndex(data, []byte(") ")) + if n < 0 { + log.Printf("ERROR: cannot find command in parentheses in %q read from %s", data, statFilepath) + return + } + data = data[n+2:] + + var p procStat + bb := bytes.NewBuffer(data) + _, err = fmt.Fscanf(bb, "%c %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d", + &p.State, &p.Ppid, &p.Pgrp, &p.Session, &p.TtyNr, &p.Tpgid, &p.Flags, &p.Minflt, &p.Cminflt, &p.Majflt, &p.Cmajflt, + &p.Utime, &p.Stime, &p.Cutime, &p.Cstime, &p.Priority, &p.Nice, &p.NumThreads, &p.ItrealValue, &p.Starttime, &p.Vsize, &p.Rss) + if err != nil { + log.Printf("ERROR: cannot parse %q read from %s: %s", data, statFilepath, err) + return + } + + // It is expensive obtaining `process_open_fds` when big number of file descriptors is opened, + // don't do it here. + + fmt.Fprintf(w, "process_cpu_seconds_total %g\n", float64(p.Utime+p.Stime)/userHZ) + fmt.Fprintf(w, "process_major_pagefaults_total %d\n", p.Majflt) + fmt.Fprintf(w, "process_minor_pagefaults_total %d\n", p.Minflt) + fmt.Fprintf(w, "process_num_threads %d\n", p.NumThreads) + fmt.Fprintf(w, "process_resident_memory_bytes %d\n", p.Rss*4096) + fmt.Fprintf(w, "process_start_time_seconds %d\n", startTimeSeconds) + fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", p.Vsize) +} + +var startTimeSeconds = time.Now().Unix() diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go new file mode 100644 index 000000000..6874de33e --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go @@ -0,0 +1,11 @@ +// +build !linux + +package metrics + +import ( + "io" +) + +func writeProcessMetrics(w io.Writer) { + // TODO: implement it +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 7fe78a42f..5fd435171 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,6 +1,6 @@ # github.com/VictoriaMetrics/fastcache v1.5.1 github.com/VictoriaMetrics/fastcache -# github.com/VictoriaMetrics/metrics v1.6.2 +# github.com/VictoriaMetrics/metrics v1.7.0 github.com/VictoriaMetrics/metrics # github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18 github.com/cespare/xxhash/v2