From b5eba7059520c9a10118ea4f024ba43761598617 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 4 Feb 2021 16:33:07 +0200 Subject: [PATCH] lib/httpserver: expose `process_open_fds` and `process_max_fds` metrics See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/402 See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1037 --- docs/CHANGELOG.md | 2 + go.mod | 2 +- go.sum | 4 +- lib/httpserver/metrics.go | 1 + .../VictoriaMetrics/metrics/metrics.go | 6 ++ .../metrics/process_metrics_linux.go | 72 +++++++++++++++++++ .../metrics/process_metrics_other.go | 4 ++ vendor/modules.txt | 2 +- 8 files changed, 89 insertions(+), 4 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7b6a9420a..dd2087dfd 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,8 @@ # tip +* FEATURE: expose `process_open_fds` and `process_max_fds` metrics. These metrics can be used for alerting when `process_open_fds` reaches `process_max_fds`. + # [v1.53.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.53.1) diff --git a/go.mod b/go.mod index 3fefed5d5..79e4b271e 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( // Do not use the original github.com/valyala/fasthttp because of issues // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.12 - github.com/VictoriaMetrics/metrics v1.12.3 + github.com/VictoriaMetrics/metrics v1.13.0 github.com/VictoriaMetrics/metricsql v0.10.0 github.com/aws/aws-sdk-go v1.37.1 github.com/cespare/xxhash/v2 v2.1.1 diff --git a/go.sum b/go.sum index d8d753535..127038478 100644 --- a/go.sum +++ b/go.sum @@ -85,8 +85,8 @@ github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6Ro github.com/VictoriaMetrics/fasthttp v1.0.12 h1:Ag0E119yrH4BTxVyjKD9TeiSImtG9bUcg/stItLJhSE= github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrtNHXBJR6N376EgiSHU= github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I= -github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= +github.com/VictoriaMetrics/metrics v1.13.0 h1:CvQROcaiptG0nCvfbGfIaES8aG0j4AyB7J3c+N6NhXs= +github.com/VictoriaMetrics/metrics v1.13.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metricsql v0.10.0 h1:45BARAP2shaL/5p67Hvz+YrWUbr0X0VCy9t+gvdIm8o= github.com/VictoriaMetrics/metricsql v0.10.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= diff --git a/lib/httpserver/metrics.go b/lib/httpserver/metrics.go index 5bfc3ef81..e577f5ce3 100644 --- a/lib/httpserver/metrics.go +++ b/lib/httpserver/metrics.go @@ -19,6 +19,7 @@ var versionRe = regexp.MustCompile(`v\d+\.\d+\.\d+`) // WritePrometheusMetrics writes all the registered metrics to w in Prometheus exposition format. func WritePrometheusMetrics(w io.Writer) { metrics.WritePrometheus(w, true) + metrics.WriteFDMetrics(w) fmt.Fprintf(w, "vm_app_version{version=%q, short_version=%q} 1\n", buildinfo.Version, versionRe.FindString(buildinfo.Version)) diff --git a/vendor/github.com/VictoriaMetrics/metrics/metrics.go b/vendor/github.com/VictoriaMetrics/metrics/metrics.go index 962ceef5b..bda1cb9ae 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/metrics.go +++ b/vendor/github.com/VictoriaMetrics/metrics/metrics.go @@ -58,11 +58,17 @@ func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { // metrics.WriteProcessMetrics(w) // }) // +// See also WrteFDMetrics. func WriteProcessMetrics(w io.Writer) { writeGoMetrics(w) writeProcessMetrics(w) } +// WriteFDMetrics writes `process_max_fds` and `process_open_fds` metrics to w. +func WriteFDMetrics(w io.Writer) { + writeFDMetrics(w) +} + // UnregisterMetric removes metric with the given name from default set. func UnregisterMetric(name string) bool { return defaultSet.UnregisterMetric(name) diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go index 444299278..d8b7c7354 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go @@ -1,11 +1,15 @@ package metrics import ( + "bufio" "bytes" "fmt" "io" "io/ioutil" "log" + "os" + "strconv" + "strings" "time" ) @@ -81,3 +85,71 @@ func writeProcessMetrics(w io.Writer) { } var startTimeSeconds = time.Now().Unix() + +// WriteFDMetrics writes process_max_fds and process_open_fds metrics to w. +func writeFDMetrics(w io.Writer) { + totalOpenFDs, err := getOpenFDsCount("/proc/self/fd") + if err != nil { + log.Printf("ERROR: cannot determine open file descriptors count: %s", err) + return + } + maxOpenFDs, err := getMaxFilesLimit("/proc/self/limits") + if err != nil { + log.Printf("ERROR: cannot determine the limit on open file descritors: %s", err) + return + } + fmt.Fprintf(w, "process_max_fds %d\n", maxOpenFDs) + fmt.Fprintf(w, "process_open_fds %d\n", totalOpenFDs) +} + +func getOpenFDsCount(path string) (uint64, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + var totalOpenFDs uint64 + for { + names, err := f.Readdirnames(512) + if err == io.EOF { + break + } + if err != nil { + return 0, fmt.Errorf("unexpected error at Readdirnames: %s", err) + } + totalOpenFDs += uint64(len(names)) + } + return totalOpenFDs, nil +} + +func getMaxFilesLimit(path string) (uint64, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + prefix := "Max open files" + scan := bufio.NewScanner(f) + for scan.Scan() { + s := scan.Text() + if !strings.HasPrefix(s, prefix) { + continue + } + text := strings.TrimSpace(s[len(prefix):]) + // Extract soft limit. + n := strings.IndexByte(text, ' ') + if n < 0 { + return 0, fmt.Errorf("cannot extract soft limit from %q", s) + } + text = text[:n] + if text == "unlimited" { + return 1<<64 - 1, nil + } + limit, err := strconv.ParseUint(text, 10, 64) + if err != nil { + return 0, fmt.Errorf("cannot parse soft limit from %q: %s", s, err) + } + return limit, nil + } + return 0, fmt.Errorf("cannot find max open files limit") +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go index 6874de33e..5e6ac935d 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go @@ -9,3 +9,7 @@ import ( func writeProcessMetrics(w io.Writer) { // TODO: implement it } + +func writeFDMetrics(w io.Writer) { + // TODO: implement it. +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 9425db02a..f2d7125d0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -14,7 +14,7 @@ github.com/VictoriaMetrics/fastcache github.com/VictoriaMetrics/fasthttp github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless -# github.com/VictoriaMetrics/metrics v1.12.3 +# github.com/VictoriaMetrics/metrics v1.13.0 github.com/VictoriaMetrics/metrics # github.com/VictoriaMetrics/metricsql v0.10.0 github.com/VictoriaMetrics/metricsql