lib/httpserver: expose process_open_fds and process_max_fds metrics

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/402
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1037

docs/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 # tip
 
+* FEATURE: expose `process_open_fds` and `process_max_fds` metrics. These metrics can be used for alerting when `process_open_fds` reaches `process_max_fds`.
+
 
 # [v1.53.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.53.1)
 

go.mod

@@ -8,7 +8,7 @@ require (
 	// Do not use the original github.com/valyala/fasthttp because of issues
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.12
-	github.com/VictoriaMetrics/metrics v1.12.3
+	github.com/VictoriaMetrics/metrics v1.13.0
 	github.com/VictoriaMetrics/metricsql v0.10.0
 	github.com/aws/aws-sdk-go v1.37.1
 	github.com/cespare/xxhash/v2 v2.1.1

go.sum

@@ -85,8 +85,8 @@ github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6Ro
 github.com/VictoriaMetrics/fasthttp v1.0.12 h1:Ag0E119yrH4BTxVyjKD9TeiSImtG9bUcg/stItLJhSE=
 github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrtNHXBJR6N376EgiSHU=
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I=
-github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
+github.com/VictoriaMetrics/metrics v1.13.0 h1:CvQROcaiptG0nCvfbGfIaES8aG0j4AyB7J3c+N6NhXs=
+github.com/VictoriaMetrics/metrics v1.13.0/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metricsql v0.10.0 h1:45BARAP2shaL/5p67Hvz+YrWUbr0X0VCy9t+gvdIm8o=
 github.com/VictoriaMetrics/metricsql v0.10.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=

lib/httpserver/metrics.go

@@ -19,6 +19,7 @@ var versionRe = regexp.MustCompile(`v\d+\.\d+\.\d+`)
 // WritePrometheusMetrics writes all the registered metrics to w in Prometheus exposition format.
 func WritePrometheusMetrics(w io.Writer) {
 	metrics.WritePrometheus(w, true)
+	metrics.WriteFDMetrics(w)
 
 	fmt.Fprintf(w, "vm_app_version{version=%q, short_version=%q} 1\n", buildinfo.Version,
 		versionRe.FindString(buildinfo.Version))

vendor/github.com/VictoriaMetrics/metrics/metrics.go

@@ -58,11 +58,17 @@ func WritePrometheus(w io.Writer, exposeProcessMetrics bool) {
 //         metrics.WriteProcessMetrics(w)
 //     })
 //
+// See also WrteFDMetrics.
 func WriteProcessMetrics(w io.Writer) {
 	writeGoMetrics(w)
 	writeProcessMetrics(w)
 }
 
+// WriteFDMetrics writes `process_max_fds` and `process_open_fds` metrics to w.
+func WriteFDMetrics(w io.Writer) {
+	writeFDMetrics(w)
+}
+
 // UnregisterMetric removes metric with the given name from default set.
 func UnregisterMetric(name string) bool {
 	return defaultSet.UnregisterMetric(name)

vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go

@@ -1,11 +1,15 @@
 package metrics
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
+	"os"
+	"strconv"
+	"strings"
 	"time"
 )
 
@@ -81,3 +85,71 @@ func writeProcessMetrics(w io.Writer) {
 }
 
 var startTimeSeconds = time.Now().Unix()
+
+// WriteFDMetrics writes process_max_fds and process_open_fds metrics to w.
+func writeFDMetrics(w io.Writer) {
+	totalOpenFDs, err := getOpenFDsCount("/proc/self/fd")
+	if err != nil {
+		log.Printf("ERROR: cannot determine open file descriptors count: %s", err)
+		return
+	}
+	maxOpenFDs, err := getMaxFilesLimit("/proc/self/limits")
+	if err != nil {
+		log.Printf("ERROR: cannot determine the limit on open file descritors: %s", err)
+		return
+	}
+	fmt.Fprintf(w, "process_max_fds %d\n", maxOpenFDs)
+	fmt.Fprintf(w, "process_open_fds %d\n", totalOpenFDs)
+}
+
+func getOpenFDsCount(path string) (uint64, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	var totalOpenFDs uint64
+	for {
+		names, err := f.Readdirnames(512)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return 0, fmt.Errorf("unexpected error at Readdirnames: %s", err)
+		}
+		totalOpenFDs += uint64(len(names))
+	}
+	return totalOpenFDs, nil
+}
+
+func getMaxFilesLimit(path string) (uint64, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	prefix := "Max open files"
+	scan := bufio.NewScanner(f)
+	for scan.Scan() {
+		s := scan.Text()
+		if !strings.HasPrefix(s, prefix) {
+			continue
+		}
+		text := strings.TrimSpace(s[len(prefix):])
+		// Extract soft limit.
+		n := strings.IndexByte(text, ' ')
+		if n < 0 {
+			return 0, fmt.Errorf("cannot extract soft limit from %q", s)
+		}
+		text = text[:n]
+		if text == "unlimited" {
+			return 1<<64 - 1, nil
+		}
+		limit, err := strconv.ParseUint(text, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse soft limit from %q: %s", s, err)
+		}
+		return limit, nil
+	}
+	return 0, fmt.Errorf("cannot find max open files limit")
+}

vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go

@@ -9,3 +9,7 @@ import (
 func writeProcessMetrics(w io.Writer) {
 	// TODO: implement it
 }
+
+func writeFDMetrics(w io.Writer) {
+	// TODO: implement it.
+}

vendor/modules.txt

@@ -14,7 +14,7 @@ github.com/VictoriaMetrics/fastcache
 github.com/VictoriaMetrics/fasthttp
 github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
-# github.com/VictoriaMetrics/metrics v1.12.3
+# github.com/VictoriaMetrics/metrics v1.13.0
 github.com/VictoriaMetrics/metrics
 # github.com/VictoriaMetrics/metricsql v0.10.0
 github.com/VictoriaMetrics/metricsql