From b50024812e496d6b4f2e640b12e2c49cdd2a6d31 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 13 May 2021 09:02:13 +0300 Subject: [PATCH] adds cgroupsv2 support (#1283) * adds cgroupv2 limits support https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1269 * small fix * changes Atoi to ParseUint --- lib/cgroup/cpu.go | 50 +++++++++++++++++++++++---- lib/cgroup/cpu_test.go | 14 ++++++++ lib/cgroup/mem.go | 10 ++++++ lib/cgroup/testdata/cgroup/cpu.max | 1 + lib/cgroup/testdata/cgroup/memory.max | 1 + lib/cgroup/testdata/self/cgroupv2 | 1 + lib/cgroup/util.go | 1 + lib/cgroup/util_test.go | 2 ++ 8 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 lib/cgroup/testdata/cgroup/cpu.max create mode 100644 lib/cgroup/testdata/cgroup/memory.max create mode 100644 lib/cgroup/testdata/self/cgroupv2 diff --git a/lib/cgroup/cpu.go b/lib/cgroup/cpu.go index 894f4dbdb..8c3972d50 100644 --- a/lib/cgroup/cpu.go +++ b/lib/cgroup/cpu.go @@ -1,6 +1,7 @@ package cgroup import ( + "fmt" "io/ioutil" "os" "runtime" @@ -41,20 +42,28 @@ func updateGOMAXPROCSToCPUQuota() { } func getCPUQuota() float64 { - quotaUS, err := getCPUStat("cpu.cfs_quota_us") + cpuQuota, err := getCPUStatGeneric() if err != nil { return 0 } - if quotaUS <= 0 { + + if cpuQuota <= 0 { // The quota isn't set. This may be the case in multilevel containers. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-674423728 return getOnlineCPUCount() } - periodUS, err := getCPUStat("cpu.cfs_period_us") - if err != nil { - return 0 + return cpuQuota +} + +func getCPUStatGeneric() (float64, error) { + quotaUS, err := getCPUStat("cpu.cfs_quota_us") + if err == nil { + periodUS, err := getCPUStat("cpu.cfs_period_us") + if err == nil { + return float64(quotaUS) / float64(periodUS), nil + } } - return float64(quotaUS) / float64(periodUS) + return getCPUStatV2("/sys/fs/cgroup", "/proc/self/cgroup") } func getCPUStat(statName string) (int64, error) { @@ -74,6 +83,35 @@ func getOnlineCPUCount() float64 { return n } +func getCPUStatV2(sysPrefix, cgroupPath string) (float64, error) { + data, err := getFileContents("cpu.max", sysPrefix, cgroupPath, "") + if err != nil { + return 0, err + } + return parseCPUMax(data) +} + +// https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#cpu +func parseCPUMax(data string) (float64, error) { + data = strings.TrimRight(data, "\r\n") + bounds := strings.Split(data, " ") + if len(bounds) != 2 { + return 0, fmt.Errorf("unexpected count: %d, want quota and period, got: %s", len(bounds), data) + } + if bounds[0] == "max" { + return -1, nil + } + quota, err := strconv.ParseUint(bounds[0], 10, 64) + if err != nil { + return 0, err + } + period, err := strconv.ParseUint(bounds[1], 10, 64) + if err != nil { + return 0, err + } + return float64(quota) / float64(period), nil +} + func countCPUs(data string) int { data = strings.TrimSpace(data) n := 0 diff --git a/lib/cgroup/cpu_test.go b/lib/cgroup/cpu_test.go index 37aaceb6c..20084b305 100644 --- a/lib/cgroup/cpu_test.go +++ b/lib/cgroup/cpu_test.go @@ -22,3 +22,17 @@ func TestCountCPUs(t *testing.T) { f("0-3", 4) f("0-6", 7) } + +func TestGetCPUStatV2(t *testing.T) { + f := func(sysPrefix, cgroupPath string, expectedCPU float64) { + t.Helper() + got, err := getCPUStatV2(sysPrefix, cgroupPath) + if err != nil { + t.Fatalf("unexpected error: %s, sysPrefix: %s, cgroupPath: %s", err, sysPrefix, cgroupPath) + } + if got != expectedCPU { + t.Fatalf("unexpected result from getCPUStatV2(%s, %s), got %f, want %f", sysPrefix, cgroupPath, got, expectedCPU) + } + } + f("testdata/cgroup", "testdata/self/cgroupv2", 2) +} diff --git a/lib/cgroup/mem.go b/lib/cgroup/mem.go index 92ac92b70..e91f8145b 100644 --- a/lib/cgroup/mem.go +++ b/lib/cgroup/mem.go @@ -13,12 +13,22 @@ func GetMemoryLimit() int64 { // This should properly determine the limit inside lxc container. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/84 n, err := getMemStat("memory.limit_in_bytes") + if err == nil { + return n + } + // https: //www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#memory-interface-files + n, err = getMemStatV2() if err != nil { return 0 } + return n } +func getMemStatV2() (int64, error) { + return getStatGeneric("memory.max", "/sys/fs/cgroup", "/proc/self/cgroup", "") +} + func getMemStat(statName string) (int64, error) { return getStatGeneric(statName, "/sys/fs/cgroup/memory", "/proc/self/cgroup", "memory") } diff --git a/lib/cgroup/testdata/cgroup/cpu.max b/lib/cgroup/testdata/cgroup/cpu.max new file mode 100644 index 000000000..96856569d --- /dev/null +++ b/lib/cgroup/testdata/cgroup/cpu.max @@ -0,0 +1 @@ +200000 100000 diff --git a/lib/cgroup/testdata/cgroup/memory.max b/lib/cgroup/testdata/cgroup/memory.max new file mode 100644 index 000000000..4f58557a7 --- /dev/null +++ b/lib/cgroup/testdata/cgroup/memory.max @@ -0,0 +1 @@ +523372036854771712 diff --git a/lib/cgroup/testdata/self/cgroupv2 b/lib/cgroup/testdata/self/cgroupv2 new file mode 100644 index 000000000..62bb6b637 --- /dev/null +++ b/lib/cgroup/testdata/self/cgroupv2 @@ -0,0 +1 @@ +0::/ \ No newline at end of file diff --git a/lib/cgroup/util.go b/lib/cgroup/util.go index 3dcd64e4c..66960adb7 100644 --- a/lib/cgroup/util.go +++ b/lib/cgroup/util.go @@ -13,6 +13,7 @@ func getStatGeneric(statName, sysfsPrefix, cgroupPath, cgroupGrepLine string) (i if err != nil { return 0, err } + data = strings.TrimRight(data, "\r\n") n, err := strconv.ParseInt(data, 10, 64) if err != nil { return 0, err diff --git a/lib/cgroup/util_test.go b/lib/cgroup/util_test.go index b6123e641..a4bc2d590 100644 --- a/lib/cgroup/util_test.go +++ b/lib/cgroup/util_test.go @@ -21,6 +21,7 @@ func TestGetStatGenericSuccess(t *testing.T) { f("cpu.cfs_period_us", "testdata/cgroup", "testdata/self/cgroup", "cpu,", 500000) f("memory.limit_in_bytes", "testdata/", "testdata/self/cgroup", "memory", 9223372036854771712) f("memory.limit_in_bytes", "testdata/cgroup", "testdata/self/cgroup", "memory", 523372036854771712) + f("memory.max", "testdata/cgroup", "testdata/self/cgroupv2", "", 523372036854771712) } func TestGetStatGenericFailure(t *testing.T) { @@ -37,4 +38,5 @@ func TestGetStatGenericFailure(t *testing.T) { f("cpu.cfs_quota_us", "testdata/", "testdata/missing_folder", "cpu,") f("cpu.cfs_period_us", "testdata/", "testdata/missing_folder", "cpu,") f("memory.limit_in_bytes", "testdata/", "testdata/none_existing_folder", "memory") + f("memory.max", "testdata/", "testdata/none_existing_folder", "") }