From b9bf3cbe3e31a8a5dc97c23c5d1d710e19666074 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Mon, 8 Feb 2021 16:46:22 +0300 Subject: [PATCH] refactored cgroups limits, (#1061) adds tests, remove os.Exec --- lib/cgroup/cpu.go | 17 +++++- lib/cgroup/cpu_test.go | 32 +++++++++++ lib/cgroup/mem.go | 55 ++++++++++++++++--- lib/cgroup/mem_test.go | 35 ++++++++++++ lib/cgroup/testdata/cgroup/cpu.cfs_period_us | 1 + lib/cgroup/testdata/cgroup/cpu.cfs_quota_us | 1 + .../testdata/cgroup/memory.limit_in_bytes | 1 + lib/cgroup/testdata/cgroup/memory.stat | 31 +++++++++++ .../cpu.cfs_period_us | 1 + .../cpu.cfs_quota_us | 1 + .../memory.limit_in_bytes | 1 + .../memory.stat | 31 +++++++++++ lib/cgroup/testdata/memory.stat | 1 + lib/cgroup/testdata/self/cgroup | 13 +++++ lib/cgroup/util.go | 41 +++++++++----- 15 files changed, 240 insertions(+), 22 deletions(-) create mode 100644 lib/cgroup/mem_test.go create mode 100644 lib/cgroup/testdata/cgroup/cpu.cfs_period_us create mode 100644 lib/cgroup/testdata/cgroup/cpu.cfs_quota_us create mode 100644 lib/cgroup/testdata/cgroup/memory.limit_in_bytes create mode 100644 lib/cgroup/testdata/cgroup/memory.stat create mode 100644 lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/cpu.cfs_period_us create mode 100644 lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/cpu.cfs_quota_us create mode 100644 lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/memory.limit_in_bytes create mode 100644 lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/memory.stat create mode 100644 lib/cgroup/testdata/memory.stat create mode 100644 lib/cgroup/testdata/self/cgroup diff --git a/lib/cgroup/cpu.go b/lib/cgroup/cpu.go index 56690e86d..7476b288c 100644 --- a/lib/cgroup/cpu.go +++ b/lib/cgroup/cpu.go @@ -3,6 +3,7 @@ package cgroup import ( "io/ioutil" "os" + "path" "runtime" "strconv" "strings" @@ -40,8 +41,20 @@ func updateGOMAXPROCSToCPUQuota() { runtime.GOMAXPROCS(gomaxprocs) } +func getCPUStat(sysPath, cgroupPath, statName string) (int64, error) { + n, err := readInt64(path.Join(sysPath, statName)) + if err == nil { + return n, nil + } + subPath, err := grepFirstMatch(cgroupPath, "cpu,", 2, ":") + if err != nil { + return 0, err + } + return readInt64(path.Join(sysPath, subPath, statName)) +} + func getCPUQuota() float64 { - quotaUS, err := readInt64("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "cat /sys/fs/cgroup/cpu$(cat /proc/self/cgroup | grep cpu, | cut -d: -f3)/cpu.cfs_quota_us") + quotaUS, err := getCPUStat("/sys/fs/cgroup/cpu", "/proc/self/cgroup", "cpu.cfs_quota_us") if err != nil { return 0 } @@ -50,7 +63,7 @@ func getCPUQuota() float64 { // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-674423728 return getOnlineCPUCount() } - periodUS, err := readInt64("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "cat /sys/fs/cgroup/cpu$(cat /proc/self/cgroup | grep cpu, | cut -d: -f3)/cpu.cfs_period_us") + periodUS, err := getCPUStat("/sys/fs/cgroup/cpu", "/proc/self/cgroup", "cpu.cfs_period_us") if err != nil { return 0 } diff --git a/lib/cgroup/cpu_test.go b/lib/cgroup/cpu_test.go index 37aaceb6c..5a592e573 100644 --- a/lib/cgroup/cpu_test.go +++ b/lib/cgroup/cpu_test.go @@ -22,3 +22,35 @@ func TestCountCPUs(t *testing.T) { f("0-3", 4) f("0-6", 7) } + +func TestGetCPUStatQuota(t *testing.T) { + f := func(sysPath, cgroupPath string, want int64, wantErr bool) { + t.Helper() + got, err := getCPUStat(sysPath, cgroupPath, "cpu.cfs_quota_us") + if (err != nil && !wantErr) || (err == nil && wantErr) { + t.Fatalf("unxpected error value: %v, want err: %v", err, wantErr) + } + if got != want { + t.Fatalf("unxpected result, got: %d, want %d", got, want) + } + } + f("testdata/", "testdata/self/cgroup", -1, false) + f("testdata/cgroup", "testdata/self/cgroup", 10, false) + f("testdata/", "testdata/missing_folder", 0, true) +} + +func TestGetCPUStatPeriod(t *testing.T) { + f := func(sysPath, cgroupPath string, want int64, wantErr bool) { + t.Helper() + got, err := getCPUStat(sysPath, cgroupPath, "cpu.cfs_period_us") + if (err != nil && !wantErr) || (err == nil && wantErr) { + t.Fatalf("unxpected error value: %v, want err: %v", err, wantErr) + } + if got != want { + t.Fatalf("unxpected result, got: %d, want %d", got, want) + } + } + f("testdata/", "testdata/self/cgroup", 100000, false) + f("testdata/cgroup", "testdata/self/cgroup", 500000, false) + f("testdata/", "testdata/missing_folder", 0, true) +} diff --git a/lib/cgroup/mem.go b/lib/cgroup/mem.go index 0141371a7..a756aa654 100644 --- a/lib/cgroup/mem.go +++ b/lib/cgroup/mem.go @@ -1,5 +1,10 @@ package cgroup +import ( + "path" + "strconv" +) + // GetMemoryLimit returns cgroup memory limit func GetMemoryLimit() int64 { // Try determining the amount of memory inside docker container. @@ -8,24 +13,60 @@ func GetMemoryLimit() int64 { // Read memory limit according to https://unix.stackexchange.com/questions/242718/how-to-find-out-how-much-memory-lxc-container-is-allowed-to-consume // This should properly determine the limit inside lxc container. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/84 - n, err := readInt64("/sys/fs/cgroup/memory/memory.limit_in_bytes", "cat /sys/fs/cgroup/memory$(cat /proc/self/cgroup | grep memory | cut -d: -f3)/memory.limit_in_bytes") + n, err := getMemLimit("/sys/fs/cgroup/", "/proc/self/cgroup") if err != nil { return 0 } return n } +func getMemLimit(sysPath, cgroupPath string) (int64, error) { + n, err := readInt64(path.Join(sysPath, "memory.limit_in_bytes")) + if err == nil { + return n, nil + } + subPath, err := grepFirstMatch(cgroupPath, "memory", 2, ":") + if err != nil { + return 0, err + } + return readInt64(path.Join(sysPath, subPath, "memory.limit_in_bytes")) +} + // GetHierarchicalMemoryLimit returns hierarchical memory limit +// https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt func GetHierarchicalMemoryLimit() int64 { // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/699 - n, err := readInt64FromCommand("cat /sys/fs/cgroup/memory/memory.stat | grep hierarchical_memory_limit | cut -d' ' -f 2") - if err == nil { - return n - } - n, err = readInt64FromCommand( - "cat /sys/fs/cgroup/memory$(cat /proc/self/cgroup | grep memory | cut -d: -f3)/memory.stat | grep hierarchical_memory_limit | cut -d' ' -f 2") + n, err := getHierarchicalMemoryLimit("/sys/fs/cgroup/memory", "/proc/self/cgroup") if err != nil { return 0 } return n } + +func getHierarchicalMemoryLimit(sysPath, cgroupPath string) (int64, error) { + n, err := getMemStatDirect(sysPath) + if err == nil { + return n, nil + } + return getMemStatSubPath(sysPath, cgroupPath) +} + +func getMemStatDirect(sysPath string) (int64, error) { + memStat, err := grepFirstMatch(path.Join(sysPath, "memory.stat"), "hierarchical_memory_limit", 1, " ") + if err != nil { + return 0, err + } + return strconv.ParseInt(memStat, 10, 64) +} + +func getMemStatSubPath(sysPath, cgroupPath string) (int64, error) { + cgrps, err := grepFirstMatch(cgroupPath, "memory", 2, ":") + if err != nil { + return 0, err + } + memStat, err := grepFirstMatch(path.Join(sysPath, cgrps, "memory.stat"), "hierarchical_memory_limit", 1, " ") + if err != nil { + return 0, err + } + return strconv.ParseInt(memStat, 10, 64) +} diff --git a/lib/cgroup/mem_test.go b/lib/cgroup/mem_test.go new file mode 100644 index 000000000..9a2254437 --- /dev/null +++ b/lib/cgroup/mem_test.go @@ -0,0 +1,35 @@ +package cgroup + +import "testing" + +func TestGetMemLimit(t *testing.T) { + f := func(sysPath, cgroupPath string, want int64, wantErr bool) { + t.Helper() + got, err := getMemLimit(sysPath, cgroupPath) + if (err != nil && !wantErr) || (err == nil && wantErr) { + t.Fatalf("unxpected error: %v, wantErr: %v", err, wantErr) + } + if got != want { + t.Fatalf("unxpected result, got: %d, want %d", got, want) + } + } + f("testdata/", "testdata/self/cgroup", 9223372036854771712, false) + f("testdata/cgroup", "testdata/self/cgroup", 523372036854771712, false) + f("testdata/", "testdata/none_existing_folder", 0, true) +} + +func TestGetMemHierarchical(t *testing.T) { + f := func(sysPath, cgroupPath string, want int64, wantErr bool) { + t.Helper() + got, err := getHierarchicalMemoryLimit(sysPath, cgroupPath) + if (err != nil && !wantErr) || (err == nil && wantErr) { + t.Fatalf("unxpected error: %v, wantErr: %v", err, wantErr) + } + if got != want { + t.Fatalf("unxpected result, got: %d, want %d", got, want) + } + } + f("testdata/", "testdata/self/cgroup", 16, false) + f("testdata/cgroup", "testdata/self/cgroup", 120, false) + f("testdata/", "testdata/none_existing_folder", 0, true) +} diff --git a/lib/cgroup/testdata/cgroup/cpu.cfs_period_us b/lib/cgroup/testdata/cgroup/cpu.cfs_period_us new file mode 100644 index 000000000..516a58aff --- /dev/null +++ b/lib/cgroup/testdata/cgroup/cpu.cfs_period_us @@ -0,0 +1 @@ +500000 \ No newline at end of file diff --git a/lib/cgroup/testdata/cgroup/cpu.cfs_quota_us b/lib/cgroup/testdata/cgroup/cpu.cfs_quota_us new file mode 100644 index 000000000..9a037142a --- /dev/null +++ b/lib/cgroup/testdata/cgroup/cpu.cfs_quota_us @@ -0,0 +1 @@ +10 \ No newline at end of file diff --git a/lib/cgroup/testdata/cgroup/memory.limit_in_bytes b/lib/cgroup/testdata/cgroup/memory.limit_in_bytes new file mode 100644 index 000000000..4443b4a18 --- /dev/null +++ b/lib/cgroup/testdata/cgroup/memory.limit_in_bytes @@ -0,0 +1 @@ +523372036854771712 \ No newline at end of file diff --git a/lib/cgroup/testdata/cgroup/memory.stat b/lib/cgroup/testdata/cgroup/memory.stat new file mode 100644 index 000000000..b5b83303a --- /dev/null +++ b/lib/cgroup/testdata/cgroup/memory.stat @@ -0,0 +1,31 @@ +rss 2 +rss_huge 3 +mapped_file 4 +dirty 5 +writeback 6 +pgpgin 7 +pgpgout 8 +pgfault 9 +pgmajfault 10 +inactive_anon 11 +active_anon 12 +inactive_file 13 +active_file 14 +unevictable 15 +hierarchical_memory_limit 120 +hierarchical_memsw_limit 17 +total_cache 18 +total_rss 19 +total_rss_huge 20 +total_mapped_file 21 +total_dirty 22 +total_writeback 23 +total_pgpgin 24 +total_pgpgout 25 +total_pgfault 26 +total_pgmajfault 27 +total_inactive_anon 28 +total_active_anon 29 +total_inactive_file 30 +total_active_file 31 +total_unevictable 32 \ No newline at end of file diff --git a/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/cpu.cfs_period_us b/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/cpu.cfs_period_us new file mode 100644 index 000000000..483fb82b6 --- /dev/null +++ b/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 \ No newline at end of file diff --git a/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/cpu.cfs_quota_us b/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/cpu.cfs_quota_us new file mode 100644 index 000000000..d7d17fcbe --- /dev/null +++ b/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/cpu.cfs_quota_us @@ -0,0 +1 @@ +-1 \ No newline at end of file diff --git a/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/memory.limit_in_bytes b/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/memory.limit_in_bytes new file mode 100644 index 000000000..20598cb02 --- /dev/null +++ b/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/memory.limit_in_bytes @@ -0,0 +1 @@ +9223372036854771712 \ No newline at end of file diff --git a/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/memory.stat b/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/memory.stat new file mode 100644 index 000000000..a936c7740 --- /dev/null +++ b/lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/memory.stat @@ -0,0 +1,31 @@ +rss 2 +rss_huge 3 +mapped_file 4 +dirty 5 +writeback 6 +pgpgin 7 +pgpgout 8 +pgfault 9 +pgmajfault 10 +inactive_anon 11 +active_anon 12 +inactive_file 13 +active_file 14 +unevictable 15 +hierarchical_memory_limit 16 +hierarchical_memsw_limit 17 +total_cache 18 +total_rss 19 +total_rss_huge 20 +total_mapped_file 21 +total_dirty 22 +total_writeback 23 +total_pgpgin 24 +total_pgpgout 25 +total_pgfault 26 +total_pgmajfault 27 +total_inactive_anon 28 +total_active_anon 29 +total_inactive_file 30 +total_active_file 31 +total_unevictable 32 \ No newline at end of file diff --git a/lib/cgroup/testdata/memory.stat b/lib/cgroup/testdata/memory.stat new file mode 100644 index 000000000..564113cfa --- /dev/null +++ b/lib/cgroup/testdata/memory.stat @@ -0,0 +1 @@ +9223372036854771712 diff --git a/lib/cgroup/testdata/self/cgroup b/lib/cgroup/testdata/self/cgroup new file mode 100644 index 000000000..e6f70e0eb --- /dev/null +++ b/lib/cgroup/testdata/self/cgroup @@ -0,0 +1,13 @@ +12:perf_event:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +11:rdma:/ +10:pids:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +9:freezer:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +8:memory:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +7:devices:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +6:cpuset:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +5:hugetlb:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +4:net_cls,net_prio:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +3:blkio:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +2:cpu,cpuacct:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +1:name=systemd:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db +0::/system.slice/containerd.service \ No newline at end of file diff --git a/lib/cgroup/util.go b/lib/cgroup/util.go index eb1ca8b04..8d49b783e 100644 --- a/lib/cgroup/util.go +++ b/lib/cgroup/util.go @@ -1,27 +1,42 @@ package cgroup import ( + "bufio" "bytes" + "fmt" "io/ioutil" - "os/exec" + "os" "strconv" + "strings" ) -func readInt64(path, altCommand string) (int64, error) { +// grepFirstMatch search match line at file and returns item from it by index with given delimiter. +func grepFirstMatch(sourcePath string, match string, index int, delimiter string) (string, error) { + f, err := os.Open(sourcePath) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + text := scanner.Text() + if !strings.Contains(text, match) { + continue + } + split := strings.Split(text, delimiter) + if len(split) < index { + return "", fmt.Errorf("needed index line: %d, wasn't found at line: %q at file: %q", index, text, sourcePath) + } + return strings.TrimSpace(split[index]), nil + } + return "", fmt.Errorf("stat: %q, wasn't found at file: %q", match, sourcePath) +} + +func readInt64(path string) (int64, error) { data, err := ioutil.ReadFile(path) if err == nil { data = bytes.TrimSpace(data) return strconv.ParseInt(string(data), 10, 64) } - return readInt64FromCommand(altCommand) -} - -func readInt64FromCommand(command string) (int64, error) { - cmd := exec.Command("/bin/sh", "-c", command) - data, err := cmd.Output() - if err != nil { - return 0, err - } - data = bytes.TrimSpace(data) - return strconv.ParseInt(string(data), 10, 64) + return 0, err }