refactored cgroups limits, (#1061)

adds tests, remove os.Exec

lib/cgroup/cpu.go

@@ -3,6 +3,7 @@ package cgroup
 import (
 	"io/ioutil"
 	"os"
+	"path"
 	"runtime"
 	"strconv"
 	"strings"
@@ -40,8 +41,20 @@ func updateGOMAXPROCSToCPUQuota() {
 	runtime.GOMAXPROCS(gomaxprocs)
 }
 
+func getCPUStat(sysPath, cgroupPath, statName string) (int64, error) {
+	n, err := readInt64(path.Join(sysPath, statName))
+	if err == nil {
+		return n, nil
+	}
+	subPath, err := grepFirstMatch(cgroupPath, "cpu,", 2, ":")
+	if err != nil {
+		return 0, err
+	}
+	return readInt64(path.Join(sysPath, subPath, statName))
+}
+
 func getCPUQuota() float64 {
-	quotaUS, err := readInt64("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "cat /sys/fs/cgroup/cpu$(cat /proc/self/cgroup | grep cpu, | cut -d: -f3)/cpu.cfs_quota_us")
+	quotaUS, err := getCPUStat("/sys/fs/cgroup/cpu", "/proc/self/cgroup", "cpu.cfs_quota_us")
 	if err != nil {
 		return 0
 	}
@@ -50,7 +63,7 @@ func getCPUQuota() float64 {
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/685#issuecomment-674423728
 		return getOnlineCPUCount()
 	}
-	periodUS, err := readInt64("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "cat /sys/fs/cgroup/cpu$(cat /proc/self/cgroup | grep cpu, | cut -d: -f3)/cpu.cfs_period_us")
+	periodUS, err := getCPUStat("/sys/fs/cgroup/cpu", "/proc/self/cgroup", "cpu.cfs_period_us")
 	if err != nil {
 		return 0
 	}

lib/cgroup/cpu_test.go

@@ -22,3 +22,35 @@ func TestCountCPUs(t *testing.T) {
 	f("0-3", 4)
 	f("0-6", 7)
 }
+
+func TestGetCPUStatQuota(t *testing.T) {
+	f := func(sysPath, cgroupPath string, want int64, wantErr bool) {
+		t.Helper()
+		got, err := getCPUStat(sysPath, cgroupPath, "cpu.cfs_quota_us")
+		if (err != nil && !wantErr) || (err == nil && wantErr) {
+			t.Fatalf("unxpected error value: %v, want err: %v", err, wantErr)
+		}
+		if got != want {
+			t.Fatalf("unxpected result, got: %d, want %d", got, want)
+		}
+	}
+	f("testdata/", "testdata/self/cgroup", -1, false)
+	f("testdata/cgroup", "testdata/self/cgroup", 10, false)
+	f("testdata/", "testdata/missing_folder", 0, true)
+}
+
+func TestGetCPUStatPeriod(t *testing.T) {
+	f := func(sysPath, cgroupPath string, want int64, wantErr bool) {
+		t.Helper()
+		got, err := getCPUStat(sysPath, cgroupPath, "cpu.cfs_period_us")
+		if (err != nil && !wantErr) || (err == nil && wantErr) {
+			t.Fatalf("unxpected error value: %v, want err: %v", err, wantErr)
+		}
+		if got != want {
+			t.Fatalf("unxpected result, got: %d, want %d", got, want)
+		}
+	}
+	f("testdata/", "testdata/self/cgroup", 100000, false)
+	f("testdata/cgroup", "testdata/self/cgroup", 500000, false)
+	f("testdata/", "testdata/missing_folder", 0, true)
+}

lib/cgroup/mem.go

@@ -1,5 +1,10 @@
 package cgroup
 
+import (
+	"path"
+	"strconv"
+)
+
 // GetMemoryLimit returns cgroup memory limit
 func GetMemoryLimit() int64 {
 	// Try determining the amount of memory inside docker container.
@@ -8,24 +13,60 @@ func GetMemoryLimit() int64 {
 	// Read memory limit according to https://unix.stackexchange.com/questions/242718/how-to-find-out-how-much-memory-lxc-container-is-allowed-to-consume
 	// This should properly determine the limit inside lxc container.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/84
-	n, err := readInt64("/sys/fs/cgroup/memory/memory.limit_in_bytes", "cat /sys/fs/cgroup/memory$(cat /proc/self/cgroup | grep memory | cut -d: -f3)/memory.limit_in_bytes")
+	n, err := getMemLimit("/sys/fs/cgroup/", "/proc/self/cgroup")
 	if err != nil {
 		return 0
 	}
 	return n
 }
 
+func getMemLimit(sysPath, cgroupPath string) (int64, error) {
+	n, err := readInt64(path.Join(sysPath, "memory.limit_in_bytes"))
+	if err == nil {
+		return n, nil
+	}
+	subPath, err := grepFirstMatch(cgroupPath, "memory", 2, ":")
+	if err != nil {
+		return 0, err
+	}
+	return readInt64(path.Join(sysPath, subPath, "memory.limit_in_bytes"))
+}
+
 // GetHierarchicalMemoryLimit returns hierarchical memory limit
+// https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
 func GetHierarchicalMemoryLimit() int64 {
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/699
-	n, err := readInt64FromCommand("cat /sys/fs/cgroup/memory/memory.stat | grep hierarchical_memory_limit | cut -d' ' -f 2")
+	n, err := getHierarchicalMemoryLimit("/sys/fs/cgroup/memory", "/proc/self/cgroup")
-	if err == nil {
-		return n
-	}
-	n, err = readInt64FromCommand(
-		"cat /sys/fs/cgroup/memory$(cat /proc/self/cgroup | grep memory | cut -d: -f3)/memory.stat | grep hierarchical_memory_limit | cut -d' ' -f 2")
 	if err != nil {
 		return 0
 	}
 	return n
 }
+
+func getHierarchicalMemoryLimit(sysPath, cgroupPath string) (int64, error) {
+	n, err := getMemStatDirect(sysPath)
+	if err == nil {
+		return n, nil
+	}
+	return getMemStatSubPath(sysPath, cgroupPath)
+}
+
+func getMemStatDirect(sysPath string) (int64, error) {
+	memStat, err := grepFirstMatch(path.Join(sysPath, "memory.stat"), "hierarchical_memory_limit", 1, " ")
+	if err != nil {
+		return 0, err
+	}
+	return strconv.ParseInt(memStat, 10, 64)
+}
+
+func getMemStatSubPath(sysPath, cgroupPath string) (int64, error) {
+	cgrps, err := grepFirstMatch(cgroupPath, "memory", 2, ":")
+	if err != nil {
+		return 0, err
+	}
+	memStat, err := grepFirstMatch(path.Join(sysPath, cgrps, "memory.stat"), "hierarchical_memory_limit", 1, " ")
+	if err != nil {
+		return 0, err
+	}
+	return strconv.ParseInt(memStat, 10, 64)
+}

lib/cgroup/mem_test.go

@@ -0,0 +1,35 @@
+package cgroup
+
+import "testing"
+
+func TestGetMemLimit(t *testing.T) {
+	f := func(sysPath, cgroupPath string, want int64, wantErr bool) {
+		t.Helper()
+		got, err := getMemLimit(sysPath, cgroupPath)
+		if (err != nil && !wantErr) || (err == nil && wantErr) {
+			t.Fatalf("unxpected error: %v, wantErr: %v", err, wantErr)
+		}
+		if got != want {
+			t.Fatalf("unxpected result, got: %d, want %d", got, want)
+		}
+	}
+	f("testdata/", "testdata/self/cgroup", 9223372036854771712, false)
+	f("testdata/cgroup", "testdata/self/cgroup", 523372036854771712, false)
+	f("testdata/", "testdata/none_existing_folder", 0, true)
+}
+
+func TestGetMemHierarchical(t *testing.T) {
+	f := func(sysPath, cgroupPath string, want int64, wantErr bool) {
+		t.Helper()
+		got, err := getHierarchicalMemoryLimit(sysPath, cgroupPath)
+		if (err != nil && !wantErr) || (err == nil && wantErr) {
+			t.Fatalf("unxpected error: %v, wantErr: %v", err, wantErr)
+		}
+		if got != want {
+			t.Fatalf("unxpected result, got: %d, want %d", got, want)
+		}
+	}
+	f("testdata/", "testdata/self/cgroup", 16, false)
+	f("testdata/cgroup", "testdata/self/cgroup", 120, false)
+	f("testdata/", "testdata/none_existing_folder", 0, true)
+}

lib/cgroup/testdata/cgroup/cpu.cfs_period_us

@@ -0,0 +1 @@
+500000

lib/cgroup/testdata/cgroup/cpu.cfs_quota_us

@@ -0,0 +1 @@
+10

lib/cgroup/testdata/cgroup/memory.limit_in_bytes

@@ -0,0 +1 @@
+523372036854771712

lib/cgroup/testdata/cgroup/memory.stat

@@ -0,0 +1,31 @@
+rss 2
+rss_huge 3
+mapped_file 4
+dirty 5
+writeback 6
+pgpgin 7
+pgpgout 8
+pgfault 9
+pgmajfault 10
+inactive_anon 11
+active_anon 12
+inactive_file 13
+active_file 14
+unevictable 15
+hierarchical_memory_limit 120
+hierarchical_memsw_limit 17
+total_cache 18
+total_rss 19
+total_rss_huge 20
+total_mapped_file 21
+total_dirty 22
+total_writeback 23
+total_pgpgin 24
+total_pgpgout 25
+total_pgfault 26
+total_pgmajfault 27
+total_inactive_anon 28
+total_active_anon 29
+total_inactive_file 30
+total_active_file 31
+total_unevictable 32

lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/cpu.cfs_period_us

@@ -0,0 +1 @@
+100000

lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/cpu.cfs_quota_us

@@ -0,0 +1 @@
+-1

lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/memory.limit_in_bytes

@@ -0,0 +1 @@
+9223372036854771712

lib/cgroup/testdata/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db/memory.stat

@@ -0,0 +1,31 @@
+rss 2
+rss_huge 3
+mapped_file 4
+dirty 5
+writeback 6
+pgpgin 7
+pgpgout 8
+pgfault 9
+pgmajfault 10
+inactive_anon 11
+active_anon 12
+inactive_file 13
+active_file 14
+unevictable 15
+hierarchical_memory_limit 16
+hierarchical_memsw_limit 17
+total_cache 18
+total_rss 19
+total_rss_huge 20
+total_mapped_file 21
+total_dirty 22
+total_writeback 23
+total_pgpgin 24
+total_pgpgout 25
+total_pgfault 26
+total_pgmajfault 27
+total_inactive_anon 28
+total_active_anon 29
+total_inactive_file 30
+total_active_file 31
+total_unevictable 32

lib/cgroup/testdata/memory.stat

@@ -0,0 +1 @@
+9223372036854771712

lib/cgroup/testdata/self/cgroup

@@ -0,0 +1,13 @@
+12:perf_event:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+11:rdma:/
+10:pids:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+9:freezer:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+8:memory:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+7:devices:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+6:cpuset:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+5:hugetlb:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+4:net_cls,net_prio:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+3:blkio:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+2:cpu,cpuacct:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+1:name=systemd:/docker/74c9abf42b88b9a35b1b56061b08303e56fd1707fe5c5b4df93324dedb36b5db
+0::/system.slice/containerd.service

lib/cgroup/util.go

@@ -1,27 +1,42 @@
 package cgroup
 
 import (
+	"bufio"
 	"bytes"
+	"fmt"
 	"io/ioutil"
-	"os/exec"
+	"os"
 	"strconv"
+	"strings"
 )
 
-func readInt64(path, altCommand string) (int64, error) {
+// grepFirstMatch search match line at file and returns item from it by index with given delimiter.
+func grepFirstMatch(sourcePath string, match string, index int, delimiter string) (string, error) {
+	f, err := os.Open(sourcePath)
+	if err != nil {
+		return "", err
+	}
+	defer func() { _ = f.Close() }()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if !strings.Contains(text, match) {
+			continue
+		}
+		split := strings.Split(text, delimiter)
+		if len(split) < index {
+			return "", fmt.Errorf("needed index line: %d, wasn't found at line: %q at file: %q", index, text, sourcePath)
+		}
+		return strings.TrimSpace(split[index]), nil
+	}
+	return "", fmt.Errorf("stat: %q, wasn't found at file: %q", match, sourcePath)
+}
+
+func readInt64(path string) (int64, error) {
 	data, err := ioutil.ReadFile(path)
 	if err == nil {
 		data = bytes.TrimSpace(data)
 		return strconv.ParseInt(string(data), 10, 64)
 	}
-	return readInt64FromCommand(altCommand)
+	return 0, err
-}
-
-func readInt64FromCommand(command string) (int64, error) {
-	cmd := exec.Command("/bin/sh", "-c", command)
-	data, err := cmd.Output()
-	if err != nil {
-		return 0, err
-	}
-	data = bytes.TrimSpace(data)
-	return strconv.ParseInt(string(data), 10, 64)
 }