lib/storage: calculate the maximum number of rows per small part from -memory.allowedPercent

This should improve query speed over recent data on machines with big amounts of RAM

lib/memory/memory.go

@@ -10,15 +10,14 @@ import (
 
 var allowedMemPercent = flag.Float64("memory.allowedPercent", 60, "Allowed percent of system memory VictoriaMetrics caches may occupy")
 
-var allowedMemory int
+var (
+	allowedMemory   int
+	remainingMemory int
+)
 
 var once sync.Once
 
-// Allowed returns the amount of system memory allowed to use by the app.
+func initOnce() {
-//
-// The function must be called only after flag.Parse is called.
-func Allowed() int {
-	once.Do(func() {
 	if !flag.Parsed() {
 		// Do not use logger.Panicf here, since logger may be uninitialized yet.
 		panic(fmt.Errorf("BUG: memory.Allowed must be called only after flag.Parse call"))
@@ -30,7 +29,22 @@ func Allowed() int {
 
 	mem := sysTotalMemory()
 	allowedMemory = int(float64(mem) * percent)
-		logger.Infof("limiting caches to %d bytes of RAM according to -memory.allowedPercent=%g", allowedMemory, *allowedMemPercent)
+	remainingMemory = mem - allowedMemory
-	})
+	logger.Infof("limiting caches to %d bytes, leaving %d bytes to the OS according to -memory.allowedPercent=%g", allowedMemory, remainingMemory, *allowedMemPercent)
+}
+
+// Allowed returns the amount of system memory allowed to use by the app.
+//
+// The function must be called only after flag.Parse is called.
+func Allowed() int {
+	once.Do(initOnce)
 	return allowedMemory
 }
+
+// Remaining returns the amount of memory remaining to the OS.
+//
+// This function must be called only after flag.Parse is called.
+func Remaining() int {
+	once.Do(initOnce)
+	return remainingMemory
+}

lib/storage/part.go

@@ -144,7 +144,7 @@ func (p *part) MustClose() {
 	p.valuesFile.MustClose()
 	p.indexFile.MustClose()
 
-	isBig := p.ph.RowsCount > maxRowsPerSmallPart
+	isBig := p.ph.RowsCount > maxRowsPerSmallPart()
 	p.ibCache.Reset(isBig)
 }
 

lib/storage/partition.go

@@ -22,15 +22,16 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-// The maximum number of rows in a small part.
+func maxRowsPerSmallPart() uint64 {
-//
+	// Small parts are cached in the OS page cache,
-// This number limits the maximum size of small parts storage.
+	// so limit the number of rows for small part
-// Production simultation shows that the required size of the storage
+	// by the remaining free RAM.
-// may be estimated as:
+	mem := memory.Remaining()
-//
+	if mem <= 0 {
-//     maxRowsPerSmallPart * 2 * defaultPartsToMerge * mergeWorkers
+		return 100e6
-//
+	}
-const maxRowsPerSmallPart = 300e6
+	return uint64(mem) / defaultPartsToMerge
+}
 
 // The maximum number of rows per big part.
 //
@@ -885,7 +886,7 @@ func (pt *partition) mergeBigParts(isFinal bool) error {
 
 func (pt *partition) mergeSmallParts(isFinal bool) error {
 	maxRows := maxRowsByPath(pt.smallPartsPath)
-	if maxRows > maxRowsPerSmallPart {
+	if maxRows > maxRowsPerSmallPart() {
 		// The output part may go to big part,
 		// so make sure it as enough space.
 		maxBigPartRows := maxRowsByPath(pt.bigPartsPath)
@@ -955,7 +956,7 @@ func (pt *partition) mergeParts(pws []*partWrapper, stopCh <-chan struct{}) erro
 	for _, pw := range pws {
 		outRowsCount += pw.p.ph.RowsCount
 	}
-	isBigPart := outRowsCount > maxRowsPerSmallPart
+	isBigPart := outRowsCount > maxRowsPerSmallPart()
 	nocache := isBigPart
 
 	// Prepare BlockStreamWriter for destination part.