github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-31 15:06:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-05-10 15:12:19 +02:00
parent 9767a52ed0
commit 86942cb46c
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
3 changed files with 12 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/logstorage/consts.go
View file

@ -14,7 +14,7 @@ const maxUncompressedBlockSize = 2 * 1024 * 1024
const maxRowsPerBlock = 8 * 1024 * 1024 const maxRowsPerBlock = 8 * 1024 * 1024
// maxColumnsPerBlock is the maximum number of columns per block. // maxColumnsPerBlock is the maximum number of columns per block.
const maxColumnsPerBlock = 10000 const maxColumnsPerBlock = 2_000
// MaxFieldNameSize is the maximum size in bytes for field name. // MaxFieldNameSize is the maximum size in bytes for field name.
// //

13
lib/logstorage/datadb.go
View file

@ -23,6 +23,12 @@ import (
// This time shouldn't exceed a few days. // This time shouldn't exceed a few days.
const maxBigPartSize = 1e12 const maxBigPartSize = 1e12
// The maximum number of inmemory parts in the partition.
//
// The actual number of inmemory parts may exceed this value if in-memory mergers
// cannot keep up with the rate of creating new in-memory parts.
const maxInmemoryPartsPerPartition = 20
// The interval for guaranteed flush of recently ingested data from memory to on-disk parts, // The interval for guaranteed flush of recently ingested data from memory to on-disk parts,
// so they survive process crash. // so they survive process crash.
var dataFlushInterval = 5 * time.Second var dataFlushInterval = 5 * time.Second
@ -41,11 +47,6 @@ const defaultPartsToMerge = 15
// The 1.7 is good enough for production workloads. // The 1.7 is good enough for production workloads.
const minMergeMultiplier = 1.7 const minMergeMultiplier = 1.7
// The maximum number of inmemory parts in the partition.
//
// If the number of inmemory parts reaches this value, then assisted merge runs during data ingestion.
const maxInmemoryPartsPerPartition = 20
// datadb represents a database with log data // datadb represents a database with log data
type datadb struct { type datadb struct {
// mergeIdx is used for generating unique directory names for parts // mergeIdx is used for generating unique directory names for parts
@ -663,9 +664,11 @@ func (ddb *datadb) mustAddRows(lr *LogRows) {
return return
} }
inmemoryPartsConcurrencyCh <- struct{}{}
mp := getInmemoryPart() mp := getInmemoryPart()
mp.mustInitFromRows(lr) mp.mustInitFromRows(lr)
p := mustOpenInmemoryPart(ddb.pt, mp) p := mustOpenInmemoryPart(ddb.pt, mp)
<-inmemoryPartsConcurrencyCh
flushDeadline := time.Now().Add(ddb.flushInterval) flushDeadline := time.Now().Add(ddb.flushInterval)
pw := newPartWrapper(p, mp, flushDeadline) pw := newPartWrapper(p, mp, flushDeadline)

10
lib/storage/partition.go
View file

@ -28,14 +28,10 @@ import (
// This time shouldn't exceed a few days. // This time shouldn't exceed a few days.
const maxBigPartSize = 1e12 const maxBigPartSize = 1e12
// The maximum number of inmemory parts per partition. // The maximum expected number of inmemory parts per partition.
// //
// This limit allows reducing querying CPU usage under high ingestion rate. // The actual number of inmemory parts may exceed this value if in-memory mergers
// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5212 // cannot keep up with the rate of creating new in-memory parts.
//
// This number may be reached when the insertion pace outreaches merger pace.
// If this number is reached, then the data ingestion is paused until background
// mergers reduce the number of parts below this number.
const maxInmemoryParts = 60 const maxInmemoryParts = 60
// Default number of parts to merge at once. // Default number of parts to merge at once.