github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/{mergeset,storage}: perform at most one assisted merge per each call to addRows/addItems

Browse source 
This should reduce tail latency during data ingestion.

This shouldn't slow down data ingestion in the worst case, since assisted merges are spread among
distinct addRows/addItems calls after this change.
This commit is contained in:
Aliaksandr Valialkin 2023-10-01 22:17:38 +02:00
parent 7373d04d54
commit 3ca6fea858
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
2 changed files with 60 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
lib/mergeset/table.go
View file

@ -773,45 +773,41 @@ func needAssistedMerge(pws []*partWrapper, maxParts int) bool {
}
func (tb *Table) assistedMergeForInmemoryParts() {
for {
tb.partsLock.Lock()
needMerge := needAssistedMerge(tb.inmemoryParts, maxInmemoryParts)
tb.partsLock.Unlock()
if !needMerge {
return
}
atomic.AddUint64(&tb.inmemoryAssistedMerges, 1)
err := tb.mergeInmemoryParts()
if err == nil {
continue
}
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) {
return
}
logger.Panicf("FATAL: cannot assist with merging inmemory parts: %s", err)
tb.partsLock.Lock()
needMerge := needAssistedMerge(tb.inmemoryParts, maxInmemoryParts)
tb.partsLock.Unlock()
if !needMerge {
return
}
atomic.AddUint64(&tb.inmemoryAssistedMerges, 1)
err := tb.mergeInmemoryParts()
if err == nil {
return
}
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) {
return
}
logger.Panicf("FATAL: cannot assist with merging inmemory parts: %s", err)
}
func (tb *Table) assistedMergeForFileParts() {
for {
tb.partsLock.Lock()
needMerge := needAssistedMerge(tb.fileParts, maxFileParts)
tb.partsLock.Unlock()
if !needMerge {
return
}
atomic.AddUint64(&tb.fileAssistedMerges, 1)
err := tb.mergeExistingParts(false)
if err == nil {
continue
}
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) {
return
}
logger.Panicf("FATAL: cannot assist with merging file parts: %s", err)
tb.partsLock.Lock()
needMerge := needAssistedMerge(tb.fileParts, maxFileParts)
tb.partsLock.Unlock()
if !needMerge {
return
}
atomic.AddUint64(&tb.fileAssistedMerges, 1)
err := tb.mergeExistingParts(false)
if err == nil {
return
}
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) {
return
}
logger.Panicf("FATAL: cannot assist with merging file parts: %s", err)
}
func getNotInMergePartsCount(pws []*partWrapper) int {

64
lib/storage/partition.go
View file

@ -634,45 +634,41 @@ func needAssistedMerge(pws []*partWrapper, maxParts int) bool {
}
func (pt *partition) assistedMergeForInmemoryParts() {
for {
pt.partsLock.Lock()
needMerge := needAssistedMerge(pt.inmemoryParts, maxInmemoryPartsPerPartition)
pt.partsLock.Unlock()
if !needMerge {
return
}
atomic.AddUint64(&pt.inmemoryAssistedMerges, 1)
err := pt.mergeInmemoryParts()
if err == nil {
continue
}
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) {
return
}
logger.Panicf("FATAL: cannot merge inmemory parts: %s", err)
pt.partsLock.Lock()
needMerge := needAssistedMerge(pt.inmemoryParts, maxInmemoryPartsPerPartition)
pt.partsLock.Unlock()
if !needMerge {
return
}
atomic.AddUint64(&pt.inmemoryAssistedMerges, 1)
err := pt.mergeInmemoryParts()
if err == nil {
return
}
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) {
return
}
logger.Panicf("FATAL: cannot merge inmemory parts: %s", err)
}
func (pt *partition) assistedMergeForSmallParts() {
for {
pt.partsLock.Lock()
needMerge := needAssistedMerge(pt.smallParts, maxSmallPartsPerPartition)
pt.partsLock.Unlock()
if !needMerge {
return
}
atomic.AddUint64(&pt.smallAssistedMerges, 1)
err := pt.mergeExistingParts(false)
if err == nil {
continue
}
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) {
return
}
logger.Panicf("FATAL: cannot merge small parts: %s", err)
pt.partsLock.Lock()
needMerge := needAssistedMerge(pt.smallParts, maxSmallPartsPerPartition)
pt.partsLock.Unlock()
if !needMerge {
return
}
atomic.AddUint64(&pt.smallAssistedMerges, 1)
err := pt.mergeExistingParts(false)
if err == nil {
return
}
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) {
return
}
logger.Panicf("FATAL: cannot merge small parts: %s", err)
}
func getNotInMergePartsCount(pws []*partWrapper) int {