github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/mergeset: do not panic on too long items passed to Table.AddItems()

Browse source 
Instead, log a sample of these long items once per 5 seconds into error log,
so users could notice and fix the issue with too long labels or too many labels.

Previously this panic could occur in production when ingesting samples with too long labels.
This commit is contained in:
Aliaksandr Valialkin 2024-02-12 19:32:16 +02:00
parent 397bb8771b
commit 99aaa5067f
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
3 changed files with 15 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
lib/mergeset/table.go
View file

@ -244,8 +244,13 @@ func (ris *rawItemsShard) addItems(tb *Table, items [][]byte) [][]byte {
ibs = append(ibs, ib)
continue
}
ris.mu.Unlock()
logger.Panicf("BUG: cannot insert too big item into an empty inmemoryBlock; len(item)=%d; the caller should be responsible for avoiding too big items", len(item))
// Skip too long item
itemPrefix := item
if len(itemPrefix) > 128 {
itemPrefix = itemPrefix[:128]
}
tooLongItemLogger.Errorf("skipping adding too long item to indexdb: len(item)=%d; it souldn't exceed %d bytes; item prefix=%q", len(item), maxInmemoryBlockSize, itemPrefix)
}
ris.ibs = ibs
ris.mu.Unlock()
@ -255,6 +260,8 @@ func (ris *rawItemsShard) addItems(tb *Table, items [][]byte) [][]byte {
return tailItems
}
var tooLongItemLogger = logger.WithThrottler("tooLongItem", 5*time.Second)
type partWrapper struct {
p *part
@ -600,8 +607,8 @@ func (tb *Table) UpdateMetrics(m *TableMetrics) {
// AddItems adds the given items to the tb.
//
// The function panics when items contains an item with length exceeding maxInmemoryBlockSize.
// It is caller's responsibility to make sure there are no too long items.
// The function ignores items with length exceeding maxInmemoryBlockSize.
// It logs the ignored items, so users could notice and fix the issue.
func (tb *Table) AddItems(items [][]byte) {
tb.rawItems.addItems(tb, items)
atomic.AddUint64(&tb.itemsAdded, uint64(len(items)))

10
lib/mergeset/table_test.go
View file

@ -41,15 +41,7 @@ func TestTableAddItemsTooLongItem(t *testing.T) {
var isReadOnly uint32
tb := MustOpenTable(path, nil, nil, &isReadOnly)
func() {
defer func() {
if r := recover(); r == nil {
t.Fatalf("expecting panic")
}
}()
tb.AddItems([][]byte{make([]byte, maxInmemoryBlockSize+1)})
}()
t.Logf("foobar")
tb.AddItems([][]byte{make([]byte, maxInmemoryBlockSize+1)})
tb.MustClose()
_ = os.RemoveAll(path)
}

6
lib/storage/metric_name.go
View file

@ -651,9 +651,9 @@ func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label, accoun
case <-truncatedLabelsLogTicker.C:
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
// because labelsToString() will be called with each trackTruncatedLabels call.
logger.Warnf("truncated label value as it exceeds configured maximal label value length: max %d, actual %d;"+
" truncated label: %s; original labels: %s; tenant: %d:%d; either reduce the label value length or increase -maxLabelValueLen=%d;",
maxLabelValueLen, len(truncated.Value), truncated.Name, labelsToString(labels), accountID, projectID, maxLabelValueLen)
logger.Warnf("truncate value for label %s because its length=%d exceeds -maxLabelValueLen=%d; "+
"original labels: %s; tenant: %d:%d; either reduce the label value length or increase -maxLabelValueLen command-line flag value",
truncated.Name, len(truncated.Value), maxLabelValueLen, labelsToString(labels), accountID, projectID)
default:
}
}