mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
5034aa0773
- Add Try* prefix to functions, which return bool result in order to improve readability and reduce the probability of missing check for the result returned from these functions. - Call the adjustSampleValues() only once on input samples. Previously it was called on every attempt to flush data to peristent queue. - Properly restore the initial state of WriteRequest passed to tryPushWriteRequest() before returning from this function after unsuccessful push to persistent queue. Previously a part of WriteRequest samples may be lost in such case. - Add -remoteWrite.dropSamplesOnOverload command-line flag, which can be used for dropping incoming samples instead of returning 429 Too Many Requests error to the client when -remoteWrite.disableOnDiskQueue is set and the remote storage cannot keep up with the data ingestion rate. - Add vmagent_remotewrite_samples_dropped_total metric, which counts the number of dropped samples. - Add vmagent_remotewrite_push_failures_total metric, which counts the number of unsuccessful attempts to push data to persistent queue when -remoteWrite.disableOnDiskQueue is set. - Remove vmagent_remotewrite_aggregation_metrics_dropped_total and vm_promscrape_push_samples_dropped_total metrics, because they are replaced with vmagent_remotewrite_samples_dropped_total metric. - Update 'Disabling on-disk persistence' docs at docs/vmagent.md - Update stale comments in the code Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5088 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2110
73 lines
2.3 KiB
Go
73 lines
2.3 KiB
Go
package remotewrite
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"testing"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
|
)
|
|
|
|
func TestPushWriteRequest(t *testing.T) {
|
|
rowsCounts := []int{1, 10, 100, 1e3, 1e4}
|
|
expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
|
|
expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
|
|
for i, rowsCount := range rowsCounts {
|
|
expectedBlockLenProm := expectedBlockLensProm[i]
|
|
expectedBlockLenVM := expectedBlockLensVM[i]
|
|
t.Run(fmt.Sprintf("%d", rowsCount), func(t *testing.T) {
|
|
testPushWriteRequest(t, rowsCount, expectedBlockLenProm, expectedBlockLenVM)
|
|
})
|
|
}
|
|
}
|
|
|
|
func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expectedBlockLenVM int) {
|
|
f := func(isVMRemoteWrite bool, expectedBlockLen int, tolerancePrc float64) {
|
|
t.Helper()
|
|
wr := newTestWriteRequest(rowsCount, 20)
|
|
pushBlockLen := 0
|
|
pushBlock := func(block []byte) bool {
|
|
if pushBlockLen > 0 {
|
|
panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
|
|
}
|
|
pushBlockLen = len(block)
|
|
return true
|
|
}
|
|
if !tryPushWriteRequest(wr, pushBlock, isVMRemoteWrite) {
|
|
t.Fatalf("cannot push data to to remote storage")
|
|
}
|
|
if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
|
|
t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
|
|
rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
|
|
}
|
|
}
|
|
|
|
// Check Prometheus remote write
|
|
f(false, expectedBlockLenProm, 0)
|
|
|
|
// Check VictoriaMetrics remote write
|
|
f(true, expectedBlockLenVM, 15)
|
|
}
|
|
|
|
func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteRequest {
|
|
var wr prompbmarshal.WriteRequest
|
|
for i := 0; i < seriesCount; i++ {
|
|
var labels []prompbmarshal.Label
|
|
for j := 0; j < labelsCount; j++ {
|
|
labels = append(labels, prompbmarshal.Label{
|
|
Name: fmt.Sprintf("label_%d_%d", i, j),
|
|
Value: fmt.Sprintf("value_%d_%d", i, j),
|
|
})
|
|
}
|
|
wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
|
|
Labels: labels,
|
|
Samples: []prompbmarshal.Sample{
|
|
{
|
|
Value: float64(i),
|
|
Timestamp: 1000 * int64(i),
|
|
},
|
|
},
|
|
})
|
|
}
|
|
return &wr
|
|
}
|