mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
8baa5177aa
This msy be useful when ingesting logs from different sources, which store the log message in different fields.
For example, `_msg_field=message,event.data,some_field` will get log message from the first non-empty field:
`message`, `event.data` and `some_field`.
(cherry picked from commit ed73f8350b)
50 lines
1.3 KiB
Go
50 lines
1.3 KiB
Go
package elasticsearch
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
|
)
|
|
|
|
func BenchmarkReadBulkRequest(b *testing.B) {
|
|
b.Run("gzip:off", func(b *testing.B) {
|
|
benchmarkReadBulkRequest(b, false)
|
|
})
|
|
b.Run("gzip:on", func(b *testing.B) {
|
|
benchmarkReadBulkRequest(b, true)
|
|
})
|
|
}
|
|
|
|
func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
|
|
data := `{"create":{"_index":"filebeat-8.8.0"}}
|
|
{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
|
|
{"create":{"_index":"filebeat-8.8.0"}}
|
|
{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
|
|
{"create":{"_index":"filebeat-8.8.0"}}
|
|
{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
|
|
`
|
|
if isGzip {
|
|
data = compressData(data)
|
|
}
|
|
dataBytes := bytesutil.ToUnsafeBytes(data)
|
|
|
|
timeField := "@timestamp"
|
|
msgFields := []string{"message"}
|
|
blp := &insertutils.BenchmarkLogMessageProcessor{}
|
|
|
|
b.ReportAllocs()
|
|
b.SetBytes(int64(len(data)))
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
r := &bytes.Reader{}
|
|
for pb.Next() {
|
|
r.Reset(dataBytes)
|
|
_, err := readBulkRequest(r, isGzip, timeField, msgFields, blp)
|
|
if err != nil {
|
|
panic(fmt.Errorf("unexpected error: %w", err))
|
|
}
|
|
}
|
|
})
|
|
}
|