mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2025-03-21 15:45:01 +00:00
0451a1c9e0
- Properly decode protobuf-encoded Loki request if it has no Content-Encoding header. Protobuf Loki message is snappy-encoded by default, so snappy decoding must be used when Content-Encoding header is missing. - Return back the previous signatures of parseJSONRequest and parseProtobufRequest functions. This eliminates the churn in tests for these functions. This also fixes broken benchmarks BenchmarkParseJSONRequest and BenchmarkParseProtobufRequest, which consume the whole request body on the first iteration and do nothing on subsequent iterations. - Put the CHANGELOG entries into correct places, since they were incorrectly put into already released versions of VictoriaMetrics and VictoriaLogs. - Add support for reading zstd-compressed data ingestion requests into the remaining protocols at VictoriaLogs and VictoriaMetrics. - Remove the `encoding` arg from PutUncompressedReader() - it has enough information about the passed reader arg in order to properly deal with it. - Add ReadUncompressedData to lib/protoparser/common for reading uncompressed data from the reader until EOF. This allows removing repeated code across request-based protocol parsers without streaming mode. - Consistently limit data ingestion request sizes, which can be read by ReadUncompressedData function. Previously this wasn't the case for all the supported protocols. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8416 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8380 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8300
106 lines
2.3 KiB
Go
106 lines
2.3 KiB
Go
package stream
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"fmt"
|
|
"reflect"
|
|
"testing"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
|
|
)
|
|
|
|
func TestParseFailure(t *testing.T) {
|
|
f := func(req string) {
|
|
t.Helper()
|
|
|
|
callback := func(_ []newrelic.Row) error {
|
|
panic(fmt.Errorf("unexpected call into callback"))
|
|
}
|
|
r := bytes.NewReader([]byte(req))
|
|
if err := Parse(r, "", callback); err == nil {
|
|
t.Fatalf("expecting non-empty error")
|
|
}
|
|
}
|
|
f("")
|
|
f("foo")
|
|
f("{}")
|
|
f("[1,2,3]")
|
|
}
|
|
|
|
func TestParseSuccess(t *testing.T) {
|
|
f := func(req string, expectedRows []newrelic.Row) {
|
|
t.Helper()
|
|
|
|
callback := func(rows []newrelic.Row) error {
|
|
if !reflect.DeepEqual(rows, expectedRows) {
|
|
return fmt.Errorf("unexpected rows\ngot\n%v\nwant\n%v", rows, expectedRows)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Parse from uncompressed reader
|
|
r := bytes.NewReader([]byte(req))
|
|
if err := Parse(r, "", callback); err != nil {
|
|
t.Fatalf("unexpected error when parsing uncompressed request: %s", err)
|
|
}
|
|
|
|
var bb bytes.Buffer
|
|
zw := gzip.NewWriter(&bb)
|
|
if _, err := zw.Write([]byte(req)); err != nil {
|
|
t.Fatalf("cannot compress request: %s", err)
|
|
}
|
|
if err := zw.Close(); err != nil {
|
|
t.Fatalf("cannot close compressed writer: %s", err)
|
|
}
|
|
if err := Parse(&bb, "gzip", callback); err != nil {
|
|
t.Fatalf("unexpected error when parsing compressed request: %s", err)
|
|
}
|
|
}
|
|
|
|
f("[]", nil)
|
|
f(`[{"Events":[]}]`, nil)
|
|
f(`[{
|
|
"EntityID":28257883748326179,
|
|
"IsAgent":true,
|
|
"Events":[
|
|
{
|
|
"eventType":"SystemSample",
|
|
"timestamp":1690286061,
|
|
"entityKey":"macbook-pro.local",
|
|
"dc": "1",
|
|
"diskWritesPerSecond":-34.21,
|
|
"uptime":762376
|
|
}
|
|
],
|
|
"ReportingAgentID":28257883748326179
|
|
}]`, []newrelic.Row{
|
|
{
|
|
Tags: []newrelic.Tag{
|
|
{
|
|
Key: []byte("eventType"),
|
|
Value: []byte("SystemSample"),
|
|
},
|
|
{
|
|
Key: []byte("entityKey"),
|
|
Value: []byte("macbook-pro.local"),
|
|
},
|
|
{
|
|
Key: []byte("dc"),
|
|
Value: []byte("1"),
|
|
},
|
|
},
|
|
Samples: []newrelic.Sample{
|
|
{
|
|
Name: []byte("diskWritesPerSecond"),
|
|
Value: -34.21,
|
|
},
|
|
{
|
|
Name: []byte("uptime"),
|
|
Value: 762376,
|
|
},
|
|
},
|
|
Timestamp: 1690286061000,
|
|
},
|
|
})
|
|
}
|