mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
120f3bc467
- Compare the actual free disk space to the value provided via -storage.minFreeDiskSpaceBytes directly inside the Storage.IsReadOnly(). This should work fast in most cases. This simplifies the logic at lib/storage. - Do not take into account -storage.minFreeDiskSpaceBytes during background merges, since it results in uncontrolled growth of small parts when the free disk space approaches -storage.minFreeDiskSpaceBytes. The background merge logic uses another mechanism for determining whether there is enough disk space for the merge - it reserves the needed disk space before the merge and releases it after the merge. This prevents from out of disk space errors during background merge. - Properly handle corner cases for flushing in-memory data to disk when the storage enters read-only mode. This is better than losing the in-memory data. - Return back Storage.MustAddRows() instead of Storage.AddRows(), since the only case when AddRows() can return error is when the storage is in read-only mode. This case must be handled by the caller by calling Storage.IsReadOnly() before adding rows to the storage. This simplifies the code a bit, since the caller of Storage.MustAddRows() shouldn't handle errors returned by Storage.AddRows(). - Properly store parsed logs to Storage if parts of the request contain invalid log lines. Previously the parsed logs could be lost in this case. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4737 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4945
130 lines
3.3 KiB
Go
130 lines
3.3 KiB
Go
package loki
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
|
)
|
|
|
|
func TestParseJSONRequestFailure(t *testing.T) {
|
|
f := func(s string) {
|
|
t.Helper()
|
|
n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
|
|
t.Fatalf("unexpected call to parseJSONRequest callback!")
|
|
})
|
|
if err == nil {
|
|
t.Fatalf("expecting non-nil error")
|
|
}
|
|
if n != 0 {
|
|
t.Fatalf("unexpected number of parsed lines: %d; want 0", n)
|
|
}
|
|
}
|
|
f(``)
|
|
|
|
// Invalid json
|
|
f(`{}`)
|
|
f(`[]`)
|
|
f(`"foo"`)
|
|
f(`123`)
|
|
|
|
// invalid type for `streams` item
|
|
f(`{"streams":123}`)
|
|
|
|
// Missing `values` item
|
|
f(`{"streams":[{}]}`)
|
|
|
|
// Invalid type for `values` item
|
|
f(`{"streams":[{"values":"foobar"}]}`)
|
|
|
|
// Invalid type for `stream` item
|
|
f(`{"streams":[{"stream":[],"values":[]}]}`)
|
|
|
|
// Invalid type for `values` individual item
|
|
f(`{"streams":[{"values":[123]}]}`)
|
|
|
|
// Invalid length of `values` individual item
|
|
f(`{"streams":[{"values":[[]]}]}`)
|
|
f(`{"streams":[{"values":[["123"]]}]}`)
|
|
f(`{"streams":[{"values":[["123","456","789"]]}]}`)
|
|
|
|
// Invalid type for timestamp inside `values` individual item
|
|
f(`{"streams":[{"values":[[123,"456"]}]}`)
|
|
|
|
// Invalid type for log message
|
|
f(`{"streams":[{"values":[["123",1234]]}]}`)
|
|
}
|
|
|
|
func TestParseJSONRequestSuccess(t *testing.T) {
|
|
f := func(s string, resultExpected string) {
|
|
t.Helper()
|
|
var lines []string
|
|
n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
|
|
var a []string
|
|
for _, f := range fields {
|
|
a = append(a, f.String())
|
|
}
|
|
line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
|
|
lines = append(lines, line)
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %s", err)
|
|
}
|
|
if n != len(lines) {
|
|
t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
|
|
}
|
|
result := strings.Join(lines, "\n")
|
|
if result != resultExpected {
|
|
t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
|
|
}
|
|
}
|
|
|
|
// Empty streams
|
|
f(`{"streams":[]}`, ``)
|
|
f(`{"streams":[{"values":[]}]}`, ``)
|
|
f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
|
|
f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
|
|
|
|
// Empty stream labels
|
|
f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
|
|
f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
|
|
|
|
// Non-empty stream labels
|
|
f(`{"streams":[{"stream":{
|
|
"label1": "value1",
|
|
"label2": "value2"
|
|
},"values":[
|
|
["1577836800000000001", "foo bar"],
|
|
["1477836900005000002", "abc"],
|
|
["147.78369e9", "foobar"]
|
|
]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
|
|
_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
|
|
_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
|
|
|
|
// Multiple streams
|
|
f(`{
|
|
"streams": [
|
|
{
|
|
"stream": {
|
|
"foo": "bar",
|
|
"a": "b"
|
|
},
|
|
"values": [
|
|
["1577836800000000001", "foo bar"],
|
|
["1577836900005000002", "abc"]
|
|
]
|
|
},
|
|
{
|
|
"stream": {
|
|
"x": "y"
|
|
},
|
|
"values": [
|
|
["1877836900005000002", "yx"]
|
|
]
|
|
}
|
|
]
|
|
}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
|
|
_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
|
|
_time:1877836900005000002 "x":"y" "_msg":"yx"`)
|
|
}
|