github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity
VictoriaMetrics/lib/protoparser/prometheus/parser_test.go
Ted Possible 0206a01d03
Exemplar support (#5982) 
This code adds Exemplars to VMagent and the promscrape parser adhering
to OpenMetrics Specifications. This will allow forwarding of exemplars
to Prometheus and other third party apps that support OpenMetrics specs.

---------

Signed-off-by: Ted Possible <ted_possible@cable.comcast.com>
(cherry picked from commit 5a3abfa041)
2024-05-10 13:14:17 +02:00

713 lines
16 KiB
Go
Raw Blame History

package prometheus
import (
"math"
"reflect"
"testing"
)
func TestGetRowsDiff(t *testing.T) {
f := func(s1, s2, resultExpected string) {
t.Helper()
result := GetRowsDiff(s1, s2)
if result != resultExpected {
t.Fatalf("unexpected result for GetRowsDiff(%q, %q); got %q; want %q", s1, s2, result, resultExpected)
}
}
f("", "", "")
f("", "foo 1", "")
f(" ", "foo 1", "")
f("foo 123", "", "foo 0\n")
f("foo 123", "bar 3", "foo 0\n")
f("foo 123", "bar 3\nfoo 344", "")
f("foo{x=\"y\", z=\"a a a\"} 123", "bar 3\nfoo{x=\"y\", z=\"b b b\"} 344", "foo{x=\"y\",z=\"a a a\"} 0\n")
f("foo{bar=\"baz\"} 123\nx 3.4 5\ny 5 6", "x 34 342", "foo{bar=\"baz\"} 0\ny 0\n")
}
func TestAreIdenticalSeriesFast(t *testing.T) {
f := func(s1, s2 string, resultExpected bool) {
t.Helper()
result := AreIdenticalSeriesFast(s1, s2)
if result != resultExpected {
t.Fatalf("unexpected result for AreIdenticalSeries(%q, %q); got %v; want %v", s1, s2, result, resultExpected)
}
}
f("", "", true)
f("", "a 1", false) // different number of metrics
f(" ", " a 1", false) // different number of metrics
f("a 1", "", false) // different number of metrics
f(" a 1", " ", false) // different number of metrics
f("foo", "foo", true) // consider series identical if they miss value
f("foo 1", "foo 1", true)
f("foo 1", "foo 2", true)
f("foo 1 ", "foo 2 ", true)
f("foo 1 ", "foo 2 ", false) // different number of spaces
f("foo 1 ", "foo 2 ", false) // different number of spaces
f("foo nan", "foo -inf", true)
f("foo 1 # coment x", "foo 2 #comment y", true)
f(" foo 1", " foo 1", true)
f(" foo 1", " foo 1", false) // different number of spaces in front of metric
f(" foo 1", " foo 1", false) // different number of spaces in front of metric
f("foo 1", "bar 1", false) // different metric name
f("foo 1", "fooo 1", false) // different metric name
f("foo 123", "foo 32.32", true)
f(`foo{bar="x"} -3.3e-6`, `foo{bar="x"} 23343`, true)
f(`foo{} 1`, `foo{} 234`, true)
f(`foo {x="y x" } 234`, `foo {x="y x" } 43.342`, true)
f(`foo {x="y x"} 234`, `foo{x="y x"} 43.342`, false) // different spaces
f("foo 2\nbar 3", "foo 34.43\nbar -34.3", true)
f("foo 2\nbar 3", "foo 34.43\nbarz -34.3", false) // different metric names
f("\nfoo 13\n", "\nfoo 3.4\n", true)
f("\nfoo 13", "\nfoo 3.4\n", false) // different number of blank lines
f("\nfoo 13\n", "\nfoo 3.4", false) // different number of blank lines
f("\n\nfoo 1", "\n\nfoo 34.43", true)
f("\n\nfoo 3434\n", "\n\nfoo 43\n", true)
f("\nfoo 1", "\n\nfoo 34.43", false) // different number of blank lines
f("#foo{bar}", "#baz", true)
f("", "#baz", false) // different number of comments
f("#foo{bar}", "", false) // different number of comments
f("#foo{bar}", "bar 3", false) // different number of comments
f("foo{bar} 2", "#bar 3", false) // different number of comments
f("#foo\n", "#bar", false) // different number of blank lines
f("#foo{bar}\n#baz", "#baz\n#xdsfds dsf", true)
f("# foo\nbar 234\nbaz{x=\"y\", z=\"\"} 3", "# foo\nbar 3.3\nbaz{x=\"y\", z=\"\"} 4323", true)
f("# foo\nbar 234\nbaz{x=\"z\", z=\"\"} 3", "# foo\nbar 3.3\nbaz{x=\"y\", z=\"\"} 4323", false) // different label value
f("foo {bar=\"xfdsdsffdsa\"} 1", "foo {x=\"y\"} 2", false) // different labels
f("foo {x=\"z\"} 1", "foo {x=\"y\"} 2", false) // different label value
// Lines with timestamps
f("foo 1 2", "foo 234 4334", true)
f("foo 2", "foo 3 4", false) // missing timestamp
f("foo 2 1", "foo 3", false) // missing timestamp
f("foo{bar=\"b az\"} 2 5", "foo{bar=\"b az\"} +6.3 7.43", true)
f("foo{bar=\"b az\"} 2 5 # comment ss ", "foo{bar=\"b az\"} +6.3 7.43 # comment as ", true)
f("foo{bar=\"b az\"} 2 5 #comment", "foo{bar=\"b az\"} +6.3 7.43 #comment {foo=\"bar\"} 21.44", true)
f("foo{bar=\"b az\"} +Inf 5", "foo{bar=\"b az\"} NaN 7.43", true)
f("foo{bar=\"b az\"} +Inf 5", "foo{bar=\"b az\"} nan 7.43", true)
f("foo{bar=\"b az\"} +Inf 5", "foo{bar=\"b az\"} nansf 7.43", false) // invalid value
// False positive - whitespace after the numeric char in the label.
f(`foo{bar=" 12.3 "} 1`, `foo{bar=" 13 "} 23`, true)
f(`foo{bar=" 12.3 "} 1 3443`, `foo{bar=" 13 "} 23 4345`, true)
f(`foo{bar=" 12.3 "} 1 3443 # {} 34`, `foo{bar=" 13 "} 23 4345 # {foo=" bar "} 34`, true)
// Metrics and labels with '#' chars
f(`foo{bar="#1"} 1`, `foo{bar="#1"} 1`, true)
f(`foo{bar="a#1"} 1`, `foo{bar="b#1"} 1.4`, false)
f(`foo{bar=" #1"} 1`, `foo{bar=" #1"} 3`, true)
f(`foo{bar=" #1 "} 1`, `foo{bar=" #1 "} -2.34 343.34 # {foo="#bar"} `, true)
f(`foo{bar=" #1"} 1`, `foo{bar="#1"} 1`, false)
f(`foo{b#ar=" #1"} 1`, `foo{b#ar=" #1"} 1.23`, true)
f(`foo{z#ar=" #1"} 1`, `foo{b#ar=" #1"} 1.23`, false)
f(`fo#o{b#ar="#1"} 1`, `fo#o{b#ar="#1"} 1.23`, true)
f(`fo#o{b#ar="#1"} 1`, `fa#o{b#ar="#1"} 1.23`, false)
// False positive - the value after '#' char can be arbitrary
f(`fo#o{b#ar="#1"} 1`, `fo#osdf 1.23`, true)
}
func TestPrevBackslashesCount(t *testing.T) {
f := func(s string, nExpected int) {
t.Helper()
n := prevBackslashesCount(s)
if n != nExpected {
t.Fatalf("unexpected value returned from prevBackslashesCount(%q); got %d; want %d", s, n, nExpected)
}
}
f(``, 0)
f(`foo`, 0)
f(`\`, 1)
f(`\\`, 2)
f(`\\\`, 3)
f(`\\\a`, 0)
f(`foo\bar`, 0)
f(`foo\\`, 2)
f(`\\foo\`, 1)
f(`\\foo\\\\`, 4)
}
func TestFindClosingQuote(t *testing.T) {
f := func(s string, nExpected int) {
t.Helper()
n := findClosingQuote(s)
if n != nExpected {
t.Fatalf("unexpected value returned from findClosingQuote(%q); got %d; want %d", s, n, nExpected)
}
}
f(``, -1)
f(`x`, -1)
f(`"`, -1)
f(`""`, 1)
f(`foobar"`, -1)
f(`"foo"`, 4)
f(`"\""`, 3)
f(`"\\"`, 3)
f(`"\"`, -1)
f(`"foo\"bar\"baz"`, 14)
}
func TestUnescapeValue(t *testing.T) {
f := func(s, resultExpected string) {
t.Helper()
result := unescapeValue(s)
if result != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(``, "")
f(`f`, "f")
f(`foobar`, "foobar")
f(`\"\n\t`, "\"\n\\t")
// Edge cases
f(`foo\bar`, "foo\\bar")
f(`foo\`, "foo\\")
}
func TestAppendEscapedValue(t *testing.T) {
f := func(s, resultExpected string) {
t.Helper()
result := appendEscapedValue(nil, s)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(``, ``)
f(`f`, `f`)
f(`foobar`, `foobar`)
f("\"\n\t\\xyz", "\\\"\\n\t\\\\xyz")
}
func TestRowsUnmarshalFailure(t *testing.T) {
f := func(s string) {
t.Helper()
var rows Rows
rows.Unmarshal(s)
if len(rows.Rows) != 0 {
t.Fatalf("unexpected number of rows parsed; got %d; want 0;\nrows:%#v", len(rows.Rows), rows.Rows)
}
// Try again
rows.Unmarshal(s)
if len(rows.Rows) != 0 {
t.Fatalf("unexpected number of rows parsed; got %d; want 0;\nrows:%#v", len(rows.Rows), rows.Rows)
}
}
// Empty lines and comments
f("")
f(" ")
f("\t")
f("\t \r")
f("\t\t \n\n # foobar")
f("#foobar")
f("#foobar\n")
// invalid tags
f("a{")
f("a { ")
f("a {foo")
f("a {foo} 3")
f("a {foo =")
f(`a {foo ="bar`)
f(`a {foo ="b\ar`)
f(`a {foo = "bar"`)
f(`a {foo ="bar",`)
f(`a {foo ="bar" , `)
f(`a {foo ="bar" , baz } 2`)
// invalid tags - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4284
f(`a{"__name__":"upsd_time_left_ns","host":"myhost", status_OB="true"} 12`)
f(`a{host:"myhost"} 12`)
f(`a{host:"myhost",foo="bar"} 12`)
// empty metric name
f(`{foo="bar"}`)
// Invalid quotes for label value
f(`{foo='bar'} 23`)
f("{foo=`bar`} 23")
// Missing value
f("aaa")
f(" aaa")
f(" aaa ")
f(" aaa \n")
f(` aa{foo="bar"} ` + "\n")
// Invalid value
f("foo bar")
f("foo bar 124")
// Invalid timestamp
f("foo 123 bar")
}
func TestRowsUnmarshalSuccess(t *testing.T) {
f := func(s string, rowsExpected *Rows) {
t.Helper()
var rows Rows
rows.Unmarshal(s)
if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
}
// Try unmarshaling again
rows.Unmarshal(s)
if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) {
t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows)
}
rows.Reset()
if len(rows.Rows) != 0 {
t.Fatalf("non-empty rows after reset: %+v", rows.Rows)
}
}
// Empty line or comment
f("", &Rows{})
f("\r", &Rows{})
f("\n\n", &Rows{})
f("\n\r\n", &Rows{})
f("\t \t\n\r\n#foobar\n # baz", &Rows{})
// Single line
f("foobar 78.9", &Rows{
Rows: []Row{{
Metric: "foobar",
Value: 78.9,
}},
})
f("foobar 123.456 789\n", &Rows{
Rows: []Row{{
Metric: "foobar",
Value: 123.456,
Timestamp: 789000,
}},
})
f("foobar{} 123.456 789.4354\n", &Rows{
Rows: []Row{{
Metric: "foobar",
Value: 123.456,
Timestamp: 789435,
}},
})
f(`# _ _
# ___ __ _ ___ ___ __ _ _ __ __| |_ __ __ _ _____ ___ __ ___ _ __| |_ ___ _ __
`+"# / __/ _` / __/ __|/ _` | '_ \\ / _` | '__/ _` |_____ / _ \\ \\/ / '_ \\ / _ \\| '__| __/ _ \\ '__|\n"+`
# | (_| (_| \__ \__ \ (_| | | | | (_| | | | (_| |_____| __/> <| |_) | (_) | | | || __/ |
# \___\__,_|___/___/\__,_|_| |_|\__,_|_| \__,_| \___/_/\_\ .__/ \___/|_| \__\___|_|
# |_|
#
# TYPE cassandra_token_ownership_ratio gauge
cassandra_token_ownership_ratio 78.9`, &Rows{
Rows: []Row{{
Metric: "cassandra_token_ownership_ratio",
Value: 78.9,
}},
})
// `#` char in label value
f(`foo{bar="#1 az"} 24`, &Rows{
Rows: []Row{{
Metric: "foo",
Tags: []Tag{{
Key: "bar",
Value: "#1 az",
}},
Value: 24,
}},
})
// `#` char in label name and label value
f(`foo{bar#2="#1 az"} 24 456`, &Rows{
Rows: []Row{{
Metric: "foo",
Tags: []Tag{{
Key: "bar#2",
Value: "#1 az",
}},
Value: 24,
Timestamp: 456000,
}},
})
// `#` char in metric name, label name and label value
f(`foo#qw{bar#2="#1 az"} 24 456 # foobar {baz="x"}`, &Rows{
Rows: []Row{{
Metric: "foo#qw",
Tags: []Tag{{
Key: "bar#2",
Value: "#1 az",
}},
Value: 24,
Timestamp: 456000,
}},
})
// Incorrectly escaped backlash. This is real-world case, which must be supported.
f(`mssql_sql_server_active_transactions_sec{loginname="domain\somelogin",env="develop"} 56`, &Rows{
Rows: []Row{{
Metric: "mssql_sql_server_active_transactions_sec",
Tags: []Tag{
{
Key: "loginname",
Value: "domain\\somelogin",
},
{
Key: "env",
Value: "develop",
},
},
Value: 56,
}},
})
// Support for Exemplars Open Metric Specification
// see: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars-1
f(`foo_bucket{le="25"} 17 # {trace_id="oHg5SJYRHA0", log_id="test_id"} 9.8 1520879607.789`, &Rows{
Rows: []Row{
{
Metric: "foo_bucket",
Tags: []Tag{
{
Key: "le",
Value: "25",
},
},
Value: 17,
Exemplar: Exemplar{
Value: 9.8,
Tags: []Tag{
{
Key: "trace_id",
Value: "oHg5SJYRHA0",
},
{
Key: "log_id",
Value: "test_id",
},
},
Timestamp: 1520879607789,
},
},
}})
f(`foo_bucket{le="0.01"} 0
foo_bucket{le="0.1"} 8 # {} 0.054
foo_bucket{le="1"} 11 # {trace_id="KOO5S4vxi0o"} 0.67
foo_bucket{le="10"} 17 # {trace_id="oHg5SJYRHA0"} 9.8 1520879607.789
foo_bucket{le="25"} 17 # {trace_id="oHg5SJYRHA0", log_id="test_id"} 9.8 1520879607.789
foo_bucket{nospace="exemplar"} 17 #{trace_id="oHg5SJYRHA0"} 9.8 1520879607.789
foo_bucket{le="+Inf"} 17
foo_count 17
foo_sum 324789.3
foo_created 1520430000.123`, &Rows{
Rows: []Row{
{
Metric: "foo_bucket",
Tags: []Tag{
{
Key: "le",
Value: "0.01",
},
},
},
{
Metric: "foo_bucket",
Tags: []Tag{
{
Key: "le",
Value: "0.1",
},
},
Value: 8,
Exemplar: Exemplar{
Value: 0.054,
},
},
{
Metric: "foo_bucket",
Tags: []Tag{
{
Key: "le",
Value: "1",
},
},
Value: 11,
Exemplar: Exemplar{
Value: 0.67,
Tags: []Tag{
{
Key: "trace_id",
Value: "KOO5S4vxi0o",
},
},
},
},
{
Metric: "foo_bucket",
Tags: []Tag{
{
Key: "le",
Value: "10",
},
},
Value: 17,
Exemplar: Exemplar{
Value: 9.8,
Tags: []Tag{
{
Key: "trace_id",
Value: "oHg5SJYRHA0",
},
},
Timestamp: 1520879607789,
},
},
{
Metric: "foo_bucket",
Tags: []Tag{
{
Key: "le",
Value: "25",
},
},
Value: 17,
Exemplar: Exemplar{
Value: 9.8,
Tags: []Tag{
{
Key: "trace_id",
Value: "oHg5SJYRHA0",
},
{
Key: "log_id",
Value: "test_id",
},
},
Timestamp: 1520879607789,
},
},
{
Metric: "foo_bucket",
Tags: []Tag{
{
Key: "nospace",
Value: "exemplar",
},
},
Value: 17,
Exemplar: Exemplar{
Value: 9.8,
Tags: []Tag{
{
Key: "trace_id",
Value: "oHg5SJYRHA0",
},
},
Timestamp: 1520879607789,
},
},
{
Metric: "foo_bucket",
Tags: []Tag{
{
Key: "le",
Value: "+Inf",
},
},
Value: 17,
},
{
Metric: "foo_count",
Value: 17,
},
{
Metric: "foo_sum",
Value: 324789.3,
},
{
Metric: "foo_created",
Value: 1520430000.123,
},
},
})
// "Infinity" word - this has been added in OpenMetrics.
// See https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md
// Checks for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/924
inf := math.Inf(1)
f(`
foo Infinity
bar +Infinity
baz -infinity
aaa +inf
bbb -INF
ccc INF
`, &Rows{
Rows: []Row{
{
Metric: "foo",
Value: inf,
},
{
Metric: "bar",
Value: inf,
},
{
Metric: "baz",
Value: -inf,
},
{
Metric: "aaa",
Value: inf,
},
{
Metric: "bbb",
Value: -inf,
},
{
Metric: "ccc",
Value: inf,
},
},
})
// Timestamp bigger than 1<<31.
// It should be parsed in milliseconds.
f("aaa 1123 429496729600", &Rows{
Rows: []Row{{
Metric: "aaa",
Value: 1123,
Timestamp: 429496729600,
}},
})
// Floating-point timestamps in OpenMetric format.
f("aaa 1123 42949.567", &Rows{
Rows: []Row{{
Metric: "aaa",
Value: 1123,
Timestamp: 42949567,
}},
})
// Tags
f(`foo{bar="baz"} 1 2`, &Rows{
Rows: []Row{{
Metric: "foo",
Tags: []Tag{{
Key: "bar",
Value: "baz",
}},
Value: 1,
Timestamp: 2000,
}},
})
f(`foo{bar="b\"a\\z"} -1.2`, &Rows{
Rows: []Row{{
Metric: "foo",
Tags: []Tag{{
Key: "bar",
Value: "b\"a\\z",
}},
Value: -1.2,
}},
})
// Empty tags
f(`foo {bar="baz",aa="",x="y",="z"} 1 2`, &Rows{
Rows: []Row{{
Metric: "foo",
Tags: []Tag{
{
Key: "bar",
Value: "baz",
},
{
Key: "aa",
Value: "",
},
{
Key: "x",
Value: "y",
},
},
Value: 1,
Timestamp: 2000,
}},
})
// Trailing comma after tag
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/350
f(`foo{bar="baz",} 1 2`, &Rows{
Rows: []Row{{
Metric: "foo",
Tags: []Tag{{
Key: "bar",
Value: "baz",
}},
Value: 1,
Timestamp: 2000,
}},
})
// Multi lines
f("# foo\n # bar ba zzz\nfoo 0.3 2\naaa 3\nbar.baz 0.34 43\n", &Rows{
Rows: []Row{
{
Metric: "foo",
Value: 0.3,
Timestamp: 2000,
},
{
Metric: "aaa",
Value: 3,
},
{
Metric: "bar.baz",
Value: 0.34,
Timestamp: 43000,
},
},
})
// Multi lines with invalid line
f("\t foo\t { } 0.3\t 2\naaa\n bar.baz 0.34 43\n", &Rows{
Rows: []Row{
{
Metric: "foo",
Value: 0.3,
Timestamp: 2000,
},
{
Metric: "bar.baz",
Value: 0.34,
Timestamp: 43000,
},
},
})
// Spaces around tags
f(`vm_accounting { name="vminsertRows", accountID = "1" , projectID= "1" } 277779100`, &Rows{
Rows: []Row{
{
Metric: "vm_accounting",
Tags: []Tag{
{
Key: "name",
Value: "vminsertRows",
},
{
Key: "accountID",
Value: "1",
},
{
Key: "projectID",
Value: "1",
},
},
Value: 277779100,
Timestamp: 0,
},
},
})
}
Reference in a new issue View git blame Copy permalink