VictoriaMetrics/lib/logstorage/pipe_unpack_syslog_test.go

package logstorage

import (
	"testing"
)

func TestParsePipeUnpackSyslogSuccess(t *testing.T) {
	f := func(pipeStr string) {
		t.Helper()
		expectParsePipeSuccess(t, pipeStr)
	}

	f(`unpack_syslog`)
	f(`unpack_syslog keep_original_fields`)
	f(`unpack_syslog if (a:x)`)
	f(`unpack_syslog if (a:x) keep_original_fields`)
	f(`unpack_syslog from x`)
	f(`unpack_syslog from x keep_original_fields`)
	f(`unpack_syslog if (a:x) from x`)
	f(`unpack_syslog from x result_prefix abc`)
	f(`unpack_syslog if (a:x) from x result_prefix abc`)
	f(`unpack_syslog result_prefix abc`)
	f(`unpack_syslog if (a:x) result_prefix abc`)
}

func TestParsePipeUnpackSyslogFailure(t *testing.T) {
	f := func(pipeStr string) {
		t.Helper()
		expectParsePipeFailure(t, pipeStr)
	}

	f(`unpack_syslog foo`)
	f(`unpack_syslog if`)
	f(`unpack_syslog if (x:y) foobar`)
	f(`unpack_syslog from`)
	f(`unpack_syslog from x y`)
	f(`unpack_syslog from x if`)
	f(`unpack_syslog from x result_prefix`)
	f(`unpack_syslog from x result_prefix a b`)
	f(`unpack_syslog from x result_prefix a if`)
	f(`unpack_syslog result_prefix`)
	f(`unpack_syslog result_prefix a b`)
	f(`unpack_syslog result_prefix a if`)
}

func TestPipeUnpackSyslog(t *testing.T) {
	f := func(pipeStr string, rows, rowsExpected [][]Field) {
		t.Helper()
		expectPipeResults(t, pipeStr, rows, rowsExpected)
	}

	// no skip empty results
	f("unpack_syslog", [][]Field{
		{
			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
			{"foo", "321"},
		},
	}, [][]Field{
		{
			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
			{"foo", "321"},
			{"priority", "165"},
			{"facility", "20"},
			{"severity", "5"},
			{"format", "rfc5424"},
			{"timestamp", "2023-06-03T17:42:32.123456789Z"},
			{"hostname", "mymachine.example.com"},
			{"app_name", "appname"},
			{"proc_id", "12345"},
			{"msg_id", "ID47"},
			{"message", "This is a test message with structured data"},
		},
	})

	// keep original fields
	f("unpack_syslog keep_original_fields", [][]Field{
		{
			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
			{"foo", "321"},
			{"app_name", "foobar"},
			{"msg_id", "baz"},
		},
	}, [][]Field{
		{
			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
			{"foo", "321"},
			{"priority", "165"},
			{"facility", "20"},
			{"severity", "5"},
			{"format", "rfc5424"},
			{"timestamp", "2023-06-03T17:42:32.123456789Z"},
			{"hostname", "mymachine.example.com"},
			{"app_name", "foobar"},
			{"proc_id", "12345"},
			{"msg_id", "baz"},
			{"message", "This is a test message with structured data"},
		},
	})

	// unpack from other field
	f("unpack_syslog from x", [][]Field{
		{
			{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
		},
	}, [][]Field{
		{
			{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
			{"priority", "165"},
			{"facility", "20"},
			{"severity", "5"},
			{"format", "rfc5424"},
			{"timestamp", "2023-06-03T17:42:32.123456789Z"},
			{"hostname", "mymachine.example.com"},
			{"app_name", "appname"},
			{"proc_id", "12345"},
			{"msg_id", "ID47"},
			{"message", "This is a test message with structured data"},
		},
	})

	// failed if condition
	f("unpack_syslog if (foo:bar)", [][]Field{
		{
			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
		},
	}, [][]Field{
		{
			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
		},
	})

	// matched if condition
	f("unpack_syslog if (appname)", [][]Field{
		{
			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
		},
	}, [][]Field{
		{
			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
			{"priority", "165"},
			{"facility", "20"},
			{"severity", "5"},
			{"format", "rfc5424"},
			{"timestamp", "2023-06-03T17:42:32.123456789Z"},
			{"hostname", "mymachine.example.com"},
			{"app_name", "appname"},
			{"proc_id", "12345"},
			{"msg_id", "ID47"},
			{"message", "This is a test message with structured data"},
		},
	})

	// single row, unpack from missing field
	f("unpack_syslog from x", [][]Field{
		{
			{"_msg", `foo=bar`},
		},
	}, [][]Field{
		{
			{"_msg", `foo=bar`},
		},
	})

	// single row, unpack from non-syslog field
	f("unpack_syslog from x", [][]Field{
		{
			{"x", `foobar`},
		},
	}, [][]Field{
		{
			{"x", `foobar`},
			{"format", "rfc3164"},
			{"message", "foobar"},
		},
	})

	// multiple rows with distinct number of fields
	f("unpack_syslog from x result_prefix qwe_", [][]Field{
		{
			{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
		},
		{
			{"x", `<163>1 2024-12-13T18:21:43Z mymachine.example.com appname2 345 ID7 - foobar`},
			{"y", `z=bar`},
		},
	}, [][]Field{
		{
			{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
			{"qwe_priority", "165"},
			{"qwe_facility", "20"},
			{"qwe_severity", "5"},
			{"qwe_format", "rfc5424"},
			{"qwe_timestamp", "2023-06-03T17:42:32.123456789Z"},
			{"qwe_hostname", "mymachine.example.com"},
			{"qwe_app_name", "appname"},
			{"qwe_proc_id", "12345"},
			{"qwe_msg_id", "ID47"},
			{"qwe_message", "This is a test message with structured data"},
		},
		{
			{"x", `<163>1 2024-12-13T18:21:43Z mymachine.example.com appname2 345 ID7 - foobar`},
			{"y", `z=bar`},
			{"qwe_priority", "163"},
			{"qwe_facility", "20"},
			{"qwe_severity", "3"},
			{"qwe_format", "rfc5424"},
			{"qwe_timestamp", "2024-12-13T18:21:43Z"},
			{"qwe_hostname", "mymachine.example.com"},
			{"qwe_app_name", "appname2"},
			{"qwe_proc_id", "345"},
			{"qwe_msg_id", "ID7"},
			{"qwe_message", "foobar"},
		},
	})
}

func TestPipeUnpackSyslogUpdateNeededFields(t *testing.T) {
	f := func(s string, neededFields, unneededFields, neededFieldsExpected, unneededFieldsExpected string) {
		t.Helper()
		expectPipeNeededFields(t, s, neededFields, unneededFields, neededFieldsExpected, unneededFieldsExpected)
	}

	// all the needed fields
	f("unpack_syslog", "*", "", "*", "")
	f("unpack_syslog keep_original_fields", "*", "", "*", "")
	f("unpack_syslog if (y:z) from x", "*", "", "*", "")

	// all the needed fields, unneeded fields do not intersect with src
	f("unpack_syslog from x", "*", "f1,f2", "*", "f1,f2")
	f("unpack_syslog if (y:z) from x", "*", "f1,f2", "*", "f1,f2")
	f("unpack_syslog if (f1:z) from x", "*", "f1,f2", "*", "f2")

	// all the needed fields, unneeded fields intersect with src
	f("unpack_syslog from x", "*", "f2,x", "*", "f2")
	f("unpack_syslog if (y:z) from x", "*", "f2,x", "*", "f2")
	f("unpack_syslog if (f2:z) from x", "*", "f1,f2,x", "*", "f1")

	// needed fields do not intersect with src
	f("unpack_syslog from x", "f1,f2", "", "f1,f2,x", "")
	f("unpack_syslog if (y:z) from x", "f1,f2", "", "f1,f2,x,y", "")
	f("unpack_syslog if (f1:z) from x", "f1,f2", "", "f1,f2,x", "")

	// needed fields intersect with src
	f("unpack_syslog from x", "f2,x", "", "f2,x", "")
	f("unpack_syslog if (y:z) from x", "f2,x", "", "f2,x,y", "")
	f("unpack_syslog if (f2:z y:qwe) from x", "f2,x", "", "f2,x,y", "")
}