VictoriaMetrics/lib/logstorage/block_test.go

package logstorage

import (
	"fmt"
	"reflect"
	"testing"
)

func TestBlockMustInitFromRows(t *testing.T) {
	f := func(timestamps []int64, rows [][]Field, bExpected *block) {
		t.Helper()
		b := getBlock()
		defer putBlock(b)

		rowsProcessed := b.MustInitFromRows(timestamps, rows)
		if rowsProcessed != len(rows) {
			t.Fatalf("unexpected rowsProcessed; got %d; want %d", rowsProcessed, len(rows))
		}
		if b.uncompressedSizeBytes() >= maxUncompressedBlockSize {
			t.Fatalf("expecting non-full block")
		}
		if !reflect.DeepEqual(b, bExpected) {
			t.Fatalf("unexpected block;\ngot\n%v\nwant\n%v", b, bExpected)
		}
		if n := b.Len(); n != len(timestamps) {
			t.Fatalf("unexpected block len; got %d; want %d", n, len(timestamps))
		}
		b.assertValid()
	}

	// An empty log entries
	f(nil, nil, &block{})
	f([]int64{}, [][]Field{}, &block{})

	// A single row
	timestamps := []int64{1234}
	rows := [][]Field{
		{
			{
				Name:  "msg",
				Value: "foo",
			},
			{
				Name:  "level",
				Value: "error",
			},
		},
	}
	bExpected := &block{
		timestamps: []int64{1234},
		constColumns: []Field{
			{
				Name:  "level",
				Value: "error",
			},
			{
				Name:  "msg",
				Value: "foo",
			},
		},
	}
	f(timestamps, rows, bExpected)

	// Multiple log entries with the same set of fields
	timestamps = []int64{3, 5}
	rows = [][]Field{
		{
			{
				Name:  "job",
				Value: "foo",
			},
			{
				Name:  "instance",
				Value: "host1",
			},
		},
		{
			{
				Name:  "job",
				Value: "foo",
			},
			{
				Name:  "instance",
				Value: "host2",
			},
		},
	}
	bExpected = &block{
		timestamps: []int64{3, 5},
		columns: []column{
			{
				name:   "instance",
				values: []string{"host1", "host2"},
			},
		},
		constColumns: []Field{
			{
				Name:  "job",
				Value: "foo",
			},
		},
	}
	f(timestamps, rows, bExpected)

	// Multiple log entries with distinct set of fields
	timestamps = []int64{3, 5, 10}
	rows = [][]Field{
		{
			{
				Name:  "msg",
				Value: "foo",
			},
			{
				Name:  "b",
				Value: "xyz",
			},
		},
		{
			{
				Name:  "b",
				Value: "xyz",
			},
			{
				Name:  "a",
				Value: "aaa",
			},
		},
		{
			{
				Name:  "b",
				Value: "xyz",
			},
		},
	}
	bExpected = &block{
		timestamps: []int64{3, 5, 10},
		columns: []column{
			{
				name:   "a",
				values: []string{"", "aaa", ""},
			},
			{
				name:   "msg",
				values: []string{"foo", "", ""},
			},
		},
		constColumns: []Field{
			{
				Name:  "b",
				Value: "xyz",
			},
		},
	}
	f(timestamps, rows, bExpected)
}

func TestBlockMustInitFromRowsFullBlock(t *testing.T) {
	const rowsCount = 2000
	timestamps := make([]int64, rowsCount)
	rows := make([][]Field, rowsCount)
	for i := range timestamps {
		fields := make([]Field, 10)
		for j := range fields {
			fields[j] = Field{
				Name:  fmt.Sprintf("field_%d", j),
				Value: "very very looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong value",
			}
		}
		rows[i] = fields
	}

	b := getBlock()
	defer putBlock(b)
	rowsProcessed := b.MustInitFromRows(timestamps, rows)
	if rowsProcessed != len(rows) {
		t.Fatalf("unexpected rowsProcessed; got %d; want %d", rowsProcessed, len(rows))
	}
	b.assertValid()
	if n := b.Len(); n != len(rows) {
		t.Fatalf("unexpected total log entries; got %d; want %d", n, len(rows))
	}
	if n := b.uncompressedSizeBytes(); n < maxUncompressedBlockSize {
		t.Fatalf("expecting full block with %d bytes; got %d bytes", maxUncompressedBlockSize, n)
	}
}

func TestBlockMustInitWithNonEmptyOffset(t *testing.T) {
	f := func(rowsCount int, fieldsPerRow int, expectedRowsProcessed int) {
		t.Helper()
		timestamps := make([]int64, rowsCount)
		rows := make([][]Field, rowsCount)
		for i := range timestamps {
			fields := make([]Field, fieldsPerRow)
			for j := range fields {
				fields[j] = Field{
					Name:  fmt.Sprintf("field_%d_%d", i, j),
					Value: "very very looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong value",
				}
			}
			rows[i] = fields
		}
		b := getBlock()
		defer putBlock(b)
		rowsProcessed := b.MustInitFromRows(timestamps, rows)
		if rowsProcessed != expectedRowsProcessed {
			t.Fatalf("unexpected rowsProcessed; got %d; want %d", rowsProcessed, expectedRowsProcessed)
		}
		b.assertValid()
		if n := b.Len(); n != rowsProcessed {
			t.Fatalf("unexpected total log entries; got %d; want %d", n, rowsProcessed)
		}
	}
	f(10, 300, 6)
	f(10, 10, 10)
	f(15, 30, 15)
	f(maxColumnsPerBlock+1000, 1, maxColumnsPerBlock)
}