github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-01 14:47:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/mergeset: use sort.IsSorted instead of sort.SliceIsSorted in inmemoryBlock.isSorted in order to reduce memory allocations

Browse source
This commit is contained in:
Aliaksandr Valialkin 2019-09-19 20:13:00 +03:00
parent 89234f395d
commit ebbef20535
2 changed files with 21 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
lib/mergeset/encoding.go
View file

@ -11,10 +11,20 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
) )
type byteSliceSorter [][]byte
func (s byteSliceSorter) Len() int { return len(s) }
func (s byteSliceSorter) Less(i, j int) bool {
return string(s[i]) < string(s[j])
}
func (s byteSliceSorter) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
type inmemoryBlock struct { type inmemoryBlock struct {
commonPrefix []byte commonPrefix []byte
data []byte data []byte
items [][]byte items byteSliceSorter
} }
func (ib *inmemoryBlock) Reset() { func (ib *inmemoryBlock) Reset() {
@ -77,12 +87,9 @@ func (ib *inmemoryBlock) Add(x []byte) bool {
// It must fit CPU cache size, i.e. 64KB for the current CPUs. // It must fit CPU cache size, i.e. 64KB for the current CPUs.
const maxInmemoryBlockSize = 64 * 1024 const maxInmemoryBlockSize = 64 * 1024
func (ib *inmemoryBlock) itemsLess(i, j int) bool {
return string(ib.items[i]) < string(ib.items[j])
}
func (ib *inmemoryBlock) sort() { func (ib *inmemoryBlock) sort() {
sort.Slice(ib.items, ib.itemsLess) // Use sort.Sort instead of sort.Slice in order to eliminate memory allocation.
sort.Sort(&ib.items)
bb := bbPool.Get() bb := bbPool.Get()
b := bytesutil.Resize(bb.B, len(ib.data)) b := bytesutil.Resize(bb.B, len(ib.data))
b = b[:0] b = b[:0]
@ -120,7 +127,8 @@ func checkMarshalType(mt marshalType) error {
} }
func (ib *inmemoryBlock) isSorted() bool { func (ib *inmemoryBlock) isSorted() bool {
return sort.SliceIsSorted(ib.items, ib.itemsLess) // Use sort.IsSorted instead of sort.SliceIsSorted in order to eliminate memory allocation.
return sort.IsSorted(&ib.items)
} }
// MarshalUnsortedData marshals unsorted items from ib to sb. // MarshalUnsortedData marshals unsorted items from ib to sb.
@ -138,6 +146,8 @@ func (ib *inmemoryBlock) MarshalUnsortedData(sb *storageBlock, firstItemDst, com
return ib.marshalData(sb, firstItemDst, commonPrefixDst, compressLevel) return ib.marshalData(sb, firstItemDst, commonPrefixDst, compressLevel)
} }
var isInTest bool
// MarshalUnsortedData marshals sorted items from ib to sb. // MarshalUnsortedData marshals sorted items from ib to sb.
// //
// It also: // It also:
@ -146,9 +156,9 @@ func (ib *inmemoryBlock) MarshalUnsortedData(sb *storageBlock, firstItemDst, com
// - returns the number of items encoded including the first item. // - returns the number of items encoded including the first item.
// - returns the marshal type used for the encoding. // - returns the marshal type used for the encoding.
func (ib *inmemoryBlock) MarshalSortedData(sb *storageBlock, firstItemDst, commonPrefixDst []byte, compressLevel int) ([]byte, []byte, uint32, marshalType) { func (ib *inmemoryBlock) MarshalSortedData(sb *storageBlock, firstItemDst, commonPrefixDst []byte, compressLevel int) ([]byte, []byte, uint32, marshalType) {
// if !ib.isSorted() { if isInTest && !ib.isSorted() {
// logger.Panicf("BUG: %d items must be sorted; items:\n%s", len(ib.items), ib.debugItemsString()) logger.Panicf("BUG: %d items must be sorted; items:\n%s", len(ib.items), ib.debugItemsString())
// } }
ib.updateCommonPrefix() ib.updateCommonPrefix()
return ib.marshalData(sb, firstItemDst, commonPrefixDst, compressLevel) return ib.marshalData(sb, firstItemDst, commonPrefixDst, compressLevel)
} }

1
lib/mergeset/table_search_test.go
View file

@ -11,6 +11,7 @@ import (
) )
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
isInTest = true
n := m.Run() n := m.Run()
os.Exit(n) os.Exit(n)
} }