github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-02-09 15:27:11 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: disable composite index usage when querying old data

Browse source
This commit is contained in:
Aliaksandr Valialkin 2021-02-10 14:37:14 +02:00
parent 17d5a03f6e
commit 148422bcba
9 changed files with 296 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
app/vmstorage/transport/server.go
View file

@ -29,8 +29,6 @@ var (
maxTagValuesPerSearch = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned per search")
maxTagValueSuffixesPerSearch = flag.Int("search.maxTagValueSuffixesPerSearch", 100e3, "The maximum number of tag value suffixes returned from /metrics/find")
maxMetricsPerSearch = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series each search can scan")
disableCompositeTagFilters = flag.Bool("search.disableCompositeTagFilters", false, "Whether to disable composite tag filters. This option is useful "+
"for querying old data, which is created before v1.54.0 release. Note that disabled composite tag filters may reduce query performance")
precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss")
disableRPCCompression = flag.Bool(`rpc.disableCompression`, false, "Disable compression of RPC traffic. This reduces CPU usage at the cost of higher network bandwidth usage")
@ -1270,9 +1268,6 @@ func (ctx *vmselectRequestCtx) setupTfss(s *storage.Storage, tr storage.TimeRang
accountID := ctx.sq.AccountID
projectID := ctx.sq.ProjectID
for _, tagFilters := range ctx.sq.TagFilterss {
if !*disableCompositeTagFilters {
tagFilters = storage.ConvertToCompositeTagFilters(tagFilters)
}
tfs := storage.NewTagFilters(accountID, projectID)
for i := range tagFilters {
tf := &tagFilters[i]

2
docs/CHANGELOG.md
View file

@ -2,7 +2,7 @@
# tip
* FEATURE: optimize searching for time series by label filters where individual filters match big number of time series (more than a million). For example, the query `up{job="foobar"}` should work faster if `{job="foobar"}` matches a million of time series, while `up{job="foobar"}` matches much lower number of time series. The optimization can be disabled by passing `-search.disableCompositeTagFilters` command-line flag to VictoriaMetrics.
* FEATURE: optimize searching for time series by label filters where individual filters match big number of time series (more than a million). For example, the query `up{job="foobar"}` should work faster if `{job="foobar"}` matches a million of time series, while `up{job="foobar"}` matches much lower number of time series.
* FEATURE: single-node VictoriaMetrics now accepts requests to handlers with `/prometheus` and `/graphite` prefixes such as `/prometheus/api/v1/query`. This improves compatibility with [handlers from VictoriaMetrics cluster](https://victoriametrics.github.io/Cluster-VictoriaMetrics.html#url-format).
* FEATURE: expose `process_open_fds` and `process_max_fds` metrics. These metrics can be used for alerting when `process_open_fds` reaches `process_max_fds`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/402 and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1037
* FEATURE: vmalert: add `-datasource.appendTypePrefix` command-line option for querying both Prometheus and Graphite datasource in cluster version of VictoriaMetrics. See [these docs](https://victoriametrics.github.io/vmalert.html#graphite) for details.

43
lib/fs/fs.go
View file

@ -3,6 +3,7 @@ package fs
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"regexp"
@ -257,6 +258,48 @@ func SymlinkRelative(srcPath, dstPath string) error {
return os.Symlink(srcPathRel, dstPath)
}
// CopyDirectory copies all the files in srcPath to dstPath.
func CopyDirectory(srcPath, dstPath string) error {
fis, err := ioutil.ReadDir(srcPath)
if err != nil {
return err
}
if err := MkdirAllIfNotExist(dstPath); err != nil {
return err
}
for _, fi := range fis {
if !fi.Mode().IsRegular() {
// Skip non-files
continue
}
src := filepath.Join(srcPath, fi.Name())
dst := filepath.Join(dstPath, fi.Name())
if err := copyFile(src, dst); err != nil {
return err
}
}
MustSyncPath(dstPath)
return nil
}
func copyFile(srcPath, dstPath string) error {
src, err := os.Open(srcPath)
if err != nil {
return err
}
defer MustClose(src)
dst, err := os.Create(dstPath)
if err != nil {
return err
}
defer MustClose(dst)
if _, err := io.Copy(dst, src); err != nil {
return err
}
MustSyncPath(dstPath)
return nil
}
// ReadFullData reads len(data) bytes from r.
func ReadFullData(r io.Reader, data []byte) error {
n, err := io.ReadFull(r, data)

63
lib/storage/index_db.go
View file

@ -117,10 +117,13 @@ type indexDB struct {
// metricIDs, since it usually requires 1 bit per deleted metricID.
deletedMetricIDs atomic.Value
deletedMetricIDsUpdateLock sync.Mutex
// The minimum timestamp when queries with composite index can be used.
minTimestampForCompositeIndex int64
}
// openIndexDB opens index db from the given path with the given caches.
func openIndexDB(path string, metricIDCache, metricNameCache, tsidCache *workingsetcache.Cache) (*indexDB, error) {
func openIndexDB(path string, metricIDCache, metricNameCache, tsidCache *workingsetcache.Cache, minTimestampForCompositeIndex int64) (*indexDB, error) {
if metricIDCache == nil {
logger.Panicf("BUG: metricIDCache must be non-nil")
}
@ -152,6 +155,8 @@ func openIndexDB(path string, metricIDCache, metricNameCache, tsidCache *working
tsidCache: tsidCache,
uselessTagFiltersCache: workingsetcache.New(mem/128, time.Hour),
metricIDsPerDateTagFilterCache: workingsetcache.New(mem/128, time.Hour),
minTimestampForCompositeIndex: minTimestampForCompositeIndex,
}
is := db.getIndexSearch(0, 0, noDeadline)
@ -1651,6 +1656,9 @@ func (db *indexDB) searchTSIDs(tfss []*TagFilters, tr TimeRange, maxMetrics int,
if len(tfss) == 0 {
return nil, nil
}
if tr.MinTimestamp >= db.minTimestampForCompositeIndex {
tfss = convertToCompositeTagFilterss(tfss)
}
tfKeyBuf := tagFiltersKeyBufPool.Get()
defer tagFiltersKeyBufPool.Put(tfKeyBuf)
@ -1912,7 +1920,7 @@ func (is *indexSearch) updateMetricIDsByMetricNameMatch(metricIDs, srcMetricIDs
kb := &is.kb
kb.B = is.marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
tfs = fromCompositeTagFilters(tfs, kb.B)
tfs = removeCompositeTagFilters(tfs, kb.B)
metricName := kbPool.Get()
defer kbPool.Put(metricName)
@ -2115,17 +2123,21 @@ func (is *indexSearch) getTagFilterWithMinMetricIDsCount(tfs *TagFilters, maxMet
return nil, metricIDs, nil
}
func fromCompositeTagFilters(tfs []*tagFilter, prefix []byte) []*tagFilter {
tfsNew := make([]*tagFilter, 0, len(tfs))
func removeCompositeTagFilters(tfs []*tagFilter, prefix []byte) []*tagFilter {
if !hasCompositeTagFilters(tfs, prefix) {
return tfs
}
var tagKey []byte
var name []byte
tfsNew := make([]*tagFilter, 0, len(tfs)+1)
for _, tf := range tfs {
if !bytes.HasPrefix(tf.prefix, prefix) {
tfsNew = append(tfsNew, tf)
continue
}
suffix := tf.prefix[len(prefix):]
var tagKey, tail []byte
var err error
tail, tagKey, err = unmarshalTagValue(tagKey[:0], suffix)
_, tagKey, err = unmarshalTagValue(tagKey[:0], suffix)
if err != nil {
logger.Panicf("BUG: cannot unmarshal tag key from suffix=%q: %s", suffix, err)
}
@ -2139,20 +2151,49 @@ func fromCompositeTagFilters(tfs []*tagFilter, prefix []byte) []*tagFilter {
if err != nil {
logger.Panicf("BUG: cannot unmarshal nameLen from tagKey %q: %s", tagKey, err)
}
if nameLen == 0 {
logger.Panicf("BUG: nameLen must be greater than 0")
}
if uint64(len(tagKey)) < nameLen {
logger.Panicf("BUG: expecting at %d bytes for name in tagKey=%q; got %d bytes", nameLen, tagKey, len(tagKey))
}
name = append(name[:0], tagKey[:nameLen]...)
tagKey = tagKey[nameLen:]
tfNew := *tf
tfNew.key = append(tfNew.key[:0], tagKey...)
tfNew.prefix = append(tfNew.prefix[:0], prefix...)
tfNew.prefix = marshalTagValue(tfNew.prefix, tagKey)
tfNew.prefix = append(tfNew.prefix, tail...)
var tfNew tagFilter
if err := tfNew.Init(prefix, tagKey, tf.value, tf.isNegative, tf.isRegexp); err != nil {
logger.Panicf("BUG: cannot initialize {%s=%q} filter: %s", tagKey, tf.value, err)
}
tfsNew = append(tfsNew, &tfNew)
}
if len(name) > 0 {
var tfNew tagFilter
if err := tfNew.Init(prefix, nil, name, false, false); err != nil {
logger.Panicf("BUG: unexpected error when initializing {__name__=%q} filter: %s", name, err)
}
tfsNew = append(tfsNew, &tfNew)
}
return tfsNew
}
func hasCompositeTagFilters(tfs []*tagFilter, prefix []byte) bool {
var tagKey []byte
for _, tf := range tfs {
if !bytes.HasPrefix(tf.prefix, prefix) {
continue
}
suffix := tf.prefix[len(prefix):]
var err error
_, tagKey, err = unmarshalTagValue(tagKey[:0], suffix)
if err != nil {
logger.Panicf("BUG: cannot unmarshal tag key from suffix=%q: %s", suffix, err)
}
if len(tagKey) > 0 && tagKey[0] == compositeTagKeyPrefix {
return true
}
}
return false
}
func matchTagFilters(mn *MetricName, tfs []*tagFilter, kb *bytesutil.ByteBuffer) (bool, error) {
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs, mn.AccountID, mn.ProjectID)
for i, tf := range tfs {

10
lib/storage/index_db_test.go
View file

@ -472,7 +472,7 @@ func TestIndexDBOpenClose(t *testing.T) {
defer tsidCache.Stop()
for i := 0; i < 5; i++ {
db, err := openIndexDB("test-index-db", metricIDCache, metricNameCache, tsidCache)
db, err := openIndexDB("test-index-db", metricIDCache, metricNameCache, tsidCache, 0)
if err != nil {
t.Fatalf("cannot open indexDB: %s", err)
}
@ -497,7 +497,7 @@ func TestIndexDB(t *testing.T) {
defer tsidCache.Stop()
dbName := "test-index-db-serial"
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache)
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache, 0)
if err != nil {
t.Fatalf("cannot open indexDB: %s", err)
}
@ -527,7 +527,7 @@ func TestIndexDB(t *testing.T) {
// Re-open the db and verify it works as expected.
db.MustClose()
db, err = openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache)
db, err = openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache, 0)
if err != nil {
t.Fatalf("cannot open indexDB: %s", err)
}
@ -551,7 +551,7 @@ func TestIndexDB(t *testing.T) {
defer tsidCache.Stop()
dbName := "test-index-db-concurrent"
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache)
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache, 0)
if err != nil {
t.Fatalf("cannot open indexDB: %s", err)
}
@ -1536,7 +1536,7 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
defer tsidCache.Stop()
dbName := "test-index-db-ts-range"
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache)
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache, 0)
if err != nil {
t.Fatalf("cannot open indexDB: %s", err)
}

6
lib/storage/index_db_timing_test.go
View file

@ -51,7 +51,7 @@ func BenchmarkIndexDBAddTSIDs(b *testing.B) {
defer tsidCache.Stop()
const dbName = "bench-index-db-add-tsids"
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache)
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache, 0)
if err != nil {
b.Fatalf("cannot open indexDB: %s", err)
}
@ -119,7 +119,7 @@ func BenchmarkHeadPostingForMatchers(b *testing.B) {
defer tsidCache.Stop()
const dbName = "bench-head-posting-for-matchers"
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache)
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache, 0)
if err != nil {
b.Fatalf("cannot open indexDB: %s", err)
}
@ -302,7 +302,7 @@ func BenchmarkIndexDBGetTSIDs(b *testing.B) {
defer tsidCache.Stop()
const dbName = "bench-index-db-get-tsids"
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache)
db, err := openIndexDB(dbName, metricIDCache, metricNameCache, tsidCache, 0)
if err != nil {
b.Fatalf("cannot open indexDB: %s", err)
}

78
lib/storage/storage.go
View file

@ -108,6 +108,9 @@ type Storage struct {
// which may be in the process of flushing to disk by concurrently running
// snapshot process.
snapshotLock sync.Mutex
// The minimum timestamp when composite index search can be used.
minTimestampForCompositeIndex int64
}
type pendingHourMetricIDEntry struct {
@ -137,14 +140,9 @@ func OpenStorage(path string, retentionMsecs int64) (*Storage, error) {
stop: make(chan struct{}),
}
if err := fs.MkdirAllIfNotExist(path); err != nil {
return nil, fmt.Errorf("cannot create a directory for the storage at %q: %w", path, err)
}
snapshotsPath := path + "/snapshots"
if err := fs.MkdirAllIfNotExist(snapshotsPath); err != nil {
return nil, fmt.Errorf("cannot create %q: %w", snapshotsPath, err)
}
// Protect from concurrent opens.
flockF, err := fs.CreateFlockFile(path)
@ -153,6 +151,12 @@ func OpenStorage(path string, retentionMsecs int64) (*Storage, error) {
}
s.flockF = flockF
// Pre-create snapshots directory if it is missing.
snapshotsPath := path + "/snapshots"
if err := fs.MkdirAllIfNotExist(snapshotsPath); err != nil {
return nil, fmt.Errorf("cannot create %q: %w", snapshotsPath, err)
}
// Load caches.
mem := memory.Allowed()
s.tsidCache = s.mustLoadCache("MetricName->TSID", "metricName_tsid", mem/3)
@ -176,10 +180,11 @@ func OpenStorage(path string, retentionMsecs int64) (*Storage, error) {
// Load indexdb
idbPath := path + "/indexdb"
idbSnapshotsPath := idbPath + "/snapshots"
isEmptyDB := !fs.IsPathExist(idbPath)
if err := fs.MkdirAllIfNotExist(idbSnapshotsPath); err != nil {
return nil, fmt.Errorf("cannot create %q: %w", idbSnapshotsPath, err)
}
idbCurr, idbPrev, err := openIndexDBTables(idbPath, s.metricIDCache, s.metricNameCache, s.tsidCache)
idbCurr, idbPrev, err := openIndexDBTables(idbPath, s.metricIDCache, s.metricNameCache, s.tsidCache, s.minTimestampForCompositeIndex)
if err != nil {
return nil, fmt.Errorf("cannot open indexdb tables at %q: %w", idbPath, err)
}
@ -195,6 +200,13 @@ func OpenStorage(path string, retentionMsecs int64) (*Storage, error) {
}
s.tb = tb
// Load metadata
metadataDir := path + "/metadata"
if err := fs.MkdirAllIfNotExist(metadataDir); err != nil {
return nil, fmt.Errorf("cannot create %q: %w", metadataDir, err)
}
s.minTimestampForCompositeIndex = mustGetMinTimestampForCompositeIndex(metadataDir, isEmptyDB)
s.startCurrHourMetricIDsUpdater()
s.startNextDayMetricIDsUpdater()
s.startRetentionWatcher()
@ -250,7 +262,7 @@ func (s *Storage) CreateSnapshot() (string, error) {
}
fs.MustSyncPath(dstDataDir)
idbSnapshot := fmt.Sprintf("%s/indexdb/snapshots/%s", s.path, snapshotName)
idbSnapshot := fmt.Sprintf("%s/indexdb/snapshots/%s", srcDir, snapshotName)
idb := s.idb()
currSnapshot := idbSnapshot + "/" + idb.name
if err := idb.tb.CreateSnapshotAt(currSnapshot); err != nil {
@ -268,8 +280,13 @@ func (s *Storage) CreateSnapshot() (string, error) {
return "", fmt.Errorf("cannot create symlink from %q to %q: %w", idbSnapshot, dstIdbDir, err)
}
srcMetadataDir := srcDir + "/metadata"
dstMetadataDir := dstDir + "/metadata"
if err := fs.CopyDirectory(srcMetadataDir, dstMetadataDir); err != nil {
return "", fmt.Errorf("cannot copy metadata: %s", err)
}
fs.MustSyncPath(dstDir)
fs.MustSyncPath(srcDir + "/snapshots")
logger.Infof("created Storage snapshot for %q at %q in %.3f seconds", srcDir, dstDir, time.Since(startTime).Seconds())
return snapshotName, nil
@ -552,7 +569,7 @@ func (s *Storage) mustRotateIndexDB() {
// Create new indexdb table.
newTableName := nextIndexDBTableName()
idbNewPath := s.path + "/indexdb/" + newTableName
idbNew, err := openIndexDB(idbNewPath, s.metricIDCache, s.metricNameCache, s.tsidCache)
idbNew, err := openIndexDB(idbNewPath, s.metricIDCache, s.metricNameCache, s.tsidCache, s.minTimestampForCompositeIndex)
if err != nil {
logger.Panicf("FATAL: cannot create new indexDB at %q: %s", idbNewPath, err)
}
@ -821,6 +838,43 @@ func marshalUint64Set(dst []byte, m *uint64set.Set) []byte {
return dst
}
func mustGetMinTimestampForCompositeIndex(metadataDir string, isEmptyDB bool) int64 {
path := metadataDir + "/minTimestampForCompositeIndex"
minTimestamp, err := loadMinTimestampForCompositeIndex(path)
if err == nil {
return minTimestamp
}
logger.Errorf("cannot read minTimestampForCompositeIndex, so trying to re-create it; error: %s", err)
date := time.Now().UnixNano() / 1e6 / msecPerDay
if !isEmptyDB {
// The current and the next day can already contain non-composite indexes,
// so they cannot be queried with composite indexes.
date += 2
} else {
date = 0
}
minTimestamp = date * msecPerDay
dateBuf := encoding.MarshalInt64(nil, minTimestamp)
if err := os.RemoveAll(path); err != nil {
logger.Fatalf("cannot remove a file with minTimestampForCompositeIndex: %s", err)
}
if err := fs.WriteFileAtomically(path, dateBuf); err != nil {
logger.Fatalf("cannot store minTimestampForCompositeIndex: %s", err)
}
return minTimestamp
}
func loadMinTimestampForCompositeIndex(path string) (int64, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
return 0, err
}
if len(data) != 8 {
return 0, fmt.Errorf("unexpected length of %q; got %d bytes; want 8 bytes", path, len(data))
}
return encoding.UnmarshalInt64(data), nil
}
func (s *Storage) mustLoadCache(info, name string, sizeBytes int) *workingsetcache.Cache {
path := s.cachePath + "/" + name
logger.Infof("loading %s cache from %q...", info, path)
@ -2076,7 +2130,7 @@ func (s *Storage) putTSIDToCache(tsid *TSID, metricName []byte) {
s.tsidCache.Set(metricName, buf)
}
func openIndexDBTables(path string, metricIDCache, metricNameCache, tsidCache *workingsetcache.Cache) (curr, prev *indexDB, err error) {
func openIndexDBTables(path string, metricIDCache, metricNameCache, tsidCache *workingsetcache.Cache, minTimestampForCompositeIndex int64) (curr, prev *indexDB, err error) {
if err := fs.MkdirAllIfNotExist(path); err != nil {
return nil, nil, fmt.Errorf("cannot create directory %q: %w", path, err)
}
@ -2135,12 +2189,12 @@ func openIndexDBTables(path string, metricIDCache, metricNameCache, tsidCache *w
// Open the last two tables.
currPath := path + "/" + tableNames[len(tableNames)-1]
curr, err = openIndexDB(currPath, metricIDCache, metricNameCache, tsidCache)
curr, err = openIndexDB(currPath, metricIDCache, metricNameCache, tsidCache, minTimestampForCompositeIndex)
if err != nil {
return nil, nil, fmt.Errorf("cannot open curr indexdb table at %q: %w", currPath, err)
}
prevPath := path + "/" + tableNames[len(tableNames)-2]
prev, err = openIndexDB(prevPath, metricIDCache, metricNameCache, tsidCache)
prev, err = openIndexDB(prevPath, metricIDCache, metricNameCache, tsidCache, minTimestampForCompositeIndex)
if err != nil {
curr.MustClose()
return nil, nil, fmt.Errorf("cannot open prev indexdb table at %q: %w", prevPath, err)

45
lib/storage/tag_filters.go
View file

@ -15,15 +15,23 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
)
// ConvertToCompositeTagFilters converts tfs to composite filters.
// convertToCompositeTagFilterss converts tfss to composite filters.
//
// This converts `foo{bar="baz",x=~"a.+"}` to `{foo=bar="baz",foo=x=~"a.+"} filter.
func ConvertToCompositeTagFilters(tfs []TagFilter) []TagFilter {
func convertToCompositeTagFilterss(tfss []*TagFilters) []*TagFilters {
tfssNew := make([]*TagFilters, len(tfss))
for i, tfs := range tfss {
tfssNew[i] = convertToCompositeTagFilters(tfs)
}
return tfssNew
}
func convertToCompositeTagFilters(tfs *TagFilters) *TagFilters {
// Search for metric name filter, which must be used for creating composite filters.
var name []byte
for _, tf := range tfs {
if len(tf.Key) == 0 && !tf.IsNegative && !tf.IsRegexp {
name = tf.Value
for _, tf := range tfs.tfs {
if len(tf.key) == 0 && !tf.isNegative && !tf.isRegexp {
name = tf.value
break
}
}
@ -31,33 +39,34 @@ func ConvertToCompositeTagFilters(tfs []TagFilter) []TagFilter {
// There is no metric name filter, so composite filters cannot be created.
return tfs
}
tfsNew := make([]TagFilter, 0, len(tfs))
tfsNew := make([]tagFilter, 0, len(tfs.tfs))
var compositeKey []byte
compositeFilters := 0
for _, tf := range tfs {
if len(tf.Key) == 0 {
if tf.IsNegative || tf.IsRegexp || string(tf.Value) != string(name) {
for _, tf := range tfs.tfs {
if len(tf.key) == 0 {
if tf.isNegative || tf.isRegexp || string(tf.value) != string(name) {
tfsNew = append(tfsNew, tf)
}
continue
}
if string(tf.Key) == "__graphite__" {
if string(tf.key) == "__graphite__" {
tfsNew = append(tfsNew, tf)
continue
}
compositeKey = marshalCompositeTagKey(compositeKey[:0], name, tf.Key)
tfsNew = append(tfsNew, TagFilter{
Key: append([]byte{}, compositeKey...),
Value: append([]byte{}, tf.Value...),
IsNegative: tf.IsNegative,
IsRegexp: tf.IsRegexp,
})
compositeKey = marshalCompositeTagKey(compositeKey[:0], name, tf.key)
var tfNew tagFilter
if err := tfNew.Init(tfs.commonPrefix, compositeKey, tf.value, tf.isNegative, tf.isRegexp); err != nil {
logger.Panicf("BUG: unexpected error when creating composite tag filter for name=%q and key=%q: %s", name, tf.key, err)
}
tfsNew = append(tfsNew, tfNew)
compositeFilters++
}
if compositeFilters == 0 {
return tfs
}
return tfsNew
tfsCompiled := NewTagFilters(tfs.accountID, tfs.projectID)
tfsCompiled.tfs = tfsNew
return tfsCompiled
}
// TagFilters represents filters used for filtering tags.

105
lib/storage/tag_filters_test.go
View file

@ -9,13 +9,37 @@ import (
func TestConvertToCompositeTagFilters(t *testing.T) {
f := func(tfs, resultExpected []TagFilter) {
t.Helper()
result := ConvertToCompositeTagFilters(tfs)
accountID := uint32(123)
projectID := uint32(456)
tfsCompiled := NewTagFilters(accountID, projectID)
for _, tf := range tfs {
if err := tfsCompiled.Add(tf.Key, tf.Value, tf.IsNegative, tf.IsRegexp); err != nil {
t.Fatalf("cannot add tf=%s: %s", tf.String(), err)
}
}
resultCompiled := convertToCompositeTagFilters(tfsCompiled)
if resultCompiled.accountID != accountID {
t.Fatalf("unexpected accountID; got %d; want %d", resultCompiled.accountID, accountID)
}
if resultCompiled.projectID != projectID {
t.Fatalf("unexpected projectID; got %d; want %d", resultCompiled.projectID, projectID)
}
result := make([]TagFilter, len(resultCompiled.tfs))
for i, tf := range resultCompiled.tfs {
result[i] = TagFilter{
Key: tf.key,
Value: tf.value,
IsNegative: tf.isNegative,
IsRegexp: tf.isRegexp,
}
}
if !reflect.DeepEqual(result, resultExpected) {
t.Fatalf("unexpected result;\ngot\n%+v\nwant\n%+v", result, resultExpected)
}
}
// Empty filters
f(nil, nil)
f(nil, []TagFilter{})
// A single non-name filter
f([]TagFilter{
@ -167,7 +191,7 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
},
})
// A name filter with negative regexp non-name filter.
// A name filter with negative regexp non-name filter, which can be converted to non-regexp.
f([]TagFilter{
{
Key: nil,
@ -186,6 +210,29 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
Key: []byte("\xfe\x03barfoo"),
Value: []byte("abc"),
IsNegative: true,
IsRegexp: false,
},
})
// A name filter with negative regexp non-name filter.
f([]TagFilter{
{
Key: nil,
Value: []byte("bar"),
IsNegative: false,
IsRegexp: false,
},
{
Key: []byte("foo"),
Value: []byte("abc.+"),
IsNegative: true,
IsRegexp: true,
},
}, []TagFilter{
{
Key: []byte("\xfe\x03barfoo"),
Value: []byte("abc.+"),
IsNegative: true,
IsRegexp: true,
},
})
@ -244,7 +291,7 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
Key: []byte("\xfe\x03barfoo"),
Value: []byte("abc"),
IsNegative: true,
IsRegexp: true,
IsRegexp: false,
},
{
Key: []byte("__graphite__"),
@ -254,7 +301,7 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
},
})
// Regexp name filter with non-name filter.
// Regexp name filter, which can be converted to non-regexp, with non-name filter.
f([]TagFilter{
{
Key: nil,
@ -269,9 +316,19 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsRegexp: false,
},
}, []TagFilter{
{
Key: []byte("\xfe\x03barfoo"),
Value: []byte("abc"),
IsNegative: true,
IsRegexp: false,
},
})
// Regexp name filter with non-name filter.
f([]TagFilter{
{
Key: nil,
Value: []byte("bar"),
Value: []byte("bar.+"),
IsNegative: false,
IsRegexp: true,
},
@ -281,6 +338,42 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: true,
IsRegexp: false,
},
}, []TagFilter{
{
Key: nil,
Value: []byte("bar.+"),
IsNegative: false,
IsRegexp: true,
},
{
Key: []byte("foo"),
Value: []byte("abc"),
IsNegative: true,
IsRegexp: false,
},
})
// Regexp non-name filter, which matches anything.
f([]TagFilter{
{
Key: nil,
Value: []byte("bar"),
IsNegative: false,
IsRegexp: false,
},
{
Key: []byte("foo"),
Value: []byte(".*"),
IsNegative: false,
IsRegexp: true,
},
}, []TagFilter{
{
Key: nil,
Value: []byte("bar"),
IsNegative: false,
IsRegexp: false,
},
})
}