github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-02-19 15:30:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

Browse source
This commit is contained in:
Aliaksandr Valialkin 2020-10-07 21:40:45 +03:00
commit e7f2907138
11 changed files with 119 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/main.yml vendored
View file

@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Setup Go
uses: actions/setup-go@master
uses: actions/setup-go@main
with:
go-version: 1.15
id: go

9
CHANGELOG.md
View file

@ -1,5 +1,14 @@
# tip
* FEATURE: automatically add missing label filters to binary operands as described at https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization .
This should improve performance for queries with missing label filters in binary operands. For example, the following query should work faster now, because it shouldn't
fetch and discard time series for `node_filesystem_files_free` metric without matching labels for the left side of the expression:
```
node_filesystem_files{ host="$host", mountpoint="/" } - node_filesystem_files_free
```
* FEATURE: add `-finalMergeDelay` command-line flag for configuring the delay before final merge for per-month partitions.
The final merge is started after no new data is ingested into per-month partition during `-finalMergeDelay`.
# [v1.43.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.43.0)

4
app/vmalert/datasource/vm.go
View file

@ -93,7 +93,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
body, _ := ioutil.ReadAll(resp.Body)
return nil, fmt.Errorf("datasource returns unexpected response code %d for %s. Reponse body %s", resp.StatusCode, req.URL, body)
return nil, fmt.Errorf("datasource returns unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
}
r := &response{}
if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
@ -106,7 +106,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
return nil, fmt.Errorf("unknown status: %s, Expected success or error ", r.Status)
}
if r.Data.ResultType != rtVector {
return nil, fmt.Errorf("unknown restul type:%s. Expected vector", r.Data.ResultType)
return nil, fmt.Errorf("unknown result type:%s. Expected vector", r.Data.ResultType)
}
return r.metrics()
}

2
app/vmalert/datasource/vm_test.go
View file

@ -75,7 +75,7 @@ func TestVMSelectQuery(t *testing.T) {
t.Fatalf("expected error status got nil")
}
if _, err := am.Query(ctx, query); err == nil {
t.Fatalf("expected unkown status got nil")
t.Fatalf("expected unknown status got nil")
}
if _, err := am.Query(ctx, query); err == nil {
t.Fatalf("expected non-vector resultType error got nil")

1
app/vmselect/promql/exec.go
View file

@ -175,6 +175,7 @@ func parsePromQLWithCache(q string) (metricsql.Expr, error) {
if pcv == nil {
e, err := metricsql.Parse(q)
if err == nil {
e = metricsql.Optimize(e)
e = adjustCmpOps(e)
}
pcv = &parseCacheValue{

4
app/vmstorage/main.go
View file

@ -29,6 +29,9 @@ var (
// DataPath is a path to storage data.
DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
finalMergeDelay = flag.Duration("finalMergeDelay", 30*time.Second, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
"Query speed and disk space usage is usually reduced after the final merge is complete. Too low delay for final merge may result in increased "+
"disk IO usage and CPU usage")
bigMergeConcurrency = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
@ -66,6 +69,7 @@ func InitWithoutMetrics() {
logger.Fatalf("invalid `-precisionBits`: %s", err)
}
storage.SetFinalMergeDelay(*finalMergeDelay)
storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
storage.SetSmallMergeWorkersCount(*smallMergeConcurrency)

2
go.mod
View file

@ -9,7 +9,7 @@ require (
// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
github.com/VictoriaMetrics/fasthttp v1.0.7
github.com/VictoriaMetrics/metrics v1.12.3
github.com/VictoriaMetrics/metricsql v0.6.0
github.com/VictoriaMetrics/metricsql v0.7.0
github.com/aws/aws-sdk-go v1.35.3
github.com/cespare/xxhash/v2 v2.1.1
github.com/go-kit/kit v0.10.0

7
go.sum
View file

@ -83,8 +83,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.7/go.mod h1:eaGv8oDDOWE8JQa6GgBgpCwjJhR
github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I=
github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
github.com/VictoriaMetrics/metricsql v0.6.0 h1:JnHUmifuA3fdy1GQrmkZJFO+CwFrhLxKwzMv89wNgJ4=
github.com/VictoriaMetrics/metricsql v0.6.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
github.com/VictoriaMetrics/metricsql v0.7.0 h1:YR/OvbsCH0dwUuc3r5GayTcuTdgWHJZo+4bqbaGl7WM=
github.com/VictoriaMetrics/metricsql v0.7.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
@ -833,7 +833,6 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200927032502-5d4f70055728 h1:5wtQIAulKU5AbLQOkjxl32UufnIOqgBX72pS0AV14H0=
golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c h1:dk0ukUIHmGHqASjP0iue2261isepFCC6XRCSd1nHgDw=
golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c/go.mod h1:iQL9McJNjoIa5mjH6nYTCTZXUN6RP+XW3eib7Ya3XcI=
@ -852,7 +851,6 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6cnrWpidecD1UuSYbhKm0=
golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -1079,7 +1077,6 @@ google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201002142447-3860012362da h1:DTQYk4u7nICKkkVZsBv0/0po0ChISxAJ5CTAfUhO0PQ=
google.golang.org/genproto v0.0.0-20201002142447-3860012362da/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201006033701-bcad7cf615f2 h1:rBG1miiV00OG3NTbgxg42kPWohjVNn1sGhJyt4xyPLQ=
google.golang.org/genproto v0.0.0-20201006033701-bcad7cf615f2/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=

15
lib/storage/partition.go
View file

@ -958,7 +958,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
if !errors.Is(err, errNothingToMerge) {
return err
}
if fasttime.UnixTimestamp()-lastMergeTime > 30 {
if fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds {
// We have free time for merging into bigger parts.
// This should improve select performance.
lastMergeTime = fasttime.UnixTimestamp()
@ -980,6 +980,19 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
}
}
var finalMergeDelaySeconds = uint64(30)
// SetFinalMergeDelay sets the delay before doing final merge for partitions without newly ingested data.
//
// This function may be called only before Storage initialization.
func SetFinalMergeDelay(delay time.Duration) {
delaySeconds := int(delay.Seconds() + 0.5)
if delaySeconds <= 0 {
return
}
finalMergeDelaySeconds = uint64(delaySeconds)
}
func maxRowsByPath(path string) uint64 {
freeSpace := fs.MustGetFreeSpace(path)

83
vendor/github.com/VictoriaMetrics/metricsql/optimizer.go generated vendored Normal file
View file

@ -0,0 +1,83 @@
package metricsql
import (
"sort"
)
// Optimize optimizes e in order to improve its performance.
func Optimize(e Expr) Expr {
switch t := e.(type) {
case *BinaryOpExpr:
// Convert `foo{filters1} op bar{filters2}` to `foo{filters1, filters2} op bar{filters1, filters2}`.
// This should reduce the number of operations
// See https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization
// for details.
switch t.Op {
case "+", "-", "*", "/", "%", "^",
"==", "!=", ">", "<", ">=", "<=",
"if", "ifnot", "default":
// The optimization can be applied only to these operations.
default:
return optimizeBinaryOpArgs(t)
}
if t.JoinModifier.Op != "" {
return optimizeBinaryOpArgs(t)
}
if t.GroupModifier.Op != "" {
return optimizeBinaryOpArgs(t)
}
meLeft, ok := t.Left.(*MetricExpr)
if !ok || !meLeft.hasNonEmptyMetricGroup() {
return optimizeBinaryOpArgs(t)
}
meRight, ok := t.Right.(*MetricExpr)
if !ok || !meRight.hasNonEmptyMetricGroup() {
return optimizeBinaryOpArgs(t)
}
lfs := intersectLabelFilters(meLeft.LabelFilters[1:], meRight.LabelFilters[1:])
meLeft.LabelFilters = append(meLeft.LabelFilters[:1], lfs...)
meRight.LabelFilters = append(meRight.LabelFilters[:1], lfs...)
return t
case *FuncExpr:
for i := range t.Args {
t.Args[i] = Optimize(t.Args[i])
}
return t
case *AggrFuncExpr:
for i := range t.Args {
t.Args[i] = Optimize(t.Args[i])
}
return t
default:
return e
}
}
func optimizeBinaryOpArgs(be *BinaryOpExpr) *BinaryOpExpr {
be.Left = Optimize(be.Left)
be.Right = Optimize(be.Right)
return be
}
func intersectLabelFilters(a, b []LabelFilter) []LabelFilter {
m := make(map[string]LabelFilter, len(a)+len(b))
var buf []byte
for _, lf := range a {
buf = lf.AppendString(buf[:0])
m[string(buf)] = lf
}
for _, lf := range b {
buf = lf.AppendString(buf[:0])
m[string(buf)] = lf
}
ss := make([]string, 0, len(m))
for s := range m {
ss = append(ss, s)
}
sort.Strings(ss)
lfs := make([]LabelFilter, 0, len(ss))
for _, s := range ss {
lfs = append(lfs, m[s])
}
return lfs
}

2
vendor/modules.txt vendored
View file

@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil
github.com/VictoriaMetrics/fasthttp/stackless
# github.com/VictoriaMetrics/metrics v1.12.3
github.com/VictoriaMetrics/metrics
# github.com/VictoriaMetrics/metricsql v0.6.0
# github.com/VictoriaMetrics/metricsql v0.7.0
github.com/VictoriaMetrics/metricsql
github.com/VictoriaMetrics/metricsql/binaryop
# github.com/aws/aws-sdk-go v1.35.3