Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

This commit is contained in:
Aliaksandr Valialkin 2022-01-17 15:51:07 +02:00
commit b894f25f21
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
136 changed files with 20131 additions and 7477 deletions

View file

@ -356,6 +356,7 @@ var (
execErrors = metrics.NewCounter(`vmalert_execution_errors_total`) execErrors = metrics.NewCounter(`vmalert_execution_errors_total`)
remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`) remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
remoteWriteTotal = metrics.NewCounter(`vmalert_remotewrite_total`)
) )
func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Duration) error { func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Duration) error {
@ -369,6 +370,7 @@ func (e *executor) exec(ctx context.Context, rule Rule, resolveDuration time.Dur
if len(tss) > 0 && e.rw != nil { if len(tss) > 0 && e.rw != nil {
for _, ts := range tss { for _, ts := range tss {
remoteWriteTotal.Inc()
if err := e.rw.Push(ts); err != nil { if err := e.rw.Push(ts); err != nil {
remoteWriteErrors.Inc() remoteWriteErrors.Inc()
return fmt.Errorf("rule %q: remote write failure: %w", rule, err) return fmt.Errorf("rule %q: remote write failure: %w", rule, err)

View file

@ -17,6 +17,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"math" "math"
"net"
"net/url" "net/url"
"regexp" "regexp"
"strings" "strings"
@ -26,6 +27,7 @@ import (
textTpl "text/template" textTpl "text/template"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
"github.com/VictoriaMetrics/metricsql"
) )
// metric is private copy of datasource.Metric, // metric is private copy of datasource.Metric,
@ -61,6 +63,7 @@ var tmplFunc textTpl.FuncMap
// InitTemplateFunc initiates template helper functions // InitTemplateFunc initiates template helper functions
func InitTemplateFunc(externalURL *url.URL) { func InitTemplateFunc(externalURL *url.URL) {
// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
tmplFunc = textTpl.FuncMap{ tmplFunc = textTpl.FuncMap{
/* Strings */ /* Strings */
@ -91,6 +94,24 @@ func InitTemplateFunc(externalURL *url.URL) {
// alias for https://golang.org/pkg/strings/#ToLower // alias for https://golang.org/pkg/strings/#ToLower
"toLower": strings.ToLower, "toLower": strings.ToLower,
// stripPort splits string into host and port, then returns only host.
"stripPort": func(hostPort string) string {
host, _, err := net.SplitHostPort(hostPort)
if err != nil {
return hostPort
}
return host
},
// parseDuration parses a duration string such as "1h" into the number of seconds it represents
"parseDuration": func(d string) (float64, error) {
ms, err := metricsql.DurationValue(d, 0)
if err != nil {
return 0, err
}
return float64(ms) / 1000, nil
},
/* Numbers */ /* Numbers */
// humanize converts given number to a human readable format // humanize converts given number to a human readable format

View file

@ -391,7 +391,7 @@ func tryGetArgRollupFuncWithMetricExpr(ae *metricsql.AggrFuncExpr) (*metricsql.F
if nrf == nil { if nrf == nil {
return nil, nil return nil, nil
} }
rollupArgIdx := getRollupArgIdx(fe) rollupArgIdx := metricsql.GetRollupArgIdx(fe)
if rollupArgIdx >= len(fe.Args) { if rollupArgIdx >= len(fe.Args) {
// Incorrect number of args for rollup func. // Incorrect number of args for rollup func.
return nil, nil return nil, nil
@ -431,7 +431,7 @@ func evalExprs(ec *EvalConfig, es []metricsql.Expr) ([][]*timeseries, error) {
func evalRollupFuncArgs(ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) { func evalRollupFuncArgs(ec *EvalConfig, fe *metricsql.FuncExpr) ([]interface{}, *metricsql.RollupExpr, error) {
var re *metricsql.RollupExpr var re *metricsql.RollupExpr
rollupArgIdx := getRollupArgIdx(fe) rollupArgIdx := metricsql.GetRollupArgIdx(fe)
if len(fe.Args) <= rollupArgIdx { if len(fe.Args) <= rollupArgIdx {
return nil, nil, fmt.Errorf("expecting at least %d args to %q; got %d args; expr: %q", rollupArgIdx+1, fe.Name, len(fe.Args), fe.AppendString(nil)) return nil, nil, fmt.Errorf("expecting at least %d args to %q; got %d args; expr: %q", rollupArgIdx+1, fe.Name, len(fe.Args), fe.AppendString(nil))
} }
@ -479,7 +479,43 @@ func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr {
return &reNew return &reNew
} }
// expr may contain:
// - rollupFunc(m) if iafc is nil
// - aggrFunc(rollupFunc(m)) if iafc isn't nil
func evalRollupFunc(ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) { func evalRollupFunc(ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
if re.At == nil {
return evalRollupFuncWithoutAt(ec, funcName, rf, expr, re, iafc)
}
tssAt, err := evalExpr(ec, re.At)
if err != nil {
return nil, fmt.Errorf("cannot evaluate `@` modifier: %w", err)
}
if len(tssAt) != 1 {
return nil, fmt.Errorf("`@` modifier must return a single series; it returns %d series instead", len(tssAt))
}
atTimestamp := int64(tssAt[0].Values[0] * 1000)
ecNew := newEvalConfig(ec)
ecNew.Start = atTimestamp
ecNew.End = atTimestamp
tss, err := evalRollupFuncWithoutAt(ecNew, funcName, rf, expr, re, iafc)
if err != nil {
return nil, err
}
// expand single-point tss to the original time range.
timestamps := ec.getSharedTimestamps()
for _, ts := range tss {
v := ts.Values[0]
values := make([]float64, len(timestamps))
for i := range timestamps {
values[i] = v
}
ts.Timestamps = timestamps
ts.Values = values
}
return tss, nil
}
func evalRollupFuncWithoutAt(ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
funcName = strings.ToLower(funcName) funcName = strings.ToLower(funcName)
ecNew := ec ecNew := ec
var offset int64 var offset int64
@ -565,11 +601,12 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, funcName string, rf rollupFunc,
} }
tss := make([]*timeseries, 0, len(tssSQ)*len(rcs)) tss := make([]*timeseries, 0, len(tssSQ)*len(rcs))
var tssLock sync.Mutex var tssLock sync.Mutex
keepMetricNames := getKeepMetricNames(expr)
doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64) ([]float64, []int64) { doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64) ([]float64, []int64) {
values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps) values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps)
preFunc(values, timestamps) preFunc(values, timestamps)
for _, rc := range rcs { for _, rc := range rcs {
if tsm := newTimeseriesMap(funcName, sharedTimestamps, &tsSQ.MetricName); tsm != nil { if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &tsSQ.MetricName); tsm != nil {
rc.DoTimeseriesMap(tsm, values, timestamps) rc.DoTimeseriesMap(tsm, values, timestamps)
tssLock.Lock() tssLock.Lock()
tss = tsm.AppendTimeseriesTo(tss) tss = tsm.AppendTimeseriesTo(tss)
@ -577,7 +614,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, funcName string, rf rollupFunc,
continue continue
} }
var ts timeseries var ts timeseries
doRollupForTimeseries(funcName, rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps) doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps)
tssLock.Lock() tssLock.Lock()
tss = append(tss, &ts) tss = append(tss, &ts)
tssLock.Unlock() tssLock.Unlock()
@ -587,6 +624,22 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, funcName string, rf rollupFunc,
return tss, nil return tss, nil
} }
func getKeepMetricNames(expr metricsql.Expr) bool {
if ae, ok := expr.(*metricsql.AggrFuncExpr); ok {
// Extract rollupFunc(...) from aggrFunc(rollupFunc(...)).
// This case is possible when optimized aggrFunc calculations are used
// such as `sum(rate(...))`
if len(ae.Args) != 1 {
return false
}
expr = ae.Args[0]
}
if fe, ok := expr.(*metricsql.FuncExpr); ok {
return fe.KeepMetricNames
}
return false
}
func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, timestamps []int64) ([]float64, []int64)) { func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, timestamps []int64) ([]float64, []int64)) {
concurrency := cgroup.AvailableCPUs() concurrency := cgroup.AvailableCPUs()
if concurrency > len(tss) { if concurrency > len(tss) {
@ -736,11 +789,12 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, funcName string, rf rollupFunc
defer rml.Put(uint64(rollupMemorySize)) defer rml.Put(uint64(rollupMemorySize))
// Evaluate rollup // Evaluate rollup
keepMetricNames := getKeepMetricNames(expr)
var tss []*timeseries var tss []*timeseries
if iafc != nil { if iafc != nil {
tss, err = evalRollupWithIncrementalAggregate(funcName, iafc, rss, rcs, preFunc, sharedTimestamps) tss, err = evalRollupWithIncrementalAggregate(funcName, keepMetricNames, iafc, rss, rcs, preFunc, sharedTimestamps)
} else { } else {
tss, err = evalRollupNoIncrementalAggregate(funcName, rss, rcs, preFunc, sharedTimestamps) tss, err = evalRollupNoIncrementalAggregate(funcName, keepMetricNames, rss, rcs, preFunc, sharedTimestamps)
} }
if err != nil { if err != nil {
return nil, err return nil, err
@ -762,7 +816,7 @@ func getRollupMemoryLimiter() *memoryLimiter {
return &rollupMemoryLimiter return &rollupMemoryLimiter
} }
func evalRollupWithIncrementalAggregate(funcName string, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig, func evalRollupWithIncrementalAggregate(funcName string, keepMetricNames bool, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) { preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error { err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps) rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
@ -770,7 +824,7 @@ func evalRollupWithIncrementalAggregate(funcName string, iafc *incrementalAggrFu
ts := getTimeseries() ts := getTimeseries()
defer putTimeseries(ts) defer putTimeseries(ts)
for _, rc := range rcs { for _, rc := range rcs {
if tsm := newTimeseriesMap(funcName, sharedTimestamps, &rs.MetricName); tsm != nil { if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &rs.MetricName); tsm != nil {
rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps) rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
for _, ts := range tsm.m { for _, ts := range tsm.m {
iafc.updateTimeseries(ts, workerID) iafc.updateTimeseries(ts, workerID)
@ -778,7 +832,7 @@ func evalRollupWithIncrementalAggregate(funcName string, iafc *incrementalAggrFu
continue continue
} }
ts.Reset() ts.Reset()
doRollupForTimeseries(funcName, rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps) doRollupForTimeseries(funcName, keepMetricNames, rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
iafc.updateTimeseries(ts, workerID) iafc.updateTimeseries(ts, workerID)
// ts.Timestamps points to sharedTimestamps. Zero it, so it can be re-used. // ts.Timestamps points to sharedTimestamps. Zero it, so it can be re-used.
@ -794,7 +848,7 @@ func evalRollupWithIncrementalAggregate(funcName string, iafc *incrementalAggrFu
return tss, nil return tss, nil
} }
func evalRollupNoIncrementalAggregate(funcName string, rss *netstorage.Results, rcs []*rollupConfig, func evalRollupNoIncrementalAggregate(funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) { preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
tss := make([]*timeseries, 0, rss.Len()*len(rcs)) tss := make([]*timeseries, 0, rss.Len()*len(rcs))
var tssLock sync.Mutex var tssLock sync.Mutex
@ -802,7 +856,7 @@ func evalRollupNoIncrementalAggregate(funcName string, rss *netstorage.Results,
rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps) rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
preFunc(rs.Values, rs.Timestamps) preFunc(rs.Values, rs.Timestamps)
for _, rc := range rcs { for _, rc := range rcs {
if tsm := newTimeseriesMap(funcName, sharedTimestamps, &rs.MetricName); tsm != nil { if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &rs.MetricName); tsm != nil {
rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps) rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
tssLock.Lock() tssLock.Lock()
tss = tsm.AppendTimeseriesTo(tss) tss = tsm.AppendTimeseriesTo(tss)
@ -810,7 +864,7 @@ func evalRollupNoIncrementalAggregate(funcName string, rss *netstorage.Results,
continue continue
} }
var ts timeseries var ts timeseries
doRollupForTimeseries(funcName, rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps) doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
tssLock.Lock() tssLock.Lock()
tss = append(tss, &ts) tss = append(tss, &ts)
tssLock.Unlock() tssLock.Unlock()
@ -823,13 +877,13 @@ func evalRollupNoIncrementalAggregate(funcName string, rss *netstorage.Results,
return tss, nil return tss, nil
} }
func doRollupForTimeseries(funcName string, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName, valuesSrc []float64, timestampsSrc []int64, func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName,
sharedTimestamps []int64) { valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64) {
tsDst.MetricName.CopyFrom(mnSrc) tsDst.MetricName.CopyFrom(mnSrc)
if len(rc.TagValue) > 0 { if len(rc.TagValue) > 0 {
tsDst.MetricName.AddTag("rollup", rc.TagValue) tsDst.MetricName.AddTag("rollup", rc.TagValue)
} }
if !rollupFuncsKeepMetricGroup[funcName] { if !keepMetricNames && !rollupFuncsKeepMetricName[funcName] {
tsDst.MetricName.ResetMetricGroup() tsDst.MetricName.ResetMetricGroup()
} }
tsDst.Values = rc.Do(tsDst.Values[:0], valuesSrc, timestampsSrc) tsDst.Values = rc.Do(tsDst.Values[:0], valuesSrc, timestampsSrc)
@ -877,9 +931,11 @@ func mulNoOverflow(a, b int64) int64 {
} }
func dropStaleNaNs(funcName string, values []float64, timestamps []int64) ([]float64, []int64) { func dropStaleNaNs(funcName string, values []float64, timestamps []int64) ([]float64, []int64) {
if *noStaleMarkers || funcName == "default_rollup" { if *noStaleMarkers || funcName == "default_rollup" || funcName == "stale_samples_over_time" {
// Do not drop Prometheus staleness marks (aka stale NaNs) for default_rollup() function, // Do not drop Prometheus staleness marks (aka stale NaNs) for default_rollup() function,
// since it uses them for Prometheus-style staleness detection. // since it uses them for Prometheus-style staleness detection.
// Do not drop staleness marks for stale_samples_over_time() function, since it needs
// to calculate the number of staleness markers.
return values, timestamps return values, timestamps
} }
// Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values. // Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values.

View file

@ -965,7 +965,7 @@ func TestExecSuccess(t *testing.T) {
}) })
t.Run("exp(time()/1e3)", func(t *testing.T) { t.Run("exp(time()/1e3)", func(t *testing.T) {
t.Parallel() t.Parallel()
q := `exp(time()/1e3)` q := `exp(alias(time()/1e3, "foobar"))`
r := netstorage.Result{ r := netstorage.Result{
MetricName: metricNameExpected, MetricName: metricNameExpected,
Values: []float64{2.718281828459045, 3.3201169227365472, 4.0551999668446745, 4.953032424395115, 6.0496474644129465, 7.38905609893065}, Values: []float64{2.718281828459045, 3.3201169227365472, 4.0551999668446745, 4.953032424395115, 6.0496474644129465, 7.38905609893065},
@ -974,6 +974,73 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r} resultExpected := []netstorage.Result{r}
f(q, resultExpected) f(q, resultExpected)
}) })
t.Run("exp(time()/1e3) keep_metric_names", func(t *testing.T) {
t.Parallel()
q := `exp(alias(time()/1e3, "foobar")) keep_metric_names`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{2.718281828459045, 3.3201169227365472, 4.0551999668446745, 4.953032424395115, 6.0496474644129465, 7.38905609893065},
Timestamps: timestampsExpected,
}
r.MetricName.MetricGroup = []byte("foobar")
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run("time() @ 1h", func(t *testing.T) {
t.Parallel()
q := `time() @ 1h`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{3600, 3600, 3600, 3600, 3600, 3600},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run("time() @ start()", func(t *testing.T) {
t.Parallel()
q := `time() @ start()`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1000, 1000, 1000, 1000, 1000},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run("time() @ end()", func(t *testing.T) {
t.Parallel()
q := `time() @ end()`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{2000, 2000, 2000, 2000, 2000, 2000},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run("time() @ end() offset 10m", func(t *testing.T) {
t.Parallel()
q := `time() @ end() offset 10m`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1400, 1400, 1400, 1400, 1400, 1400},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run("time() @ (end()-10m)", func(t *testing.T) {
t.Parallel()
q := `time() @ (end()-10m)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1400, 1400, 1400, 1400, 1400, 1400},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run("rand()", func(t *testing.T) { t.Run("rand()", func(t *testing.T) {
t.Parallel() t.Parallel()
q := `round(rand()/2)` q := `round(rand()/2)`
@ -2086,6 +2153,7 @@ func TestExecSuccess(t *testing.T) {
label_set(time()*3, "foo", "x"), label_set(time()*3, "foo", "x"),
), "foo"))` ), "foo"))`
r := netstorage.Result{ r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{3000, 3600, 4200, 4800, 5400, 6000}, Values: []float64{3000, 3600, 4200, 4800, 5400, 6000},
Timestamps: timestampsExpected, Timestamps: timestampsExpected,
} }
@ -6162,12 +6230,48 @@ func TestExecSuccess(t *testing.T) {
}) })
t.Run(`rate(time())`, func(t *testing.T) { t.Run(`rate(time())`, func(t *testing.T) {
t.Parallel() t.Parallel()
q := `rate(time())` q := `rate(label_set(alias(time(), "foo"), "x", "y"))`
r := netstorage.Result{ r := netstorage.Result{
MetricName: metricNameExpected, MetricName: metricNameExpected,
Values: []float64{1, 1, 1, 1, 1, 1}, Values: []float64{1, 1, 1, 1, 1, 1},
Timestamps: timestampsExpected, Timestamps: timestampsExpected,
} }
r.MetricName.Tags = []storage.Tag{
{
Key: []byte("x"),
Value: []byte("y"),
},
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`rate(time()) keep_metric_names`, func(t *testing.T) {
t.Parallel()
q := `rate(label_set(alias(time(), "foo"), "x", "y")) keep_metric_names`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1, 1, 1, 1, 1, 1},
Timestamps: timestampsExpected,
}
r.MetricName.MetricGroup = []byte("foo")
r.MetricName.Tags = []storage.Tag{
{
Key: []byte("x"),
Value: []byte("y"),
},
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`sum(rate(time()) keep_metric_names) by (__name__)`, func(t *testing.T) {
t.Parallel()
q := `sum(rate(label_set(alias(time(), "foo"), "x", "y")) keep_metric_names) by (__name__)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1, 1, 1, 1, 1, 1},
Timestamps: timestampsExpected,
}
r.MetricName.MetricGroup = []byte("foo")
resultExpected := []netstorage.Result{r} resultExpected := []netstorage.Result{r}
f(q, resultExpected) f(q, resultExpected)
}) })

View file

@ -77,6 +77,7 @@ var rollupFuncs = map[string]newRollupFunc{
"scrape_interval": newRollupFuncOneArg(rollupScrapeInterval), "scrape_interval": newRollupFuncOneArg(rollupScrapeInterval),
"share_gt_over_time": newRollupShareGT, "share_gt_over_time": newRollupShareGT,
"share_le_over_time": newRollupShareLE, "share_le_over_time": newRollupShareLE,
"stale_samples_over_time": newRollupFuncOneArg(rollupStaleSamples),
"stddev_over_time": newRollupFuncOneArg(rollupStddev), "stddev_over_time": newRollupFuncOneArg(rollupStddev),
"stdvar_over_time": newRollupFuncOneArg(rollupStdvar), "stdvar_over_time": newRollupFuncOneArg(rollupStdvar),
"sum_over_time": newRollupFuncOneArg(rollupSum), "sum_over_time": newRollupFuncOneArg(rollupSum),
@ -86,7 +87,7 @@ var rollupFuncs = map[string]newRollupFunc{
// in order to properly handle offset and timestamps unaligned to the current step. // in order to properly handle offset and timestamps unaligned to the current step.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/415 for details. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/415 for details.
"timestamp": newRollupFuncOneArg(rollupTlast), "timestamp": newRollupFuncOneArg(rollupTlast),
"timestamp_with_name": newRollupFuncOneArg(rollupTlast), // + rollupFuncsKeepMetricGroup "timestamp_with_name": newRollupFuncOneArg(rollupTlast), // + rollupFuncsKeepMetricName
"tlast_over_time": newRollupFuncOneArg(rollupTlast), "tlast_over_time": newRollupFuncOneArg(rollupTlast),
"tmax_over_time": newRollupFuncOneArg(rollupTmax), "tmax_over_time": newRollupFuncOneArg(rollupTmax),
"tmin_over_time": newRollupFuncOneArg(rollupTmin), "tmin_over_time": newRollupFuncOneArg(rollupTmin),
@ -128,6 +129,7 @@ var rollupAggrFuncs = map[string]rollupFunc{
"rate_over_sum": rollupRateOverSum, "rate_over_sum": rollupRateOverSum,
"resets": rollupResets, "resets": rollupResets,
"scrape_interval": rollupScrapeInterval, "scrape_interval": rollupScrapeInterval,
"stale_samples_over_time": rollupStaleSamples,
"stddev_over_time": rollupStddev, "stddev_over_time": rollupStddev,
"stdvar_over_time": rollupStdvar, "stdvar_over_time": rollupStdvar,
"sum_over_time": rollupSum, "sum_over_time": rollupSum,
@ -175,7 +177,7 @@ var rollupFuncsRemoveCounterResets = map[string]bool{
// These functions don't change physical meaning of input time series, // These functions don't change physical meaning of input time series,
// so they don't drop metric name // so they don't drop metric name
var rollupFuncsKeepMetricGroup = map[string]bool{ var rollupFuncsKeepMetricName = map[string]bool{
"avg_over_time": true, "avg_over_time": true,
"default_rollup": true, "default_rollup": true,
"first_over_time": true, "first_over_time": true,
@ -244,22 +246,6 @@ func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) {
return aggrFuncNames, nil return aggrFuncNames, nil
} }
func getRollupArgIdx(fe *metricsql.FuncExpr) int {
funcName := strings.ToLower(fe.Name)
if rollupFuncs[funcName] == nil {
logger.Panicf("BUG: getRollupArgIdx is called for non-rollup func %q", fe.Name)
}
switch funcName {
case "quantile_over_time", "aggr_over_time",
"hoeffding_bound_lower", "hoeffding_bound_upper":
return 1
case "quantiles_over_time":
return len(fe.Args) - 1
default:
return 0
}
}
func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, end, step, window int64, lookbackDelta int64, sharedTimestamps []int64) ( func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, end, step, window int64, lookbackDelta int64, sharedTimestamps []int64) (
func(values []float64, timestamps []int64), []*rollupConfig, error) { func(values []float64, timestamps []int64), []*rollupConfig, error) {
preFunc := func(values []float64, timestamps []int64) {} preFunc := func(values []float64, timestamps []int64) {}
@ -442,7 +428,7 @@ type timeseriesMap struct {
m map[string]*timeseries m map[string]*timeseries
} }
func newTimeseriesMap(funcName string, sharedTimestamps []int64, mnSrc *storage.MetricName) *timeseriesMap { func newTimeseriesMap(funcName string, keepMetricNames bool, sharedTimestamps []int64, mnSrc *storage.MetricName) *timeseriesMap {
funcName = strings.ToLower(funcName) funcName = strings.ToLower(funcName)
switch funcName { switch funcName {
case "histogram_over_time", "quantiles_over_time": case "histogram_over_time", "quantiles_over_time":
@ -456,7 +442,7 @@ func newTimeseriesMap(funcName string, sharedTimestamps []int64, mnSrc *storage.
} }
var origin timeseries var origin timeseries
origin.MetricName.CopyFrom(mnSrc) origin.MetricName.CopyFrom(mnSrc)
if !rollupFuncsKeepMetricGroup[funcName] { if !keepMetricNames && !rollupFuncsKeepMetricName[funcName] {
origin.MetricName.ResetMetricGroup() origin.MetricName.ResetMetricGroup()
} }
origin.Timestamps = sharedTimestamps origin.Timestamps = sharedTimestamps
@ -1389,6 +1375,20 @@ func rollupCount(rfa *rollupFuncArg) float64 {
return float64(len(values)) return float64(len(values))
} }
func rollupStaleSamples(rfa *rollupFuncArg) float64 {
values := rfa.values
if len(values) == 0 {
return nan
}
n := 0
for _, v := range rfa.values {
if decimal.IsStaleNaN(v) {
n++
}
}
return float64(n)
}
func rollupStddev(rfa *rollupFuncArg) float64 { func rollupStddev(rfa *rollupFuncArg) float64 {
stdvar := rollupStdvar(rfa) stdvar := rollupStdvar(rfa)
return math.Sqrt(stdvar) return math.Sqrt(stdvar)

View file

@ -513,6 +513,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
f("sum2_over_time", 37951) f("sum2_over_time", 37951)
f("geomean_over_time", 39.33466603189148) f("geomean_over_time", 39.33466603189148)
f("count_over_time", 12) f("count_over_time", 12)
f("stale_samples_over_time", 0)
f("stddev_over_time", 30.752935722554287) f("stddev_over_time", 30.752935722554287)
f("stdvar_over_time", 945.7430555555555) f("stdvar_over_time", 945.7430555555555)
f("first_over_time", 123) f("first_over_time", 123)

View file

@ -120,7 +120,7 @@ var transformFuncs = map[string]transformFunc{
// These functions don't change physical meaning of input time series, // These functions don't change physical meaning of input time series,
// so they don't drop metric name // so they don't drop metric name
var transformFuncsKeepMetricGroup = map[string]bool{ var transformFuncsKeepMetricName = map[string]bool{
"ceil": true, "ceil": true,
"clamp": true, "clamp": true,
"clamp_max": true, "clamp_max": true,
@ -172,9 +172,12 @@ func newTransformFuncOneArg(tf func(v float64) float64) transformFunc {
func doTransformValues(arg []*timeseries, tf func(values []float64), fe *metricsql.FuncExpr) ([]*timeseries, error) { func doTransformValues(arg []*timeseries, tf func(values []float64), fe *metricsql.FuncExpr) ([]*timeseries, error) {
name := strings.ToLower(fe.Name) name := strings.ToLower(fe.Name)
keepMetricGroup := transformFuncsKeepMetricGroup[name] keepMetricNames := fe.KeepMetricNames
if transformFuncsKeepMetricName[name] {
keepMetricNames = true
}
for _, ts := range arg { for _, ts := range arg {
if !keepMetricGroup { if !keepMetricNames {
ts.MetricName.ResetMetricGroup() ts.MetricName.ResetMetricGroup()
} }
tf(ts.Values) tf(ts.Values)

View file

@ -1,12 +1,12 @@
{ {
"files": { "files": {
"main.css": "./static/css/main.79ff1ad2.css", "main.css": "./static/css/main.79ff1ad2.css",
"main.js": "./static/js/main.22df0342.js", "main.js": "./static/js/main.31aed9a0.js",
"static/js/27.85f0e2b0.chunk.js": "./static/js/27.85f0e2b0.chunk.js", "static/js/27.cc1b69f7.chunk.js": "./static/js/27.cc1b69f7.chunk.js",
"index.html": "./index.html" "index.html": "./index.html"
}, },
"entrypoints": [ "entrypoints": [
"static/css/main.79ff1ad2.css", "static/css/main.79ff1ad2.css",
"static/js/main.22df0342.js" "static/js/main.31aed9a0.js"
] ]
} }

View file

@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script defer="defer" src="./static/js/main.22df0342.js"></script><link href="./static/css/main.79ff1ad2.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html> <!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script defer="defer" src="./static/js/main.31aed9a0.js"></script><link href="./static/css/main.79ff1ad2.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

View file

@ -1 +1 @@
"use strict";(self.webpackChunkvmui=self.webpackChunkvmui||[]).push([[27],{4027:function(e,n,t){t.r(n),t.d(n,{getCLS:function(){return y},getFCP:function(){return g},getFID:function(){return C},getLCP:function(){return P},getTTFB:function(){return D}});var i,r,a,o,u=function(e,n){return{name:e,value:void 0===n?-1:n,delta:0,entries:[],id:"v2-".concat(Date.now(),"-").concat(Math.floor(8999999999999*Math.random())+1e12)}},c=function(e,n){try{if(PerformanceObserver.supportedEntryTypes.includes(e)){if("first-input"===e&&!("PerformanceEventTiming"in self))return;var t=new PerformanceObserver((function(e){return e.getEntries().map(n)}));return t.observe({type:e,buffered:!0}),t}}catch(e){}},f=function(e,n){var t=function t(i){"pagehide"!==i.type&&"hidden"!==document.visibilityState||(e(i),n&&(removeEventListener("visibilitychange",t,!0),removeEventListener("pagehide",t,!0)))};addEventListener("visibilitychange",t,!0),addEventListener("pagehide",t,!0)},s=function(e){addEventListener("pageshow",(function(n){n.persisted&&e(n)}),!0)},m=function(e,n,t){var i;return function(r){n.value>=0&&(r||t)&&(n.delta=n.value-(i||0),(n.delta||void 0===i)&&(i=n.value,e(n)))}},v=-1,p=function(){return"hidden"===document.visibilityState?0:1/0},d=function(){f((function(e){var n=e.timeStamp;v=n}),!0)},l=function(){return v<0&&(v=p(),d(),s((function(){setTimeout((function(){v=p(),d()}),0)}))),{get firstHiddenTime(){return v}}},g=function(e,n){var t,i=l(),r=u("FCP"),a=function(e){"first-contentful-paint"===e.name&&(f&&f.disconnect(),e.startTime<i.firstHiddenTime&&(r.value=e.startTime,r.entries.push(e),t(!0)))},o=window.performance&&performance.getEntriesByName&&performance.getEntriesByName("first-contentful-paint")[0],f=o?null:c("paint",a);(o||f)&&(t=m(e,r,n),o&&a(o),s((function(i){r=u("FCP"),t=m(e,r,n),requestAnimationFrame((function(){requestAnimationFrame((function(){r.value=performance.now()-i.timeStamp,t(!0)}))}))})))},h=!1,T=-1,y=function(e,n){h||(g((function(e){T=e.value})),h=!0);var t,i=function(n){T>-1&&e(n)},r=u("CLS",0),a=0,o=[],v=function(e){if(!e.hadRecentInput){var n=o[0],i=o[o.length-1];a&&e.startTime-i.startTime<1e3&&e.startTime-n.startTime<5e3?(a+=e.value,o.push(e)):(a=e.value,o=[e]),a>r.value&&(r.value=a,r.entries=o,t())}},p=c("layout-shift",v);p&&(t=m(i,r,n),f((function(){p.takeRecords().map(v),t(!0)})),s((function(){a=0,T=-1,r=u("CLS",0),t=m(i,r,n)})))},E={passive:!0,capture:!0},w=new Date,L=function(e,n){i||(i=n,r=e,a=new Date,F(removeEventListener),S())},S=function(){if(r>=0&&r<a-w){var e={entryType:"first-input",name:i.type,target:i.target,cancelable:i.cancelable,startTime:i.timeStamp,processingStart:i.timeStamp+r};o.forEach((function(n){n(e)})),o=[]}},b=function(e){if(e.cancelable){var n=(e.timeStamp>1e12?new Date:performance.now())-e.timeStamp;"pointerdown"==e.type?function(e,n){var t=function(){L(e,n),r()},i=function(){r()},r=function(){removeEventListener("pointerup",t,E),removeEventListener("pointercancel",i,E)};addEventListener("pointerup",t,E),addEventListener("pointercancel",i,E)}(n,e):L(n,e)}},F=function(e){["mousedown","keydown","touchstart","pointerdown"].forEach((function(n){return e(n,b,E)}))},C=function(e,n){var t,a=l(),v=u("FID"),p=function(e){e.startTime<a.firstHiddenTime&&(v.value=e.processingStart-e.startTime,v.entries.push(e),t(!0))},d=c("first-input",p);t=m(e,v,n),d&&f((function(){d.takeRecords().map(p),d.disconnect()}),!0),d&&s((function(){var a;v=u("FID"),t=m(e,v,n),o=[],r=-1,i=null,F(addEventListener),a=p,o.push(a),S()}))},k={},P=function(e,n){var t,i=l(),r=u("LCP"),a=function(e){var n=e.startTime;n<i.firstHiddenTime&&(r.value=n,r.entries.push(e)),t()},o=c("largest-contentful-paint",a);if(o){t=m(e,r,n);var v=function(){k[r.id]||(o.takeRecords().map(a),o.disconnect(),k[r.id]=!0,t(!0))};["keydown","click"].forEach((function(e){addEventListener(e,v,{once:!0,capture:!0})})),f(v,!0),s((function(i){r=u("LCP"),t=m(e,r,n),requestAnimationFrame((function(){requestAnimationFrame((function(){r.value=performance.now()-i.timeStamp,k[r.id]=!0,t(!0)}))}))}))}},D=function(e){var n,t=u("TTFB");n=function(){try{var n=performance.getEntriesByType("navigation")[0]||function(){var e=performance.timing,n={entryType:"navigation",startTime:0};for(var t in e)"navigationStart"!==t&&"toJSON"!==t&&(n[t]=Math.max(e[t]-e.navigationStart,0));return n}();if(t.value=t.delta=n.responseStart,t.value<0||t.value>performance.now())return;t.entries=[n],e(t)}catch(e){}},"complete"===document.readyState?setTimeout(n,0):addEventListener("pageshow",n)}}}]); "use strict";(self.webpackChunkvmui=self.webpackChunkvmui||[]).push([[27],{4027:function(e,n,t){t.r(n),t.d(n,{getCLS:function(){return y},getFCP:function(){return g},getFID:function(){return C},getLCP:function(){return P},getTTFB:function(){return D}});var i,r,a,o,u=function(e,n){return{name:e,value:void 0===n?-1:n,delta:0,entries:[],id:"v2-".concat(Date.now(),"-").concat(Math.floor(8999999999999*Math.random())+1e12)}},c=function(e,n){try{if(PerformanceObserver.supportedEntryTypes.includes(e)){if("first-input"===e&&!("PerformanceEventTiming"in self))return;var t=new PerformanceObserver((function(e){return e.getEntries().map(n)}));return t.observe({type:e,buffered:!0}),t}}catch(e){}},f=function(e,n){var t=function t(i){"pagehide"!==i.type&&"hidden"!==document.visibilityState||(e(i),n&&(removeEventListener("visibilitychange",t,!0),removeEventListener("pagehide",t,!0)))};addEventListener("visibilitychange",t,!0),addEventListener("pagehide",t,!0)},s=function(e){addEventListener("pageshow",(function(n){n.persisted&&e(n)}),!0)},m=function(e,n,t){var i;return function(r){n.value>=0&&(r||t)&&(n.delta=n.value-(i||0),(n.delta||void 0===i)&&(i=n.value,e(n)))}},v=-1,p=function(){return"hidden"===document.visibilityState?0:1/0},d=function(){f((function(e){var n=e.timeStamp;v=n}),!0)},l=function(){return v<0&&(v=p(),d(),s((function(){setTimeout((function(){v=p(),d()}),0)}))),{get firstHiddenTime(){return v}}},g=function(e,n){var t,i=l(),r=u("FCP"),a=function(e){"first-contentful-paint"===e.name&&(f&&f.disconnect(),e.startTime<i.firstHiddenTime&&(r.value=e.startTime,r.entries.push(e),t(!0)))},o=window.performance&&performance.getEntriesByName&&performance.getEntriesByName("first-contentful-paint")[0],f=o?null:c("paint",a);(o||f)&&(t=m(e,r,n),o&&a(o),s((function(i){r=u("FCP"),t=m(e,r,n),requestAnimationFrame((function(){requestAnimationFrame((function(){r.value=performance.now()-i.timeStamp,t(!0)}))}))})))},h=!1,T=-1,y=function(e,n){h||(g((function(e){T=e.value})),h=!0);var t,i=function(n){T>-1&&e(n)},r=u("CLS",0),a=0,o=[],v=function(e){if(!e.hadRecentInput){var n=o[0],i=o[o.length-1];a&&e.startTime-i.startTime<1e3&&e.startTime-n.startTime<5e3?(a+=e.value,o.push(e)):(a=e.value,o=[e]),a>r.value&&(r.value=a,r.entries=o,t())}},p=c("layout-shift",v);p&&(t=m(i,r,n),f((function(){p.takeRecords().map(v),t(!0)})),s((function(){a=0,T=-1,r=u("CLS",0),t=m(i,r,n)})))},E={passive:!0,capture:!0},w=new Date,L=function(e,n){i||(i=n,r=e,a=new Date,F(removeEventListener),S())},S=function(){if(r>=0&&r<a-w){var e={entryType:"first-input",name:i.type,target:i.target,cancelable:i.cancelable,startTime:i.timeStamp,processingStart:i.timeStamp+r};o.forEach((function(n){n(e)})),o=[]}},b=function(e){if(e.cancelable){var n=(e.timeStamp>1e12?new Date:performance.now())-e.timeStamp;"pointerdown"==e.type?function(e,n){var t=function(){L(e,n),r()},i=function(){r()},r=function(){removeEventListener("pointerup",t,E),removeEventListener("pointercancel",i,E)};addEventListener("pointerup",t,E),addEventListener("pointercancel",i,E)}(n,e):L(n,e)}},F=function(e){["mousedown","keydown","touchstart","pointerdown"].forEach((function(n){return e(n,b,E)}))},C=function(e,n){var t,a=l(),v=u("FID"),p=function(e){e.startTime<a.firstHiddenTime&&(v.value=e.processingStart-e.startTime,v.entries.push(e),t(!0))},d=c("first-input",p);t=m(e,v,n),d&&f((function(){d.takeRecords().map(p),d.disconnect()}),!0),d&&s((function(){var a;v=u("FID"),t=m(e,v,n),o=[],r=-1,i=null,F(addEventListener),a=p,o.push(a),S()}))},k={},P=function(e,n){var t,i=l(),r=u("LCP"),a=function(e){var n=e.startTime;n<i.firstHiddenTime&&(r.value=n,r.entries.push(e),t())},o=c("largest-contentful-paint",a);if(o){t=m(e,r,n);var v=function(){k[r.id]||(o.takeRecords().map(a),o.disconnect(),k[r.id]=!0,t(!0))};["keydown","click"].forEach((function(e){addEventListener(e,v,{once:!0,capture:!0})})),f(v,!0),s((function(i){r=u("LCP"),t=m(e,r,n),requestAnimationFrame((function(){requestAnimationFrame((function(){r.value=performance.now()-i.timeStamp,k[r.id]=!0,t(!0)}))}))}))}},D=function(e){var n,t=u("TTFB");n=function(){try{var n=performance.getEntriesByType("navigation")[0]||function(){var e=performance.timing,n={entryType:"navigation",startTime:0};for(var t in e)"navigationStart"!==t&&"toJSON"!==t&&(n[t]=Math.max(e[t]-e.navigationStart,0));return n}();if(t.value=t.delta=n.responseStart,t.value<0||t.value>performance.now())return;t.entries=[n],e(t)}catch(e){}},"complete"===document.readyState?setTimeout(n,0):addEventListener("pageshow",n)}}}]);

File diff suppressed because one or more lines are too long

View file

@ -1,81 +0,0 @@
/*
object-assign
(c) Sindre Sorhus
@license MIT
*/
/*! @preserve
* numeral.js
* version : 2.0.6
* author : Adam Draper
* license : MIT
* http://adamwdraper.github.com/Numeral-js/
*/
/**
* A better abstraction over CSS.
*
* @copyright Oleg Isonen (Slobodskoi) / Isonen 2014-present
* @website https://github.com/cssinjs/jss
* @license MIT
*/
/** @license MUI v5.2.6
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v0.20.2
* scheduler.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v16.13.1
* react-is.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v17.0.2
* react-dom.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v17.0.2
* react-is.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v17.0.2
* react-jsx-runtime.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v17.0.2
* react.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,39 @@
/*! @preserve
* numeral.js
* version : 2.0.6
* author : Adam Draper
* license : MIT
* http://adamwdraper.github.com/Numeral-js/
*/
/**
* A better abstraction over CSS.
*
* @copyright Oleg Isonen (Slobodskoi) / Isonen 2014-present
* @website https://github.com/cssinjs/jss
* @license MIT
*/
/** @license MUI v5.2.6
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v16.13.1
* react-is.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/** @license React v17.0.2
* react-is.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

View file

@ -1,4 +1,4 @@
FROM node:alpine3.14 FROM node:alpine3.15
RUN apk update && apk add --no-cache bash bash-doc bash-completion libtool autoconf automake nasm pkgconfig libpng gcc make g++ zlib-dev gawk RUN apk update && apk add --no-cache bash bash-doc bash-completion libtool autoconf automake nasm pkgconfig libpng gcc make g++ zlib-dev gawk

View file

@ -1,7 +1,13 @@
// eslint-disable-next-line @typescript-eslint/no-var-requires,no-undef // eslint-disable-next-line @typescript-eslint/no-var-requires,no-undef
const {override, addExternalBabelPlugin} = require("customize-cra"); const {override, addExternalBabelPlugin, addWebpackAlias} = require("customize-cra");
// eslint-disable-next-line no-undef // eslint-disable-next-line no-undef
module.exports = override( module.exports = override(
addExternalBabelPlugin("@babel/plugin-proposal-nullish-coalescing-operator") addExternalBabelPlugin("@babel/plugin-proposal-nullish-coalescing-operator"),
addWebpackAlias({
"react": "preact/compat",
"react-dom/test-utils": "preact/test-utils",
"react-dom": "preact/compat", // Must be below test-utils
"react/jsx-runtime": "preact/jsx-runtime"
})
); );

File diff suppressed because it is too large Load diff

View file

@ -5,20 +5,19 @@
"homepage": "./", "homepage": "./",
"dependencies": { "dependencies": {
"@date-io/dayjs": "^2.11.0", "@date-io/dayjs": "^2.11.0",
"@emotion/react": "^11.7.1",
"@emotion/styled": "^11.6.0", "@emotion/styled": "^11.6.0",
"@mui/icons-material": "^5.2.5", "@mui/icons-material": "^5.2.5",
"@mui/lab": "^5.0.0-alpha.62", "@mui/lab": "^5.0.0-alpha.64",
"@mui/material": "^5.2.4", "@mui/material": "^5.2.8",
"@mui/styles": "^5.2.3", "@mui/styles": "^5.2.3",
"@testing-library/jest-dom": "^5.16.1", "@testing-library/jest-dom": "^5.16.1",
"@testing-library/react": "^12.1.2", "@testing-library/react": "^12.1.2",
"@testing-library/user-event": "^13.5.0", "@testing-library/user-event": "^13.5.0",
"@types/jest": "^27.0.3", "@types/jest": "^27.4.0",
"@types/lodash.debounce": "^4.0.6", "@types/lodash.debounce": "^4.0.6",
"@types/lodash.get": "^4.4.6", "@types/lodash.get": "^4.4.6",
"@types/lodash.throttle": "^4.1.6", "@types/lodash.throttle": "^4.1.6",
"@types/node": "^17.0.6", "@types/node": "^17.0.8",
"@types/numeral": "^2.0.2", "@types/numeral": "^2.0.2",
"@types/qs": "^6.9.7", "@types/qs": "^6.9.7",
"@types/react": "^17.0.38", "@types/react": "^17.0.38",
@ -29,15 +28,11 @@
"lodash.get": "^4.4.2", "lodash.get": "^4.4.2",
"lodash.throttle": "^4.1.1", "lodash.throttle": "^4.1.1",
"numeral": "^2.0.6", "numeral": "^2.0.6",
"qs": "^6.10.2", "preact": "^10.6.4",
"react": "^17.0.2", "qs": "^6.10.3",
"react-dom": "^17.0.2",
"react-draggable": "^4.4.4",
"react-measure": "^2.5.2",
"react-scripts": "5.0.0",
"typescript": "~4.5.4", "typescript": "~4.5.4",
"uplot": "^1.6.18", "uplot": "^1.6.18",
"web-vitals": "^2.1.2" "web-vitals": "^2.1.3"
}, },
"scripts": { "scripts": {
"start": "react-app-rewired start", "start": "react-app-rewired start",
@ -67,8 +62,8 @@
}, },
"devDependencies": { "devDependencies": {
"@babel/plugin-proposal-nullish-coalescing-operator": "^7.16.7", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.16.7",
"@typescript-eslint/eslint-plugin": "^5.8.1", "@typescript-eslint/eslint-plugin": "^5.9.1",
"@typescript-eslint/parser": "^5.8.0", "@typescript-eslint/parser": "^5.9.1",
"customize-cra": "^1.0.0", "customize-cra": "^1.0.0",
"eslint-plugin-react": "^7.28.0", "eslint-plugin-react": "^7.28.0",
"react-app-rewired": "^2.1.11" "react-app-rewired": "^2.1.11"

View file

@ -1,4 +1,4 @@
import React from "react"; import React from "preact/compat";
import {render, screen} from "@testing-library/react"; import {render, screen} from "@testing-library/react";
import App from "./App"; import App from "./App";

View file

@ -1,11 +1,11 @@
import React, {FC} from "react"; import React, {FC} from "preact/compat";
import {SnackbarProvider} from "./contexts/Snackbar"; import {SnackbarProvider} from "./contexts/Snackbar";
import HomeLayout from "./components/Home/HomeLayout"; import HomeLayout from "./components/Home/HomeLayout";
import {StateProvider} from "./state/common/StateContext"; import {StateProvider} from "./state/common/StateContext";
import {AuthStateProvider} from "./state/auth/AuthStateContext"; import {AuthStateProvider} from "./state/auth/AuthStateContext";
import {GraphStateProvider} from "./state/graph/GraphStateContext"; import {GraphStateProvider} from "./state/graph/GraphStateContext";
import { ThemeProvider, StyledEngineProvider } from "@mui/material/styles";
import THEME from "./theme/theme"; import THEME from "./theme/theme";
import { ThemeProvider, StyledEngineProvider } from "@mui/material/styles";
import CssBaseline from "@mui/material/CssBaseline"; import CssBaseline from "@mui/material/CssBaseline";
import LocalizationProvider from "@mui/lab/LocalizationProvider"; import LocalizationProvider from "@mui/lab/LocalizationProvider";
import DayjsUtils from "@date-io/dayjs"; import DayjsUtils from "@date-io/dayjs";

View file

@ -1,5 +1,9 @@
import React, {FC} from "react"; import React, {FC} from "preact/compat";
import {AppBar, Box, Link, Toolbar, Typography} from "@mui/material"; import AppBar from "@mui/material/AppBar";
import Box from "@mui/material/Box";
import Link from "@mui/material/Link";
import Toolbar from "@mui/material/Toolbar";
import Typography from "@mui/material/Typography";
import {ExecutionControls} from "../Home/Configurator/Time/ExecutionControls"; import {ExecutionControls} from "../Home/Configurator/Time/ExecutionControls";
import {DisplayTypeSwitch} from "../Home/Configurator/DisplayTypeSwitch"; import {DisplayTypeSwitch} from "../Home/Configurator/DisplayTypeSwitch";
import Logo from "../common/Logo"; import Logo from "../common/Logo";

View file

@ -1,26 +1,24 @@
/* eslint max-lines: ["error", {"max": 300}] */ /* eslint max-lines: ["error", {"max": 300}] */
import React, {useState} from "react"; import React, {useState} from "preact/compat";
import Box from "@mui/material/Box";
import Button from "@mui/material/Button";
import Checkbox from "@mui/material/Checkbox";
import DialogActions from "@mui/material/DialogActions";
import DialogContent from "@mui/material/DialogContent";
import DialogContentText from "@mui/material/DialogContentText";
import FormControl from "@mui/material/FormControl";
import FormControlLabel from "@mui/material/FormControlLabel";
import FormHelperText from "@mui/material/FormHelperText";
import Input from "@mui/material/Input";
import InputAdornment from "@mui/material/InputAdornment";
import InputLabel from "@mui/material/InputLabel";
import Tab from "@mui/material/Tab";
import Tabs from "@mui/material/Tabs";
import TextField from "@mui/material/TextField";
import Typography from "@mui/material/Typography";
import DialogTitle from "@mui/material/DialogTitle"; import DialogTitle from "@mui/material/DialogTitle";
import Dialog from "@mui/material/Dialog"; import Dialog from "@mui/material/Dialog";
import {
Box,
Button,
Checkbox,
DialogActions,
DialogContent,
DialogContentText,
FormControl,
FormControlLabel,
FormHelperText,
Input,
InputAdornment,
InputLabel,
Tab,
Tabs,
TextField,
Typography,
} from "@mui/material";
import createStyles from "@mui/styles/createStyles"; import createStyles from "@mui/styles/createStyles";
import TabPanel from "./AuthTabPanel"; import TabPanel from "./AuthTabPanel";
import PersonIcon from "@mui/icons-material/Person"; import PersonIcon from "@mui/icons-material/Person";
@ -28,6 +26,7 @@ import LockIcon from "@mui/icons-material/Lock";
import makeStyles from "@mui/styles/makeStyles"; import makeStyles from "@mui/styles/makeStyles";
import {useAuthDispatch, useAuthState} from "../../../../state/auth/AuthStateContext"; import {useAuthDispatch, useAuthState} from "../../../../state/auth/AuthStateContext";
import {AUTH_METHOD, WithCheckbox} from "../../../../state/auth/reducer"; import {AUTH_METHOD, WithCheckbox} from "../../../../state/auth/reducer";
import {ChangeEvent, ClipboardEvent} from "react";
// TODO: make generic when creating second dialog // TODO: make generic when creating second dialog
export interface DialogProps { export interface DialogProps {
@ -76,7 +75,7 @@ export const AuthDialog: React.FC<DialogProps> = (props) => {
setTabIndex(newValue); setTabIndex(newValue);
}; };
const handleBearerChange = (event: React.ChangeEvent<HTMLInputElement>) => { const handleBearerChange = (event: ChangeEvent<HTMLInputElement>) => {
const newVal = event.target.value; const newVal = event.target.value;
if (newVal.startsWith(BEARER_PREFIX)) { if (newVal.startsWith(BEARER_PREFIX)) {
setBearerValue(newVal); setBearerValue(newVal);
@ -89,7 +88,7 @@ export const AuthDialog: React.FC<DialogProps> = (props) => {
onClose(); onClose();
}; };
const onBearerPaste = (e: React.ClipboardEvent) => { const onBearerPaste = (e: ClipboardEvent) => {
// if you're pasting token word Bearer will be added automagically // if you're pasting token word Bearer will be added automagically
const newVal = e.clipboardData.getData("text/plain"); const newVal = e.clipboardData.getData("text/plain");
if (newVal.startsWith(BEARER_PREFIX)) { if (newVal.startsWith(BEARER_PREFIX)) {

View file

@ -1,4 +1,4 @@
import React from "react"; import React from "preact/compat";
import Box from "@mui/material/Box"; import Box from "@mui/material/Box";
interface TabPanelProps { interface TabPanelProps {

View file

@ -1,10 +1,9 @@
import React, {FC} from "react"; import React, {FC} from "preact/compat";
import TableChartIcon from "@mui/icons-material/TableChart"; import TableChartIcon from "@mui/icons-material/TableChart";
import ShowChartIcon from "@mui/icons-material/ShowChart"; import ShowChartIcon from "@mui/icons-material/ShowChart";
import CodeIcon from "@mui/icons-material/Code"; import CodeIcon from "@mui/icons-material/Code";
import ToggleButton from "@mui/material/ToggleButton";
import { ToggleButton, ToggleButtonGroup } from "@mui/material"; import ToggleButtonGroup from "@mui/material/ToggleButtonGroup";
import {useAppDispatch, useAppState} from "../../../state/common/StateContext"; import {useAppDispatch, useAppState} from "../../../state/common/StateContext";
import withStyles from "@mui/styles/withStyles"; import withStyles from "@mui/styles/withStyles";

View file

@ -1,5 +1,8 @@
import React, {FC, useCallback, useMemo} from "react"; import React, {FC, useCallback, useMemo} from "preact/compat";
import {Box, FormControlLabel, TextField} from "@mui/material"; import {ChangeEvent} from "react";
import Box from "@mui/material/Box";
import FormControlLabel from "@mui/material/FormControlLabel";
import TextField from "@mui/material/TextField";
import {useGraphDispatch, useGraphState} from "../../../../state/graph/GraphStateContext"; import {useGraphDispatch, useGraphState} from "../../../../state/graph/GraphStateContext";
import debounce from "lodash.debounce"; import debounce from "lodash.debounce";
import BasicSwitch from "../../../../theme/switch"; import BasicSwitch from "../../../../theme/switch";
@ -12,7 +15,7 @@ const AxesLimitsConfigurator: FC = () => {
const onChangeYaxisLimits = () => { graphDispatch({type: "TOGGLE_ENABLE_YAXIS_LIMITS"}); }; const onChangeYaxisLimits = () => { graphDispatch({type: "TOGGLE_ENABLE_YAXIS_LIMITS"}); };
const onChangeLimit = (e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>, axis: string, index: number) => { const onChangeLimit = (e: ChangeEvent<HTMLInputElement | HTMLTextAreaElement>, axis: string, index: number) => {
const newLimits = yaxis.limits.range; const newLimits = yaxis.limits.range;
newLimits[axis][index] = +e.target.value; newLimits[axis][index] = +e.target.value;
if (newLimits[axis][0] === newLimits[axis][1] || newLimits[axis][0] > newLimits[axis][1]) return; if (newLimits[axis][0] === newLimits[axis][1] || newLimits[axis][0] > newLimits[axis][1]) return;

View file

@ -1,18 +1,20 @@
import SettingsIcon from "@mui/icons-material/Settings"; import SettingsIcon from "@mui/icons-material/Settings";
import React, {FC, useState, useRef} from "react"; import React, {FC, useState} from "preact/compat";
import AxesLimitsConfigurator from "./AxesLimitsConfigurator"; import AxesLimitsConfigurator from "./AxesLimitsConfigurator";
import {Box, Button, IconButton, Paper, Typography} from "@mui/material"; import Box from "@mui/material/Box";
import Draggable from "react-draggable"; import Button from "@mui/material/Button";
import IconButton from "@mui/material/IconButton";
import Paper from "@mui/material/Paper";
import Popover from "@mui/material/Popover";
import Typography from "@mui/material/Typography";
import makeStyles from "@mui/styles/makeStyles"; import makeStyles from "@mui/styles/makeStyles";
import CloseIcon from "@mui/icons-material/Close"; import CloseIcon from "@mui/icons-material/Close";
const useStyles = makeStyles({ const useStyles = makeStyles({
popover: { popover: {
position: "absolute",
display: "grid", display: "grid",
gridGap: "16px", gridGap: "16px",
padding: "0 0 25px", padding: "0 0 25px",
zIndex: 2,
}, },
popoverHeader: { popoverHeader: {
display: "flex", display: "flex",
@ -22,7 +24,6 @@ const useStyles = makeStyles({
padding: "6px 6px 6px 12px", padding: "6px 6px 6px 12px",
borderRadius: "4px 4px 0 0", borderRadius: "4px 4px 0 0",
color: "#FFF", color: "#FFF",
cursor: "move",
}, },
popoverBody: { popoverBody: {
display: "grid", display: "grid",
@ -32,23 +33,31 @@ const useStyles = makeStyles({
}); });
const GraphSettings: FC = () => { const GraphSettings: FC = () => {
const [open, setOpen] = useState(false); const [anchorEl, setAnchorEl] = useState<HTMLButtonElement | null>(null);
const draggableRef = useRef<HTMLDivElement>(null); const open = Boolean(anchorEl);
const position = { x: 173, y: 0 };
const classes = useStyles(); const classes = useStyles();
return <Box display="flex" px={2}> return <Box display="flex" px={2}>
<Button onClick={() => setOpen((old) => !old)} variant="outlined"> <Button variant="outlined" aria-describedby="settings-popover"
onClick={(e) => setAnchorEl(e.currentTarget)} >
<SettingsIcon sx={{fontSize: 16, marginRight: "4px"}}/> <SettingsIcon sx={{fontSize: 16, marginRight: "4px"}}/>
<span style={{lineHeight: 1, paddingTop: "1px"}}>{open ? "Hide" : "Show"} graph settings</span> <span style={{lineHeight: 1, paddingTop: "1px"}}>{open ? "Hide" : "Show"} graph settings</span>
</Button> </Button>
{open && ( <Popover
<Draggable nodeRef={draggableRef} defaultPosition={position} handle="#handle"> id="settings-popover"
<Paper elevation={3} className={classes.popover} ref={draggableRef}> open={open}
anchorEl={anchorEl}
onClose={() => setAnchorEl(null)}
anchorOrigin={{
vertical: "top",
horizontal: anchorEl ? anchorEl.offsetWidth + 15 : 200
}}
>
<Paper elevation={3} className={classes.popover}>
<div id="handle" className={classes.popoverHeader}> <div id="handle" className={classes.popoverHeader}>
<Typography variant="body1"><b>Graph Settings</b></Typography> <Typography variant="body1"><b>Graph Settings</b></Typography>
<IconButton size="small" onClick={() => setOpen(false)}> <IconButton size="small" onClick={() => setAnchorEl(null)}>
<CloseIcon style={{color: "white"}}/> <CloseIcon style={{color: "white"}}/>
</IconButton> </IconButton>
</div> </div>
@ -56,8 +65,7 @@ const GraphSettings: FC = () => {
<AxesLimitsConfigurator/> <AxesLimitsConfigurator/>
</Box> </Box>
</Paper> </Paper>
</Draggable> </Popover>
)}
</Box>; </Box>;
}; };

View file

@ -1,5 +1,6 @@
import React, {FC} from "react"; import React, {FC} from "preact/compat";
import {Box, FormControlLabel} from "@mui/material"; import Box from "@mui/material/Box";
import FormControlLabel from "@mui/material/FormControlLabel";
import {saveToStorage} from "../../../../utils/storage"; import {saveToStorage} from "../../../../utils/storage";
import {useAppDispatch, useAppState} from "../../../../state/common/StateContext"; import {useAppDispatch, useAppState} from "../../../../state/common/StateContext";
import BasicSwitch from "../../../../theme/switch"; import BasicSwitch from "../../../../theme/switch";

View file

@ -1,7 +1,13 @@
import React, {FC, useEffect, useRef, useState} from "react"; import React, {FC, useEffect, useRef, useState} from "preact/compat";
import { import Accordion from "@mui/material/Accordion";
Accordion, AccordionDetails, AccordionSummary, Box, Grid, IconButton, Typography, Tooltip, Button import AccordionDetails from "@mui/material/AccordionDetails";
} from "@mui/material"; import AccordionSummary from "@mui/material/AccordionSummary";
import Box from "@mui/material/Box";
import IconButton from "@mui/material/IconButton";
import Typography from "@mui/material/Typography";
import Tooltip from "@mui/material/Tooltip";
import Button from "@mui/material/Button";
import Portal from "@mui/material/Portal";
import QueryEditor from "./QueryEditor"; import QueryEditor from "./QueryEditor";
import {TimeSelector} from "../Time/TimeSelector"; import {TimeSelector} from "../Time/TimeSelector";
import {useAppDispatch, useAppState} from "../../../../state/common/StateContext"; import {useAppDispatch, useAppState} from "../../../../state/common/StateContext";
@ -9,7 +15,6 @@ import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
import HighlightOffIcon from "@mui/icons-material/HighlightOff"; import HighlightOffIcon from "@mui/icons-material/HighlightOff";
import AddIcon from "@mui/icons-material/Add"; import AddIcon from "@mui/icons-material/Add";
import PlayCircleOutlineIcon from "@mui/icons-material/PlayCircleOutline"; import PlayCircleOutlineIcon from "@mui/icons-material/PlayCircleOutline";
import Portal from "@mui/material/Portal";
import ServerConfigurator from "./ServerConfigurator"; import ServerConfigurator from "./ServerConfigurator";
import AdditionalSettings from "./AdditionalSettings"; import AdditionalSettings from "./AdditionalSettings";
import {ErrorTypes} from "../../../../types"; import {ErrorTypes} from "../../../../types";
@ -21,7 +26,7 @@ export interface QueryConfiguratorProps {
const QueryConfigurator: FC<QueryConfiguratorProps> = ({error, queryOptions}) => { const QueryConfigurator: FC<QueryConfiguratorProps> = ({error, queryOptions}) => {
const {serverUrl, query, queryHistory, queryControls: {autocomplete}} = useAppState(); const {query, queryHistory, queryControls: {autocomplete}} = useAppState();
const dispatch = useAppDispatch(); const dispatch = useAppDispatch();
const [expanded, setExpanded] = useState(true); const [expanded, setExpanded] = useState(true);
const queryContainer = useRef<HTMLDivElement>(null); const queryContainer = useRef<HTMLDivElement>(null);
@ -109,8 +114,8 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({error, queryOptions}) =>
</Box> </Box>
</AccordionSummary> </AccordionSummary>
<AccordionDetails> <AccordionDetails>
<Grid container columnSpacing={2}> <Box display="flex" flexWrap="wrap" gap={2}>
<Grid item xs={6} minWidth={400}> <Box flexGrow="2" minWidth="50%">
<ServerConfigurator error={error}/> <ServerConfigurator error={error}/>
{/* for portal QueryEditor */} {/* for portal QueryEditor */}
<div ref={queryContainer}/> <div ref={queryContainer}/>
@ -120,14 +125,14 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({error, queryOptions}) =>
<span style={{lineHeight: 1, paddingTop: "1px"}}>Query</span> <span style={{lineHeight: 1, paddingTop: "1px"}}>Query</span>
</Button> </Button>
</Box>} </Box>}
</Grid> </Box>
<Grid item xs> <Box flexGrow="1">
<TimeSelector setDuration={onSetDuration}/> <TimeSelector setDuration={onSetDuration}/>
</Grid> </Box>
<Grid item xs={12} pt={1}> <Box flexBasis="100%" pt={1}>
<AdditionalSettings/> <AdditionalSettings/>
</Grid> </Box>
</Grid> </Box>
</AccordionDetails> </AccordionDetails>
</Accordion> </Accordion>
</>; </>;

View file

@ -1,6 +1,8 @@
import React, {FC, useEffect, useState} from "react"; import React, {FC, useEffect, useState} from "preact/compat";
import {KeyboardEvent} from "react";
import {ErrorTypes} from "../../../../types"; import {ErrorTypes} from "../../../../types";
import {Autocomplete, TextField} from "@mui/material"; import Autocomplete from "@mui/material/Autocomplete";
import TextField from "@mui/material/TextField";
import {queryToBreakLine} from "../../../../utils/query-string"; import {queryToBreakLine} from "../../../../utils/query-string";
export interface QueryEditorProps { export interface QueryEditorProps {
@ -33,11 +35,11 @@ const QueryEditor: FC<QueryEditorProps> = ({
setValue(queryToBreakLine(query)); setValue(queryToBreakLine(query));
}, [query]); }, [query]);
const handleKeyDown = (e: React.KeyboardEvent<HTMLDivElement>): void => { const handleKeyDown = (e: KeyboardEvent<HTMLDivElement>): void => {
if (e.ctrlKey || e.metaKey) setDownMetaKeys([...downMetaKeys, e.key]); if (e.ctrlKey || e.metaKey) setDownMetaKeys([...downMetaKeys, e.key]);
}; };
const handleKeyUp = (e: React.KeyboardEvent<HTMLDivElement>): void => { const handleKeyUp = (e: KeyboardEvent<HTMLDivElement>): void => {
const {key, ctrlKey, metaKey} = e; const {key, ctrlKey, metaKey} = e;
if (downMetaKeys.includes(key)) setDownMetaKeys(downMetaKeys.filter(k => k !== key)); if (downMetaKeys.includes(key)) setDownMetaKeys(downMetaKeys.filter(k => k !== key));
const ctrlMetaKey = ctrlKey || metaKey; const ctrlMetaKey = ctrlKey || metaKey;

View file

@ -1,5 +1,8 @@
import React, {FC, useEffect, useState} from "react"; import React, {FC, useEffect, useState} from "preact/compat";
import {Box, TextField, Tooltip, IconButton} from "@mui/material"; import Box from "@mui/material/Box";
import TextField from "@mui/material/TextField";
import Tooltip from "@mui/material/Tooltip";
import IconButton from "@mui/material/IconButton";
import SecurityIcon from "@mui/icons-material/Security"; import SecurityIcon from "@mui/icons-material/Security";
import {useAppDispatch, useAppState} from "../../../../state/common/StateContext"; import {useAppDispatch, useAppState} from "../../../../state/common/StateContext";
import {AuthDialog} from "../Auth/AuthDialog"; import {AuthDialog} from "../Auth/AuthDialog";

View file

@ -1,5 +1,8 @@
import React, {FC, useCallback, useEffect, useState} from "react"; import React, {FC, useCallback, useEffect, useState} from "preact/compat";
import {Box, FormControlLabel, TextField} from "@mui/material"; import {ChangeEvent} from "react";
import Box from "@mui/material/Box";
import FormControlLabel from "@mui/material/FormControlLabel";
import TextField from "@mui/material/TextField";
import BasicSwitch from "../../../../theme/switch"; import BasicSwitch from "../../../../theme/switch";
import {useGraphDispatch, useGraphState} from "../../../../state/graph/GraphStateContext"; import {useGraphDispatch, useGraphState} from "../../../../state/graph/GraphStateContext";
import {useAppState} from "../../../../state/common/StateContext"; import {useAppState} from "../../../../state/common/StateContext";
@ -11,7 +14,7 @@ const StepConfigurator: FC = () => {
const [error, setError] = useState(false); const [error, setError] = useState(false);
const {time: {period: {step}}} = useAppState(); const {time: {period: {step}}} = useAppState();
const onChangeStep = (e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>) => { const onChangeStep = (e: ChangeEvent<HTMLInputElement | HTMLTextAreaElement>) => {
const value = +e.target.value; const value = +e.target.value;
if (value > 0) { if (value > 0) {
graphDispatch({type: "SET_CUSTOM_STEP", payload: value}); graphDispatch({type: "SET_CUSTOM_STEP", payload: value});

View file

@ -1,4 +1,4 @@
import {useEffect, useMemo, useState} from "react"; import {useEffect, useMemo, useState} from "preact/compat";
import {getQueryOptions, getQueryRangeUrl, getQueryUrl} from "../../../../api/query-range"; import {getQueryOptions, getQueryRangeUrl, getQueryUrl} from "../../../../api/query-range";
import {useAppState} from "../../../../state/common/StateContext"; import {useAppState} from "../../../../state/common/StateContext";
import {InstantMetricResult, MetricBase, MetricResult} from "../../../../api/types"; import {InstantMetricResult, MetricBase, MetricResult} from "../../../../api/types";

View file

@ -1,5 +1,8 @@
import React, {FC, useEffect, useState} from "react"; import React, {FC, useEffect, useState} from "preact/compat";
import {Box, FormControlLabel, IconButton, Tooltip} from "@mui/material"; import Box from "@mui/material/Box";
import FormControlLabel from "@mui/material/FormControlLabel";
import IconButton from "@mui/material/IconButton";
import Tooltip from "@mui/material/Tooltip";
import EqualizerIcon from "@mui/icons-material/Equalizer"; import EqualizerIcon from "@mui/icons-material/Equalizer";
import {useAppDispatch, useAppState} from "../../../../state/common/StateContext"; import {useAppDispatch, useAppState} from "../../../../state/common/StateContext";
import CircularProgressWithLabel from "../../../common/CircularProgressWithLabel"; import CircularProgressWithLabel from "../../../common/CircularProgressWithLabel";

View file

@ -1,5 +1,11 @@
import React, {FC} from "react"; import React, {FC} from "preact/compat";
import {Paper, Table, TableBody, TableCell, TableContainer, TableHead, TableRow} from "@mui/material"; import Paper from "@mui/material/Paper";
import Table from "@mui/material/Table";
import TableBody from "@mui/material/TableBody";
import TableCell from "@mui/material/TableCell";
import TableContainer from "@mui/material/TableContainer";
import TableHead from "@mui/material/TableHead";
import TableRow from "@mui/material/TableRow";
import {supportedDurations} from "../../../../utils/time"; import {supportedDurations} from "../../../../utils/time";
export const TimeDurationPopover: FC = () => { export const TimeDurationPopover: FC = () => {

View file

@ -1,5 +1,9 @@
import React, {FC, useEffect, useState} from "react"; import React, {FC, useEffect, useState} from "preact/compat";
import {Box, Popover, TextField, Typography} from "@mui/material"; import {ChangeEvent, MouseEvent, KeyboardEvent} from "react";
import Box from "@mui/material/Box";
import Popover from "@mui/material/Popover";
import TextField from "@mui/material/TextField";
import Typography from "@mui/material/Typography";
import {checkDurationLimit} from "../../../../utils/time"; import {checkDurationLimit} from "../../../../utils/time";
import {TimeDurationPopover} from "./TimeDurationPopover"; import {TimeDurationPopover} from "./TimeDurationPopover";
import {InlineBtn} from "../../../common/InlineBtn"; import {InlineBtn} from "../../../common/InlineBtn";
@ -17,11 +21,11 @@ const TimeDurationSelector: FC<TimeDurationSelector> = ({setDuration}) => {
const [durationStringFocused, setFocused] = useState(false); const [durationStringFocused, setFocused] = useState(false);
const open = Boolean(anchorEl); const open = Boolean(anchorEl);
const handleDurationChange = (event: React.ChangeEvent<HTMLInputElement>) => { const handleDurationChange = (event: ChangeEvent<HTMLInputElement>) => {
setDurationString(event.target.value); setDurationString(event.target.value);
}; };
const handlePopoverOpen = (event: React.MouseEvent<Element, MouseEvent>) => { const handlePopoverOpen = (event: MouseEvent<HTMLSpanElement>) => {
setAnchorEl(event.currentTarget); setAnchorEl(event.currentTarget);
}; };
@ -29,7 +33,7 @@ const TimeDurationSelector: FC<TimeDurationSelector> = ({setDuration}) => {
setAnchorEl(null); setAnchorEl(null);
}; };
const onKeyUp = (event: React.KeyboardEvent<HTMLInputElement>) => { const onKeyUp = (event: KeyboardEvent<HTMLInputElement>) => {
if (event.key !== "Enter") return; if (event.key !== "Enter") return;
const target = event.target as HTMLInputElement; const target = event.target as HTMLInputElement;
target.blur(); target.blur();

View file

@ -1,5 +1,7 @@
import React, {FC, useEffect, useState} from "react"; import React, {FC, useEffect, useState} from "preact/compat";
import {Box, TextField, Typography} from "@mui/material"; import Box from "@mui/material/Box";
import TextField from "@mui/material/TextField";
import Typography from "@mui/material/Typography";
import DateTimePicker from "@mui/lab/DateTimePicker"; import DateTimePicker from "@mui/lab/DateTimePicker";
import {useAppDispatch, useAppState} from "../../../../state/common/StateContext"; import {useAppDispatch, useAppState} from "../../../../state/common/StateContext";
import {dateFromSeconds, formatDateForNativeInput} from "../../../../utils/time"; import {dateFromSeconds, formatDateForNativeInput} from "../../../../utils/time";

View file

@ -1,5 +1,8 @@
import React, {FC} from "react"; import React, {FC} from "preact/compat";
import {Alert, Box, CircularProgress, Fade} from "@mui/material"; import Alert from "@mui/material/Alert";
import Box from "@mui/material/Box";
import CircularProgress from "@mui/material/CircularProgress";
import Fade from "@mui/material/Fade";
import GraphView from "./Views/GraphView"; import GraphView from "./Views/GraphView";
import TableView from "./Views/TableView"; import TableView from "./Views/TableView";
import {useAppState} from "../../state/common/StateContext"; import {useAppState} from "../../state/common/StateContext";

View file

@ -1,5 +1,7 @@
import React, {FC} from "react"; import React, {FC} from "preact/compat";
import {Box, IconButton, Tooltip} from "@mui/material"; import Box from "@mui/material/Box";
import IconButton from "@mui/material/IconButton";
import Tooltip from "@mui/material/Tooltip";
import FileCopyIcon from "@mui/icons-material/FileCopy"; import FileCopyIcon from "@mui/icons-material/FileCopy";
import {useSnack} from "../../contexts/Snackbar"; import {useSnack} from "../../contexts/Snackbar";

View file

@ -1,5 +1,7 @@
import React, {FC} from "react"; import React, {FC} from "preact/compat";
import {Box, Button, Grid, Typography} from "@mui/material"; import Box from "@mui/material/Box";
import Button from "@mui/material/Button";
import Typography from "@mui/material/Typography";
import {useSnack} from "../../contexts/Snackbar"; import {useSnack} from "../../contexts/Snackbar";
interface UrlLineProps { interface UrlLineProps {
@ -10,7 +12,7 @@ export const UrlLine: FC<UrlLineProps> = ({url}) => {
const {showInfoMessage} = useSnack(); const {showInfoMessage} = useSnack();
return <Grid item style={{backgroundColor: "#eee", width: "100%"}}> return <Box style={{backgroundColor: "#eee", width: "100%"}}>
<Box flexDirection="row" display="flex" justifyContent="space-between" alignItems="center"> <Box flexDirection="row" display="flex" justifyContent="space-between" alignItems="center">
<Box pl={2} py={1} display="flex" style={{ <Box pl={2} py={1} display="flex" style={{
flex: 1, flex: 1,
@ -38,5 +40,5 @@ export const UrlLine: FC<UrlLineProps> = ({url}) => {
}}>Copy Query Url</Button> }}>Copy Query Url</Button>
</Box> </Box>
</Box> </Box>
</Grid>; </Box>;
}; };

View file

@ -1,4 +1,4 @@
import React, {FC, useEffect, useMemo, useState} from "react"; import React, {FC, useEffect, useMemo, useState} from "preact/compat";
import {MetricResult} from "../../../api/types"; import {MetricResult} from "../../../api/types";
import LineChart from "../../LineChart/LineChart"; import LineChart from "../../LineChart/LineChart";
import {AlignedData as uPlotData, Series as uPlotSeries} from "uplot"; import {AlignedData as uPlotData, Series as uPlotSeries} from "uplot";

View file

@ -1,6 +1,7 @@
import React, {FC, useMemo} from "react"; import React, {FC, useMemo} from "preact/compat";
import {InstantMetricResult} from "../../../api/types"; import {InstantMetricResult} from "../../../api/types";
import {Box, Button} from "@mui/material"; import Box from "@mui/material/Box";
import Button from "@mui/material/Button";
import {useSnack} from "../../../contexts/Snackbar"; import {useSnack} from "../../../contexts/Snackbar";
export interface JsonViewProps { export interface JsonViewProps {

View file

@ -1,7 +1,13 @@
import React, {FC, useMemo} from "react"; import React, {FC, useMemo} from "preact/compat";
import {InstantMetricResult} from "../../../api/types"; import {InstantMetricResult} from "../../../api/types";
import {InstantDataSeries} from "../../../types"; import {InstantDataSeries} from "../../../types";
import {Paper, Table, TableBody, TableCell, TableContainer, TableHead, TableRow} from "@mui/material"; import Paper from "@mui/material/Paper";
import Table from "@mui/material/Table";
import TableBody from "@mui/material/TableBody";
import TableCell from "@mui/material/TableCell";
import TableContainer from "@mui/material/TableContainer";
import TableHead from "@mui/material/TableHead";
import TableRow from "@mui/material/TableRow";
import makeStyles from "@mui/styles/makeStyles"; import makeStyles from "@mui/styles/makeStyles";
import {useSortedCategories} from "../../../hooks/useSortedCategories"; import {useSortedCategories} from "../../../hooks/useSortedCategories";

View file

@ -1,4 +1,4 @@
import React, {FC, useMemo} from "react"; import React, {FC, useMemo} from "preact/compat";
import {hexToRGB} from "../../utils/color"; import {hexToRGB} from "../../utils/color";
import {useAppState} from "../../state/common/StateContext"; import {useAppState} from "../../state/common/StateContext";
import {LegendItem} from "../../utils/uplot/types"; import {LegendItem} from "../../utils/uplot/types";

View file

@ -1,4 +1,4 @@
import React, {FC, useCallback, useEffect, useRef, useState} from "react"; import React, {FC, useCallback, useEffect, useRef, useState} from "preact/compat";
import {useAppDispatch, useAppState} from "../../state/common/StateContext"; import {useAppDispatch, useAppState} from "../../state/common/StateContext";
import uPlot, {AlignedData as uPlotData, Options as uPlotOptions, Series as uPlotSeries, Range, Scales, Scale} from "uplot"; import uPlot, {AlignedData as uPlotData, Options as uPlotOptions, Series as uPlotSeries, Range, Scales, Scale} from "uplot";
import {useGraphState} from "../../state/graph/GraphStateContext"; import {useGraphState} from "../../state/graph/GraphStateContext";

View file

@ -1,7 +1,7 @@
import Box from "@mui/material/Box";
import CircularProgress, {CircularProgressProps} from "@mui/material/CircularProgress"; import CircularProgress, {CircularProgressProps} from "@mui/material/CircularProgress";
import {Box} from "@mui/material";
import Typography from "@mui/material/Typography"; import Typography from "@mui/material/Typography";
import React, {FC} from "react"; import React, {FC} from "preact/compat";
const CircularProgressWithLabel: FC<CircularProgressProps & { label: number }> = (props) => { const CircularProgressWithLabel: FC<CircularProgressProps & { label: number }> = (props) => {
return ( return (

View file

@ -1,6 +1,6 @@
import makeStyles from "@mui/styles/makeStyles"; import makeStyles from "@mui/styles/makeStyles";
import React from "react"; import React from "preact/compat";
import {Link} from "@mui/material"; import Link from "@mui/material/Link";
const useStyles = makeStyles({ const useStyles = makeStyles({
inlineBtn: { inlineBtn: {

View file

@ -1,8 +1,9 @@
import React, {FC} from "react"; import React, {FC} from "preact/compat";
import {SvgIcon} from "@mui/material"; import {CSSProperties} from "@mui/styles";
import SvgIcon from "@mui/material/SvgIcon";
interface LogoProps { interface LogoProps {
style?: React.CSSProperties style?: CSSProperties
} }
const Logo: FC<LogoProps> = ({style}) => ( const Logo: FC<LogoProps> = ({style}) => (

View file

@ -1,5 +1,6 @@
import React, {createContext, FC, useContext, useEffect, useState} from "react"; import React, {createContext, FC, useContext, useEffect, useState} from "preact/compat";
import {Alert, Snackbar} from "@mui/material"; import Alert from "@mui/material/Alert";
import Snackbar from "@mui/material/Snackbar";
export interface SnackModel { export interface SnackModel {
message?: string; message?: string;

View file

@ -1,4 +1,4 @@
import { useState, useEffect } from "react"; import { useState, useEffect } from "preact/compat";
const useResize = (node: HTMLElement | null): {width: number, height: number} => { const useResize = (node: HTMLElement | null): {width: number, height: number} => {
const [windowSize, setWindowSize] = useState({ const [windowSize, setWindowSize] = useState({

View file

@ -1,4 +1,4 @@
import {useMemo} from "react"; import {useMemo} from "preact/compat";
import {MetricBase} from "../api/types"; import {MetricBase} from "../api/types";
export type MetricCategory = { export type MetricCategory = {

View file

@ -1,14 +1,11 @@
import React from "react"; import React, { render } from "preact/compat";
import ReactDOM from "react-dom";
import "./index.css"; import "./index.css";
import App from "./App"; import App from "./App";
import reportWebVitals from "./reportWebVitals"; import reportWebVitals from "./reportWebVitals";
ReactDOM.render(
<React.StrictMode> const root = document.getElementById("root");
<App /> if (root) render(<App />, root);
</React.StrictMode>,
document.getElementById("root")
);
// If you want to start measuring performance in your app, pass a function // If you want to start measuring performance in your app, pass a function
// to log results (for example: reportWebVitals(console.log)) // to log results (for example: reportWebVitals(console.log))

View file

@ -1,5 +1,6 @@
import React, {createContext, Dispatch, FC, useContext, useMemo, useReducer} from "react"; import React, {createContext, FC, useContext, useMemo, useReducer} from "preact/compat";
import {AuthAction, AuthState, initialPrepopulatedState, reducer} from "./reducer"; import {AuthAction, AuthState, initialPrepopulatedState, reducer} from "./reducer";
import {Dispatch} from "react";
type AuthStateContextType = { state: AuthState, dispatch: Dispatch<AuthAction> }; type AuthStateContextType = { state: AuthState, dispatch: Dispatch<AuthAction> };

View file

@ -1,6 +1,7 @@
import React, {createContext, Dispatch, FC, useContext, useEffect, useMemo, useReducer} from "react"; import React, {createContext, FC, useContext, useEffect, useMemo, useReducer} from "preact/compat";
import {Action, AppState, initialState, reducer} from "./reducer"; import {Action, AppState, initialState, reducer} from "./reducer";
import {getQueryStringValue, setQueryStringValue} from "../../utils/query-string"; import {getQueryStringValue, setQueryStringValue} from "../../utils/query-string";
import {Dispatch} from "react";
type StateContextType = { state: AppState, dispatch: Dispatch<Action> }; type StateContextType = { state: AppState, dispatch: Dispatch<Action> };

View file

@ -1,5 +1,6 @@
import React, {createContext, Dispatch, FC, useContext, useMemo, useReducer} from "react"; import React, {createContext, FC, useContext, useMemo, useReducer} from "preact/compat";
import {GraphAction, GraphState, initialGraphState, reducer} from "./reducer"; import {GraphAction, GraphState, initialGraphState, reducer} from "./reducer";
import {Dispatch} from "react";
type GraphStateContextType = { state: GraphState, dispatch: Dispatch<GraphAction> }; type GraphStateContextType = { state: GraphState, dispatch: Dispatch<GraphAction> };

View file

@ -1,5 +1,5 @@
import {styled} from "@mui/material/styles";
import Switch from "@mui/material/Switch"; import Switch from "@mui/material/Switch";
import {styled} from "@mui/styles";
const BasicSwitch = styled(Switch)(() => ({ const BasicSwitch = styled(Switch)(() => ({
padding: 10, padding: 10,

View file

@ -4,7 +4,7 @@ DOCKER_NAMESPACE := victoriametrics
ROOT_IMAGE ?= alpine:3.15.0 ROOT_IMAGE ?= alpine:3.15.0
CERTS_IMAGE := alpine:3.15.0 CERTS_IMAGE := alpine:3.15.0
GO_BUILDER_IMAGE := golang:1.17.5-alpine GO_BUILDER_IMAGE := golang:1.17.6-alpine
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __) BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)
BASE_IMAGE := local/base:1.1.3-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __) BASE_IMAGE := local/base:1.1.3-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

View file

@ -6,15 +6,37 @@ sort: 15
## tip ## tip
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add support for `@` modifier, which is enabled by default in Prometheus starting from [Prometheus v2.33.0](https://github.com/prometheus/prometheus/pull/10121). See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1348). VictoriaMetrics extends `@` modifier with the following additional features:
* It can contain arbitrary expression. For example, `foo @ (end() - 1h)` would return `foo` value at `end - 1 hour` timestamp on the selected time range `[start ... end]`. Another example: `foo @ now() - 10m` would return `foo` value 10 minutes ago from the current time.
* It can be put everywhere in the query. For example, `sum(foo) @ start()` would calculate `sum(foo)` at `start` timestamp on the selected time range `[start ... end]`.
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add support for optional `keep_metric_names` modifier, which can be applied to all the [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) and [transform functions](https://docs.victoriametrics.com/MetricsQL.html#transform-functions). This modifier prevents from deleting metric names from function results. For example, `rate({__name__=~"foo|bar"}[5m]) keep_metric_names` leaves `foo` and `bar` metric names in `rate()` results. This feature provides an additional workaround for [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/949).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for Kubernetes service discovery in the current namespace in the same way as [Prometheus does](https://github.com/prometheus/prometheus/pull/9881). For example, the following config limits pod discovery to the namespace where vmagent runs:
```yaml
scrape_configs:
- job_name: 'kubernetes-pods'
kubernetes_sd_configs:
- role: pod
namespaces:
own_namespace: true
```
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `__meta_kubernetes_node_provider_id` label for discovered Kubernetes nodes in the same way as [Prometheus does](https://github.com/prometheus/prometheus/pull/9603).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): log error message when remote storage returns 400 or 409 http errors. This should simplify detection and debugging of this case. See [this issue](vmagent_remotewrite_packets_dropped_total). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): log error message when remote storage returns 400 or 409 http errors. This should simplify detection and debugging of this case. See [this issue](vmagent_remotewrite_packets_dropped_total).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `promscrape_stale_samples_created_total` metric for monitoring the total number of created stale samples when scraping Prometheus targets. See [these docs](https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers) for the information on when stale samples (aka staleness markers) can be created.
* FEATURE: [vmrestore](https://docs.victoriametrics.com/vmrestore.html): store `restore-in-progress` file in `-dst` directory while `vmrestore` is running. This file is automatically deleted when `vmrestore` is successfully finished. This helps detecting incompletely restored data on VictoriaMetrics start. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1958). * FEATURE: [vmrestore](https://docs.victoriametrics.com/vmrestore.html): store `restore-in-progress` file in `-dst` directory while `vmrestore` is running. This file is automatically deleted when `vmrestore` is successfully finished. This helps detecting incompletely restored data on VictoriaMetrics start. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1958).
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): print the last sample timestamp when the data migration is interrupted either by user or by error. This helps continuing the data migration from the interruption moment. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1236). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): print the last sample timestamp when the data migration is interrupted either by user or by error. This helps continuing the data migration from the interruption moment. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1236).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): expose `vmalert_remotewrite_total` metric at `/metrics` page. This makes possible calculating SLOs for error rate during writing recording rules and alert state to `-remoteWrite.url` with the query `vmalert_remotewrite_errors_total / vmalert_remotewrite_total`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2040). Thanks to @afoninsky .
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `stripPort` template function in the same way as [Prometheus does](https://github.com/prometheus/prometheus/pull/10002).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `parseDuration` template function in the same way as [Prometheus does](https://github.com/prometheus/prometheus/pull/8817).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `stale_samples_over_time(m[d])` function for calculating the number of [staleness marks](https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers) for time series `m` over the duration `d`. This function may be useful for detecting flapping metrics at scrape targets, which periodically disappear and then appear again.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): make sure that `vmagent` replicas scrape the same targets at different time offsets when [replication is enabled in vmagent clustering mode](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets). This guarantees that the [deduplication](https://docs.victoriametrics.com/#deduplication) consistently leaves samples from the same `vmagent` replica. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): make sure that `vmagent` replicas scrape the same targets at different time offsets when [replication is enabled in vmagent clustering mode](https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets). This guarantees that the [deduplication](https://docs.victoriametrics.com/#deduplication) consistently leaves samples from the same `vmagent` replica.
* BUGFIX: return the proper response stub from `/api/v1/query_exemplars` handler, which is needed for Grafana v8+. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1999). * BUGFIX: return the proper response stub from `/api/v1/query_exemplars` handler, which is needed for Grafana v8+. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1999).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): fix a few edge cases and improve migration speed for OpenTSDB importer. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2019). * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): fix a few edge cases and improve migration speed for OpenTSDB importer. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2019).
* BUGFIX: fix possible data race when searching for time series matching `{key=~"value|"}` filter over time range covering multipe days. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2032). Thanks to @waldoweng for the provided fix. * BUGFIX: fix possible data race when searching for time series matching `{key=~"value|"}` filter over time range covering multipe days. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2032). Thanks to @waldoweng for the provided fix.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not send staleness markers on graceful shutdown. This follows Prometheus behavior. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2013#issuecomment-1006994079). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not send staleness markers on graceful shutdown. This follows Prometheus behavior. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2013#issuecomment-1006994079).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly set `__address__` label in `dockerswarm_sd_config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2038). Thanks to @ashtuchkin for the fix.
## [v1.71.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.71.0) ## [v1.71.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.71.0)

View file

@ -33,9 +33,10 @@ This functionality can be evaluated at [an editable Grafana dashboard](https://p
- Graphite-compatible filters can be passed via `{__graphite__="foo.*.bar"}` syntax. See [these docs](https://docs.victoriametrics.com/#selecting-graphite-metrics). VictoriaMetrics also can be used as Graphite datasource in Grafana. See [these docs](https://docs.victoriametrics.com/#graphite-api-usage) for details. See also [label_graphite_group](#label_graphite_group) function, which can be used for extracting the given groups from Graphite metric name. - Graphite-compatible filters can be passed via `{__graphite__="foo.*.bar"}` syntax. See [these docs](https://docs.victoriametrics.com/#selecting-graphite-metrics). VictoriaMetrics also can be used as Graphite datasource in Grafana. See [these docs](https://docs.victoriametrics.com/#graphite-api-usage) for details. See also [label_graphite_group](#label_graphite_group) function, which can be used for extracting the given groups from Graphite metric name.
- Lookbehind window in square brackets may be omitted. VictoriaMetrics automatically selects the lookbehind window depending on the current step used for building the graph (e.g. `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries)). For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`. It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana. - Lookbehind window in square brackets may be omitted. VictoriaMetrics automatically selects the lookbehind window depending on the current step used for building the graph (e.g. `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries)). For instance, the following query is valid in VictoriaMetrics: `rate(node_network_receive_bytes_total)`. It is equivalent to `rate(node_network_receive_bytes_total[$__interval])` when used in Grafana.
- [Aggregate functions](#aggregate-functions) accept arbitrary number of args. For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`. - [Aggregate functions](#aggregate-functions) accept arbitrary number of args. For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
- [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query. For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
- Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier). For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
- [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax. For instance, `rate(metric[10i] offset 5i)` would return per-second rate over a range covering 10 previous steps with the offset of 5 steps. - [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax. For instance, `rate(metric[10i] offset 5i)` would return per-second rate over a range covering 10 previous steps with the offset of 5 steps.
- [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier) may be put anywere in the query. For instance, `sum(foo) offset 24h`. - [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier) may be put anywere in the query. For instance, `sum(foo) offset 24h`.
- [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier) may be negative. For example, `q offset -1h`.
- Lookbehind window in square brackets and [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier) may be fractional. For instance, `rate(node_network_receive_bytes_total[1.5m] offset 0.5d)`. - Lookbehind window in square brackets and [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier) may be fractional. For instance, `rate(node_network_receive_bytes_total[1.5m] offset 0.5d)`.
- The duration suffix is optional. The duration is in seconds if the suffix is missing. For example, `rate(m[300] offset 1800)` is equivalent to `rate(m[5m]) offset 30m`. - The duration suffix is optional. The duration is in seconds if the suffix is missing. For example, `rate(m[300] offset 1800)` is equivalent to `rate(m[5m]) offset 30m`.
- The duration can be placed anywhere in the query. For example, `sum_over_time(m[1h]) / 1h` is equivalent to `sum_over_time(m[1h]) / 3600`. - The duration can be placed anywhere in the query. For example, `sum_over_time(m[1h]) / 1h` is equivalent to `sum_over_time(m[1h]) / 3600`.
@ -48,6 +49,7 @@ This functionality can be evaluated at [an editable Grafana dashboard](https://p
- `ifnot` binary operator. `q1 ifnot q2` removes values from `q1` for existing values from `q2`. - `ifnot` binary operator. `q1 ifnot q2` removes values from `q1` for existing values from `q2`.
- String literals may be concatenated. This is useful with `WITH` templates: `WITH (commonPrefix="long_metric_prefix_") {__name__=commonPrefix+"suffix1"} / {__name__=commonPrefix+"suffix2"}`. - String literals may be concatenated. This is useful with `WITH` templates: `WITH (commonPrefix="long_metric_prefix_") {__name__=commonPrefix+"suffix1"} / {__name__=commonPrefix+"suffix2"}`.
- `WITH` templates. This feature simplifies writing and managing complex queries. Go to [WITH templates playground](https://play.victoriametrics.com/promql/expand-with-exprs) and try it. - `WITH` templates. This feature simplifies writing and managing complex queries. Go to [WITH templates playground](https://play.victoriametrics.com/promql/expand-with-exprs) and try it.
- `keep_metric_names` modifier can be applied to all the [rollup functions](#rollup-functions) and [transform functions](#transform-functions). This modifier prevents from dropping metric names in function results. For example, `rate({__name__=~"foo|bar"}[5m]) keep_metric_names` leaves `foo` and `bar` metric names in the resulting time series.
## MetricsQL functions ## MetricsQL functions
@ -70,6 +72,7 @@ MetricsQL provides the following functions:
* If lookbehind window in square brackets is missing, then MetricsQL automatically sets the lookbehind window to the interval between points on the graph (aka `step` query arg at [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries), `$__interval` value from Grafana or `1i` duration in MetricsQL). For example, `rate(http_requests_total)` is equivalent to `rate(http_requests_total[$__interval])` in Grafana. It is also equivalent to `rate(http_requests_total[1i])`. * If lookbehind window in square brackets is missing, then MetricsQL automatically sets the lookbehind window to the interval between points on the graph (aka `step` query arg at [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries), `$__interval` value from Grafana or `1i` duration in MetricsQL). For example, `rate(http_requests_total)` is equivalent to `rate(http_requests_total[$__interval])` in Grafana. It is also equivalent to `rate(http_requests_total[1i])`.
* Every [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) in MetricsQL must be wrapped into a rollup function. Otherwise it is automatically wrapped into [default_rollup](#default_rollup). For example, `foo{bar="baz"}` is automatically converted to `default_rollup(foo{bar="baz"}[1i])` before performing the calculations. * Every [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) in MetricsQL must be wrapped into a rollup function. Otherwise it is automatically wrapped into [default_rollup](#default_rollup). For example, `foo{bar="baz"}` is automatically converted to `default_rollup(foo{bar="baz"}[1i])` before performing the calculations.
* If something other than [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) is passed to rollup function, then the inner arg is automatically converted to a [subquery](#subqueries). * If something other than [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) is passed to rollup function, then the inner arg is automatically converted to a [subquery](#subqueries).
* All the rollup functions accept optional `keep_metric_names` modifier. If it is set, then the function keeps metric names in results. For example, `rate({__name__=~"foo|bar}[5m]) keep_metric_names` leaves `foo` and `bar` metric names in results.
See also [implicit query conversions](#implicit-query-conversions). See also [implicit query conversions](#implicit-query-conversions).
@ -84,7 +87,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### ascent_over_time #### ascent_over_time
`ascent_over_time(series_selector[d])` calculates ascent of raw sample values on the given lookbehind window `d`. The calculations are performed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Useful for tracking height gains in GPS tracking. Metric names are stripped from the resulting rollups. See also [descent_over_time](#descent_over_time). `ascent_over_time(series_selector[d])` calculates ascent of raw sample values on the given lookbehind window `d`. The calculations are performed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Useful for tracking height gains in GPS tracking. Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [descent_over_time](#descent_over_time).
#### avg_over_time #### avg_over_time
@ -92,35 +95,35 @@ See also [implicit query conversions](#implicit-query-conversions).
#### changes #### changes
`changes(series_selector[d])` calculates the number of times the raw samples changed on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Unlike `changes()` in Prometheus it takes into account the change from the last sample before the given lookbehind window `d`. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [changes_prometheus](#changes_prometheus). `changes(series_selector[d])` calculates the number of times the raw samples changed on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Unlike `changes()` in Prometheus it takes into account the change from the last sample before the given lookbehind window `d`. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [changes_prometheus](#changes_prometheus).
#### changes_prometheus #### changes_prometheus
`changes_prometheus(series_selector[d])` calculates the number of times the raw samples changed on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It doesn't take into account the change from the last sample before the given lookbehind window `d` in the same way as Prometheus does. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [changes](#changes). `changes_prometheus(series_selector[d])` calculates the number of times the raw samples changed on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It doesn't take into account the change from the last sample before the given lookbehind window `d` in the same way as Prometheus does. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [changes](#changes).
#### count_eq_over_time #### count_eq_over_time
`count_eq_over_time(series_selector[d], eq)` calculates the number of raw samples on the given lookbehind window `d`, which are equal to `eq`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [count_over_time](#count_over_time). `count_eq_over_time(series_selector[d], eq)` calculates the number of raw samples on the given lookbehind window `d`, which are equal to `eq`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [count_over_time](#count_over_time).
#### count_gt_over_time #### count_gt_over_time
`count_gt_over_time(series_selector[d], gt)` calculates the number of raw samples on the given lookbehind window `d`, which are bigger than `gt`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [count_over_time](#count_over_time). `count_gt_over_time(series_selector[d], gt)` calculates the number of raw samples on the given lookbehind window `d`, which are bigger than `gt`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [count_over_time](#count_over_time).
#### count_le_over_time #### count_le_over_time
`count_le_over_time(series_selector[d], le)` calculates the number of raw samples on the given lookbehind window `d`, which don't exceed `le`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [count_over_time](#count_over_time). `count_le_over_time(series_selector[d], le)` calculates the number of raw samples on the given lookbehind window `d`, which don't exceed `le`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [count_over_time](#count_over_time).
#### count_ne_over_time #### count_ne_over_time
`count_ne_over_time(series_selector[d], ne)` calculates the number of raw samples on the given lookbehind window `d`, which aren't equal to `ne`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [count_over_time](#count_over_time). `count_ne_over_time(series_selector[d], ne)` calculates the number of raw samples on the given lookbehind window `d`, which aren't equal to `ne`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [count_over_time](#count_over_time).
#### count_over_time #### count_over_time
`count_over_time(series_selector[d])` calculates the number of raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [count_le_over_time](#count_le_over_time), [count_gt_over_time](#count_gt_over_time), [count_eq_over_time](#count_eq_over_time) and [count_ne_over_time](#count_ne_over_time). `count_over_time(series_selector[d])` calculates the number of raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [count_le_over_time](#count_le_over_time), [count_gt_over_time](#count_gt_over_time), [count_eq_over_time](#count_eq_over_time) and [count_ne_over_time](#count_ne_over_time).
#### decreases_over_time #### decreases_over_time
`decreases_over_time(series_selector[d])` calculates the number of raw sample value decreases over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [increases_over_time](#increases_over_time). `decreases_over_time(series_selector[d])` calculates the number of raw sample value decreases over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [increases_over_time](#increases_over_time).
#### default_rollup #### default_rollup
@ -128,27 +131,27 @@ See also [implicit query conversions](#implicit-query-conversions).
#### delta #### delta
`delta(series_selector[d])` calculates the difference between the last sample before the given lookbehind window `d` and the last sample at the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). The behaviour of `delta()` function in MetricsQL is slighly different to the behaviour of `delta()` function in Prometheus. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [increase](#increase) and [delta_prometheus](#delta_prometheus). `delta(series_selector[d])` calculates the difference between the last sample before the given lookbehind window `d` and the last sample at the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). The behaviour of `delta()` function in MetricsQL is slighly different to the behaviour of `delta()` function in Prometheus. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [increase](#increase) and [delta_prometheus](#delta_prometheus).
#### delta_prometheus #### delta_prometheus
`delta_prometheus(series_selector[d])` calculates the difference between the first and the last samples at the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). The behaviour of `delta_prometheus()` is close to the behaviour of `delta()` function in Prometheus. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. See also [delta](#delta). `delta_prometheus(series_selector[d])` calculates the difference between the first and the last samples at the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). The behaviour of `delta_prometheus()` is close to the behaviour of `delta()` function in Prometheus. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [delta](#delta).
#### deriv #### deriv
`deriv(series_selector[d])` calculates per-second derivative over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). The derivative is calculated using linear regression. Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [deriv_fast](#deriv_fast) and [ideriv](#ideriv). `deriv(series_selector[d])` calculates per-second derivative over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). The derivative is calculated using linear regression. Metric names are stripped from the resulting rollups. This function is supported by PromQL. Add `keep_metric_names` modifier in order to keep metric names. See also [deriv_fast](#deriv_fast) and [ideriv](#ideriv).
#### deriv_fast #### deriv_fast
`deriv_fast(series_selector[d])` calculates per-second derivative using the first and the last raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [deriv](#deriv) and [ideriv](#ideriv). `deriv_fast(series_selector[d])` calculates per-second derivative using the first and the last raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [deriv](#deriv) and [ideriv](#ideriv).
#### descent_over_time #### descent_over_time
`descent_over_time(series_selector[d])` calculates descent of raw sample values on the given lookbehind window `d`. The calculations are performed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Useful for tracking height loss in GPS tracking. Metric names are stripped from the resulting rollups. See also [ascent_over_time](#ascent_over_time). `descent_over_time(series_selector[d])` calculates descent of raw sample values on the given lookbehind window `d`. The calculations are performed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Useful for tracking height loss in GPS tracking. Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [ascent_over_time](#ascent_over_time).
#### distinct_over_time #### distinct_over_time
`distinct_over_time(series_selector[d])` returns the number of distinct raw sample values on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. `distinct_over_time(series_selector[d])` returns the number of distinct raw sample values on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names.
#### duration_over_time #### duration_over_time
@ -160,7 +163,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### geomean_over_time #### geomean_over_time
`geomean_over_time(series_selector[d])` calculates [geometric mean](https://en.wikipedia.org/wiki/Geometric_mean) over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. `geomean_over_time(series_selector[d])` calculates [geometric mean](https://en.wikipedia.org/wiki/Geometric_mean) over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names.
#### histogram_over_time #### histogram_over_time
@ -180,19 +183,19 @@ See also [implicit query conversions](#implicit-query-conversions).
#### idelta #### idelta
`idelta(series_selector[d])` calculates the difference between the last two raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. This function is supported by PromQL. `idelta(series_selector[d])` calculates the difference between the last two raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### ideriv #### ideriv
`ideriv(series_selector[d])` calculates the per-second derivative based on the last two raw samples over the given lookbehind window `d`. The derivative is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [deriv](#deriv). `ideriv(series_selector[d])` calculates the per-second derivative based on the last two raw samples over the given lookbehind window `d`. The derivative is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [deriv](#deriv).
#### increase #### increase
`increase(series_selector[d])` calculates the increase over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It is expected that the `series_selector` returns time series of [counter type](https://prometheus.io/docs/concepts/metric_types/#counter). Unlike Prometheus it takes into account the last sample before the given lookbehind window `d` when calculating the result. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta). `increase(series_selector[d])` calculates the increase over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It is expected that the `series_selector` returns time series of [counter type](https://prometheus.io/docs/concepts/metric_types/#counter). Unlike Prometheus it takes into account the last sample before the given lookbehind window `d` when calculating the result. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
#### increase_prometheus #### increase_prometheus
`increase_prometheus(series_selector[d])` calculates the increase over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It is expected that the `series_selector` returns time series of [counter type](https://prometheus.io/docs/concepts/metric_types/#counter). It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [increase_pure](#increase_pure) and [increase](#increase). `increase_prometheus(series_selector[d])` calculates the increase over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It is expected that the `series_selector` returns time series of [counter type](https://prometheus.io/docs/concepts/metric_types/#counter). It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details. Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [increase_pure](#increase_pure) and [increase](#increase).
#### increase_pure #### increase_pure
@ -200,19 +203,19 @@ See also [implicit query conversions](#implicit-query-conversions).
#### increases_over_time #### increases_over_time
`increases_over_time(series_selector[d])` calculates the number of raw sample value increases over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [decreases_over_time](#decreases_over_time). `increases_over_time(series_selector[d])` calculates the number of raw sample value increases over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [decreases_over_time](#decreases_over_time).
#### integrate #### integrate
`integrate(series_selector[d])` calculates the integral over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. `integrate(series_selector[d])` calculates the integral over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names.
#### irate #### irate
`irate(series_selector[d])` calculates the "instant" per-second increase rate over the last two raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It is expected that the `series_selector` returns time series of [counter type](https://prometheus.io/docs/concepts/metric_types/#counter). Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [rate](#rate). `irate(series_selector[d])` calculates the "instant" per-second increase rate over the last two raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It is expected that the `series_selector` returns time series of [counter type](https://prometheus.io/docs/concepts/metric_types/#counter). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [rate](#rate).
#### lag #### lag
`lag(series_selector[d])` returns the duration in seconds between the last sample on the given lookbehind window `d` and the timestamp of the current point. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [lifetime](#lifetime) and [duration_over_time](#duration_over_time). `lag(series_selector[d])` returns the duration in seconds between the last sample on the given lookbehind window `d` and the timestamp of the current point. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [lifetime](#lifetime) and [duration_over_time](#duration_over_time).
#### last_over_time #### last_over_time
@ -220,7 +223,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### lifetime #### lifetime
`lifetime(series_selector[d])` returns the duration in seconds between the last and the first sample on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [duration_over_time](#duration_over_time) and [lag](#lag). `lifetime(series_selector[d])` returns the duration in seconds between the last and the first sample on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [duration_over_time](#duration_over_time) and [lag](#lag).
#### max_over_time #### max_over_time
@ -244,7 +247,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### present_over_time #### present_over_time
`present_over_time(series_selector[d])` returns 1 if there is at least a single raw sample on the given lookbehind window `d`. Otherwise an empty result is returned. Metric names are stripped from the resulting rollups. This function is supported by PromQL. `present_over_time(series_selector[d])` returns 1 if there is at least a single raw sample on the given lookbehind window `d`. Otherwise an empty result is returned. Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### quantile_over_time #### quantile_over_time
@ -256,19 +259,19 @@ See also [implicit query conversions](#implicit-query-conversions).
#### range_over_time #### range_over_time
`range_over_time(series_selector[d])` calculates value range over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_selector[d])`. Metric names are stripped from the resulting rollups. `range_over_time(series_selector[d])` calculates value range over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_selector[d])`. Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names.
#### rate #### rate
`rate(series_selector[d])` calculates the average per-second increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It is expected that the `series_selector` returns time series of [counter type](https://prometheus.io/docs/concepts/metric_types/#counter). Metric names are stripped from the resulting rollups. This function is supported by PromQL. `rate(series_selector[d])` calculates the average per-second increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It is expected that the `series_selector` returns time series of [counter type](https://prometheus.io/docs/concepts/metric_types/#counter). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### rate_over_sum #### rate_over_sum
`rate_over_sum(series_selector[d])` calculates per-second rate over the sum of raw samples on the given lookbehind window `d`. The calculations are performed indiviually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. `rate_over_sum(series_selector[d])` calculates per-second rate over the sum of raw samples on the given lookbehind window `d`. The calculations are performed indiviually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names.
#### resets #### resets
`resets(series_selector[d])` returns the number of [counter](https://prometheus.io/docs/concepts/metric_types/#counter) resets over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It is expected that the `series_selector` returns time series of [counter type](https://prometheus.io/docs/concepts/metric_types/#counter). Metric names are stripped from the resulting rollups. This function is supported by PromQL. `resets(series_selector[d])` returns the number of [counter](https://prometheus.io/docs/concepts/metric_types/#counter) resets over the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). It is expected that the `series_selector` returns time series of [counter type](https://prometheus.io/docs/concepts/metric_types/#counter). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### rollup #### rollup
@ -280,55 +283,59 @@ See also [implicit query conversions](#implicit-query-conversions).
#### rollup_delta #### rollup_delta
`rollup_delta(series_selector[d])` calculates differences between adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated differences. The calculations are performed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [rollup_increase](#rollup_increase). `rollup_delta(series_selector[d])` calculates differences between adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated differences. The calculations are performed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [rollup_increase](#rollup_increase).
#### rollup_deriv #### rollup_deriv
`rollup_deriv(series_selector[d])` calculates per-second derivatives for adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second derivatives. The calculations are performed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. `rollup_deriv(series_selector[d])` calculates per-second derivatives for adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second derivatives. The calculations are performed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names.
#### rollup_increase #### rollup_increase
`rollup_increase(series_selector[d])` calculates increases for adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated increases. The calculations are performed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [rollup_delta](#rollup_delta). `rollup_increase(series_selector[d])` calculates increases for adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated increases. The calculations are performed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
#### rollup_rate #### rollup_rate
`rollup_rate(series_selector[d])` calculates per-second change rates for adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates. The calculations are perfomed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. `rollup_rate(series_selector[d])` calculates per-second change rates for adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates. The calculations are perfomed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names.
#### rollup_scrape_interval #### rollup_scrape_interval
`rollup_scrape_interval(series_selector[d])` calculates the interval in seconds between adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated interval. The calculations are perfomed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [scrape_interval](#scrape_interval). `rollup_scrape_interval(series_selector[d])` calculates the interval in seconds between adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated interval. The calculations are perfomed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [scrape_interval](#scrape_interval).
#### scrape_interval #### scrape_interval
`scrape_interval(series_selector[d])` calculates the average interval in seconds between raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [rollup_scrape_interval](#rollup_scrape_interval). `scrape_interval(series_selector[d])` calculates the average interval in seconds between raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [rollup_scrape_interval](#rollup_scrape_interval).
#### share_gt_over_time #### share_gt_over_time
`share_gt_over_time(series_selector[d], gt)` returns share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are bigger than `gt`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Useful for calculating SLI and SLO. Example: `share_gt_over_time(up[24h], 0)` - returns service availability for the last 24 hours. See also [share_le_over_time](#share_le_over_time). `share_gt_over_time(series_selector[d], gt)` returns share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are bigger than `gt`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. Useful for calculating SLI and SLO. Example: `share_gt_over_time(up[24h], 0)` - returns service availability for the last 24 hours. See also [share_le_over_time](#share_le_over_time).
#### share_le_over_time #### share_le_over_time
`share_le_over_time(series_selector[d], le)` returns share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are smaller or equal to `le`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Useful for calculating SLI and SLO. Example: `share_le_over_time(memory_usage_bytes[24h], 100*1024*1024)` returns the share of time series values for the last 24 hours when memory usage was below or equal to 100MB. See also [share_gt_over_time](#share_gt_over_time). `share_le_over_time(series_selector[d], le)` returns share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are smaller or equal to `le`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. Useful for calculating SLI and SLO. Example: `share_le_over_time(memory_usage_bytes[24h], 100*1024*1024)` returns the share of time series values for the last 24 hours when memory usage was below or equal to 100MB. See also [share_gt_over_time](#share_gt_over_time).
#### stale_samples_over_time
`stale_samples_over_time(series_selector[d])` calculates the number of [staleness markers](https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers) on the given lookbehind window `d` per each time series matching the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names.
#### stddev_over_time #### stddev_over_time
`stddev_over_time(series_selector[d])` calculates standard deviation over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [stdvar_over_time](#stdvar_over_time). `stddev_over_time(series_selector[d])` calculates standard deviation over raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [stdvar_over_time](#stdvar_over_time).
#### stdvar_over_time #### stdvar_over_time
`stdvar_over_time(series_selector[d])` calculates stadnard variance over raw samples on the given lookbheind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [stddev_over_time](#stddev_over_time). `stdvar_over_time(series_selector[d])` calculates stadnard variance over raw samples on the given lookbheind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [stddev_over_time](#stddev_over_time).
#### sum_over_time #### sum_over_time
`sum_over_time(series_selector[d])` calculates the sum of raw sample values on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. This function is supported by PromQL. `sum_over_time(series_selector[d])` calculates the sum of raw sample values on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### sum2_over_time #### sum2_over_time
`sum2_over_time(series_selector[d])` calculates the sum of squares for raw sample values on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. `sum2_over_time(series_selector[d])` calculates the sum of squares for raw sample values on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names.
#### timestamp #### timestamp
`timestamp(series_selector[d])` returns the timestamp in seconds for the last raw sample on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. This function is supported by PromQL. See also [timestamp_with_name](#timestamp_with_name). `timestamp(series_selector[d])` returns the timestamp in seconds for the last raw sample on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [timestamp_with_name](#timestamp_with_name).
#### timestamp_with_name #### timestamp_with_name
@ -336,7 +343,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### tfirst_over_time #### tfirst_over_time
`tfirst_over_time(series_selector[d])` returns the timestamp in seconds for the first raw sample on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [first_over_time](#first_over_time). `tfirst_over_time(series_selector[d])` returns the timestamp in seconds for the first raw sample on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [first_over_time](#first_over_time).
#### tlast_over_time #### tlast_over_time
@ -344,21 +351,22 @@ See also [implicit query conversions](#implicit-query-conversions).
#### tmax_over_time #### tmax_over_time
`tmax_over_time(series_selector[d])` returns the timestamp in seconds for the raw sample with the maximum value on the given lookbehind window `d`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [max_over_time](#max_over_time). `tmax_over_time(series_selector[d])` returns the timestamp in seconds for the raw sample with the maximum value on the given lookbehind window `d`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [max_over_time](#max_over_time).
#### tmin_over_time #### tmin_over_time
`tmin_over_time(series_selector[d])` returns the timestamp in seconds for the raw sample with the minimum value on the given lookbehind window `d`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [min_over_time](#min_over_time). `tmin_over_time(series_selector[d])` returns the timestamp in seconds for the raw sample with the minimum value on the given lookbehind window `d`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names. See also [min_over_time](#min_over_time).
#### zscore_over_time #### zscore_over_time
`zscore_over_time(series_selector[d])` calculates returns [z-score](https://en.wikipedia.org/wiki/Standard_score) for raw samples on the given lookbehind window `d`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. `zscore_over_time(series_selector[d])` calculates returns [z-score](https://en.wikipedia.org/wiki/Standard_score) for raw samples on the given lookbehind window `d`. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. Add `keep_metric_names` modifier in order to keep metric names.
### Transform functions ### Transform functions
**Transform functions** calculate transformations over rollup results. For example, `abs(delta(temperature[24h]))` calculates the absolute value for every point of every time series returned from the rollup `delta(temperature[24h])`. Additional details: **Transform functions** calculate transformations over rollup results. For example, `abs(delta(temperature[24h]))` calculates the absolute value for every point of every time series returned from the rollup `delta(temperature[24h])`. Additional details:
* If transform function is applied directly to a [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors), then the [default_rollup()](#default_rollup) function is automatically applied before calculating the transformations. For example, `abs(temperature)` is implicitly transformed to `abs(default_rollup(temperature[1i]))`. * If transform function is applied directly to a [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors), then the [default_rollup()](#default_rollup) function is automatically applied before calculating the transformations. For example, `abs(temperature)` is implicitly transformed to `abs(default_rollup(temperature[1i]))`.
* All the transform functions accept optional `keep_metric_names` modifier. If it is set, then the function doesn't drop metric names from the resulting time series. For example, `ln({__name__=~"foo|bar"}) keep_metric_names` leaves `foo` and `bar` metric names in results.
See also [implicit query conversions](#implicit-query-conversions). See also [implicit query conversions](#implicit-query-conversions).
@ -373,39 +381,39 @@ See also [implicit query conversions](#implicit-query-conversions).
#### acos #### acos
`acos(q)` returns [inverse cosine](https://en.wikipedia.org/wiki/Inverse_trigonometric_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [asin](#asin) and [cos](#cos). `acos(q)` returns [inverse cosine](https://en.wikipedia.org/wiki/Inverse_trigonometric_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [asin](#asin) and [cos](#cos).
#### acosh #### acosh
`acosh(q)` returns [inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [sinh](#cosh). `acosh(q)` returns [inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [sinh](#cosh).
#### asin #### asin
`asin(q)` returns [inverse sine](https://en.wikipedia.org/wiki/Inverse_trigonometric_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [acos](#acos) and [sin](#sin). `asin(q)` returns [inverse sine](https://en.wikipedia.org/wiki/Inverse_trigonometric_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [acos](#acos) and [sin](#sin).
#### asinh #### asinh
`asinh(q)` returns [inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [sinh](#sinh). `asinh(q)` returns [inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [sinh](#sinh).
#### atan #### atan
`atan(q)` returns [inverse tangent](https://en.wikipedia.org/wiki/Inverse_trigonometric_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [tan](#tan). `atan(q)` returns [inverse tangent](https://en.wikipedia.org/wiki/Inverse_trigonometric_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [tan](#tan).
#### atanh #### atanh
`atanh(q)` returns [inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [tanh](#tanh). `atanh(q)` returns [inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [tanh](#tanh).
#### bitmap_and #### bitmap_and
`bitmap_and(q, mask)` - calculates bitwise `v & mask` for every `v` point of every time series returned from `q`. Metric names are stripped from the resulting series. `bitmap_and(q, mask)` - calculates bitwise `v & mask` for every `v` point of every time series returned from `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names.
#### bitmap_or #### bitmap_or
`bitmap_or(q, mask)` calculates bitwise `v | mask` for every `v` point of every time series returned from `q`. Metric names are stripped from the resulting series. `bitmap_or(q, mask)` calculates bitwise `v | mask` for every `v` point of every time series returned from `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names.
#### bitmap_xor #### bitmap_xor
`bitmap_xor(q, mask)` calculates bitwise `v ^ mask` for every `v` point of every time series returned from `q`. Metric names are stripped from the resulting series. `bitmap_xor(q, mask)` calculates bitwise `v ^ mask` for every `v` point of every time series returned from `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names.
#### buckets_limit #### buckets_limit
@ -429,27 +437,27 @@ See also [implicit query conversions](#implicit-query-conversions).
#### cos #### cos
`cos(q)` returns `cos(v)` for every `v` point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [sin](#sin). `cos(q)` returns `cos(v)` for every `v` point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [sin](#sin).
#### cosh #### cosh
`cosh(q)` returns [hyperbolic cosine](https://en.wikipedia.org/wiki/Hyperbolic_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. This function is supported by PromQL. See also [acosh](#acosh). `cosh(q)` returns [hyperbolic cosine](https://en.wikipedia.org/wiki/Hyperbolic_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. This function is supported by PromQL. See also [acosh](#acosh).
#### day_of_month #### day_of_month
`day_of_month(q)` returns the day of month for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[1...31]`. Metric names are stripped from the resulting series. This function is supported by PromQL. `day_of_month(q)` returns the day of month for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[1...31]`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### day_of_week #### day_of_week
`day_of_week(q)` returns the day of week for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[0...6]`, where `0` means Sunday and `6` means Saturday. Metric names are stripped from the resulting series. This function is supported by PromQL. `day_of_week(q)` returns the day of week for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[0...6]`, where `0` means Sunday and `6` means Saturday. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### days_in_month #### days_in_month
`days_in_month(q)` returns the number of days in the month identified by every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[28...31]`. Metric names are stripped from the resulting series. This function is supported by PromQL. `days_in_month(q)` returns the number of days in the month identified by every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[28...31]`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### deg #### deg
`deg(q)` converts [Radians to degrees](https://en.wikipedia.org/wiki/Radian#Conversions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [rad](#rad). `deg(q)` converts [Radians to degrees](https://en.wikipedia.org/wiki/Radian#Conversions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [rad](#rad).
#### end #### end
@ -457,7 +465,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### exp #### exp
`exp(q)` calculates the `e^v` for every point `v` of every time series returned by `q`. Metric names are stripped from the resulting series. See also [ln](#ln). This function is supported by PromQL. `exp(q)` calculates the `e^v` for every point `v` of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. See also [ln](#ln). This function is supported by PromQL.
#### floor #### floor
@ -489,7 +497,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### hour #### hour
`hour(q)` returns the hour for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[0...23]`. Metric names are stripped from the resulting series. This function is supported by PromQL. `hour(q)` returns the hour for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[0...23]`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### interpolate #### interpolate
@ -509,23 +517,23 @@ See also [implicit query conversions](#implicit-query-conversions).
#### ln #### ln
`ln(q)` calculates `ln(v)` for every point `v` of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [exp](#exp) and [log2](#log2). `ln(q)` calculates `ln(v)` for every point `v` of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [exp](#exp) and [log2](#log2).
#### log2 #### log2
`log2(q)` calculates `log2(v)` for every point `v` of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [log10](#log10) and [ln](#ln). `log2(q)` calculates `log2(v)` for every point `v` of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [log10](#log10) and [ln](#ln).
#### log10 #### log10
`log10(q)` calculates `log10(v)` for every point `v` of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [log2](#log2) and [ln](#ln). `log10(q)` calculates `log10(v)` for every point `v` of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [log2](#log2) and [ln](#ln).
#### minute #### minute
`minute(q)` returns the minute for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[0...59]`. Metric names are stripped from the resulting series. This function is supported by PromQL. `minute(q)` returns the minute for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[0...59]`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### month #### month
`month(q)` returns the month for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[1...12]`, where `1` means January and `12` means December. Metric names are stripped from the resulting series. This function is supported by PromQL. `month(q)` returns the month for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. The returned values are in the range `[1...12]`, where `1` means January and `12` means December. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### now #### now
@ -537,7 +545,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### rad #### rad
`rad(q)` converts [degrees to Radians](https://en.wikipedia.org/wiki/Radian#Conversions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. See also [deg](#deg). `rad(q)` converts [degrees to Radians](https://en.wikipedia.org/wiki/Radian#Conversions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL. See also [deg](#deg).
#### prometheus_buckets #### prometheus_buckets
@ -586,7 +594,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### range_sum #### range_sum
`range_sum(q)` calculates the sum of points per each time series returned by `q`. Metric names are stripped from the resulting series. `range_sum(q)` calculates the sum of points per each time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names.
#### remove_resets #### remove_resets
@ -614,7 +622,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### running_sum #### running_sum
`running_sum(q)` calculates the running sum per each time series returned by `q`. Metric names are stripped from the resulting series. `running_sum(q)` calculates the running sum per each time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names.
#### scalar #### scalar
@ -622,23 +630,23 @@ See also [implicit query conversions](#implicit-query-conversions).
#### sgn #### sgn
`sgn(q)` returns `1` if `v>0`, `-1` if `v<0` and `0` if `v==0` for every point `v` of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. `sgn(q)` returns `1` if `v>0`, `-1` if `v<0` and `0` if `v==0` for every point `v` of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### sin #### sin
`sin(q)` returns `sin(v)` for every `v` point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by MetricsQL. See also [cos](#cos). `sin(q)` returns `sin(v)` for every `v` point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by MetricsQL. See also [cos](#cos).
#### sinh #### sinh
`sinh(q)` returns [hyperbolic sine](https://en.wikipedia.org/wiki/Hyperbolic_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by MetricsQL. See also [cosh](#cosh). `sinh(q)` returns [hyperbolic sine](https://en.wikipedia.org/wiki/Hyperbolic_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by MetricsQL. See also [cosh](#cosh).
#### tan #### tan
`tan(q)` returns `tan(v)` for every `v` point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by MetricsQL. See also [atan](#atan). `tan(q)` returns `tan(v)` for every `v` point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by MetricsQL. See also [atan](#atan).
#### tanh #### tanh
`tanh(q)` returns [hyperbolic tangent](https://en.wikipedia.org/wiki/Hyperbolic_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by MetricsQL. See also [atanh](#atanh). `tanh(q)` returns [hyperbolic tangent](https://en.wikipedia.org/wiki/Hyperbolic_functions) for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by MetricsQL. See also [atanh](#atanh).
#### smooth_exponential #### smooth_exponential
@ -662,7 +670,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### sqrt #### sqrt
`sqrt(q)` calculates square root for every point of every time series returned by `q`. Metric names are stripped from the resulting series. This function is supported by PromQL. `sqrt(q)` calculates square root for every point of every time series returned by `q`. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
#### start #### start
@ -694,7 +702,7 @@ See also [implicit query conversions](#implicit-query-conversions).
#### year #### year
`year(q)` returns the year for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. Metric names are stripped from the resulting series. This function is supported by PromQL. `year(q)` returns the year for every point of every time series returned by `q`. It is expected that `q` returns unix timestamps. Metric names are stripped from the resulting series. Add `keep_metric_names` modifier in order to keep metric names. This function is supported by PromQL.
### Label manipulation functions ### Label manipulation functions

View file

@ -148,7 +148,7 @@ vminsert-example-vmcluster-persistent ClusterIP 10.107.47.136 <none>
``` ```
To scrape metrics from Kubernetes with a VictoriaMetrics Cluster we will need to install [VMAgent](https://docs.victoriametrics.com/vmagent.html) with some additional configurations. To scrape metrics from Kubernetes with a VictoriaMetrics Cluster we will need to install [VMAgent](https://docs.victoriametrics.com/vmagent.html) with some additional configurations.
Copy `vminsert-example-vmcluster-persistent` (or whatever user put into metadata.name field [https://docs.victoriametrics.com/getting-started-with-vm-operator.html#example-cluster-config](https://docs.victoriametrics.com/getting-started-with-vm-operator.html#example-cluster-config)) service name and add it to the `remoteWrite` URL from [quick-start example](https://github.com/VictoriaMetrics/operator/blob/master/docs/quick-start.MD#vmagent). Copy `vminsert-example-vmcluster-persistent` (or whatever user put into metadata.name field [https://docs.victoriametrics.com/guides/getting-started-with-vm-operator.html#example-cluster-config](https://docs.victoriametrics.com/guides/getting-started-with-vm-operator.html#example-cluster-config)) service name and add it to the `remoteWrite` URL from [quick-start example](https://github.com/VictoriaMetrics/operator/blob/master/docs/quick-start.MD#vmagent).
Here is an example of the full configuration that we need to apply: Here is an example of the full configuration that we need to apply:
<div class="with-copy" markdown="1"> <div class="with-copy" markdown="1">

18
go.mod
View file

@ -10,8 +10,8 @@ require (
// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/fasthttp v1.1.0
github.com/VictoriaMetrics/metrics v1.18.1 github.com/VictoriaMetrics/metrics v1.18.1
github.com/VictoriaMetrics/metricsql v0.34.0 github.com/VictoriaMetrics/metricsql v0.37.0
github.com/aws/aws-sdk-go v1.42.29 github.com/aws/aws-sdk-go v1.42.35
github.com/cespare/xxhash/v2 v2.1.2 github.com/cespare/xxhash/v2 v2.1.2
github.com/cheggaaa/pb/v3 v3.0.8 github.com/cheggaaa/pb/v3 v3.0.8
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
@ -19,7 +19,7 @@ require (
github.com/go-kit/kit v0.12.0 github.com/go-kit/kit v0.12.0
github.com/golang/snappy v0.0.4 github.com/golang/snappy v0.0.4
github.com/influxdata/influxdb v1.9.5 github.com/influxdata/influxdb v1.9.5
github.com/klauspost/compress v1.13.6 github.com/klauspost/compress v1.14.1
github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/oklog/ulid v1.3.1 github.com/oklog/ulid v1.3.1
@ -29,18 +29,18 @@ require (
github.com/valyala/fastjson v1.6.3 github.com/valyala/fastjson v1.6.3
github.com/valyala/fastrand v1.1.0 github.com/valyala/fastrand v1.1.0
github.com/valyala/fasttemplate v1.2.1 github.com/valyala/fasttemplate v1.2.1
github.com/valyala/gozstd v1.14.2 github.com/valyala/gozstd v1.15.1
github.com/valyala/quicktemplate v1.7.0 github.com/valyala/quicktemplate v1.7.0
golang.org/x/net v0.0.0-20220105145211-5b0dc2dfae98 golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
google.golang.org/api v0.64.0 google.golang.org/api v0.65.0
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
) )
require ( require (
cloud.google.com/go v0.100.2 // indirect cloud.google.com/go v0.100.2 // indirect
cloud.google.com/go/compute v0.1.0 // indirect cloud.google.com/go/compute v1.0.0 // indirect
cloud.google.com/go/iam v0.1.0 // indirect cloud.google.com/go/iam v0.1.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
@ -68,7 +68,7 @@ require (
golang.org/x/text v0.3.7 // indirect golang.org/x/text v0.3.7 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220106181925-4b6d468c965f // indirect google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0 // indirect
google.golang.org/grpc v1.43.0 // indirect google.golang.org/grpc v1.43.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect

38
go.sum
View file

@ -39,8 +39,9 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o=
cloud.google.com/go/bigtable v1.3.0/go.mod h1:z5EyKrPE8OQmeg4h5MNdKvuSnI9CCT49Ki3f23aBzio= cloud.google.com/go/bigtable v1.3.0/go.mod h1:z5EyKrPE8OQmeg4h5MNdKvuSnI9CCT49Ki3f23aBzio=
cloud.google.com/go/compute v0.1.0 h1:rSUBvAyVwNJ5uQCKNJFMwPtTvJkfN38b6Pvb9zZoqJ8=
cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
cloud.google.com/go/compute v1.0.0 h1:SJYBzih8Jj9EUm6IDirxKG0I0AGWduhtb6BmdqWarw4=
cloud.google.com/go/compute v1.0.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/iam v0.1.0 h1:W2vbGCrE3Z7J/x3WXLxxGl9LMSB2uhsAA7Ss/6u/qRY= cloud.google.com/go/iam v0.1.0 h1:W2vbGCrE3Z7J/x3WXLxxGl9LMSB2uhsAA7Ss/6u/qRY=
@ -112,11 +113,10 @@ github.com/VictoriaMetrics/fastcache v1.8.0 h1:ybZqS7kRy8YVzYsI09GLzQhs7iqS6cOEH
github.com/VictoriaMetrics/fastcache v1.8.0/go.mod h1:n7Sl+ioh/HlWeYHLSIBIE8TcZFHg/+xgvomWSS5xuEE= github.com/VictoriaMetrics/fastcache v1.8.0/go.mod h1:n7Sl+ioh/HlWeYHLSIBIE8TcZFHg/+xgvomWSS5xuEE=
github.com/VictoriaMetrics/fasthttp v1.1.0 h1:3crd4YWHsMwu60GUXRH6OstowiFvqrwS4a/ueoLdLL0= github.com/VictoriaMetrics/fasthttp v1.1.0 h1:3crd4YWHsMwu60GUXRH6OstowiFvqrwS4a/ueoLdLL0=
github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR2uydjiWvoLp5ZTqQ= github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR2uydjiWvoLp5ZTqQ=
github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0= github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0=
github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
github.com/VictoriaMetrics/metricsql v0.34.0 h1:zF9yzRyNCAxzgEBBnE4y/p0QYNpSQp2jGEBCVE2fUD0= github.com/VictoriaMetrics/metricsql v0.37.0 h1:zFKC+XJpEhp0TtTa6pD0pnyg9sDLH4U5nCeDUT8eUAw=
github.com/VictoriaMetrics/metricsql v0.34.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VictoriaMetrics/metricsql v0.37.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
@ -161,8 +161,8 @@ github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go v1.42.29 h1:7nR+Ls5GGw1ZglG6YOfLzI8kDSy8didom/pLsb/K0fc= github.com/aws/aws-sdk-go v1.42.35 h1:N4N9buNs4YlosI9N0+WYrq8cIZwdgv34yRbxzZlTvFs=
github.com/aws/aws-sdk-go v1.42.29/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/aws/aws-sdk-go v1.42.35/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o=
@ -652,8 +652,9 @@ github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.14.1 h1:hLQYb23E8/fO+1u53d02A97a8UnsddcvYzq4ERRU4ds=
github.com/klauspost/compress v1.14.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@ -963,15 +964,13 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus=
github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/valyala/gozstd v1.14.2 h1:mtK5+UU774dXzuWtCqukhLyVOCM5NClDU3wUDazx90w= github.com/valyala/gozstd v1.15.1 h1:hpmJYWOpNs0rjDOMdbwWZplfNlfh1tOy0v7XiR9+iGQ=
github.com/valyala/gozstd v1.14.2/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/valyala/gozstd v1.15.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
github.com/valyala/histogram v1.1.2/go.mod h1:CZAr6gK9dbD7hYx2s8WSPh0p5x5wETjC+2b3PJVtEdg=
github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY=
github.com/valyala/quicktemplate v1.7.0 h1:LUPTJmlVcb46OOUY3IeD9DojFpAVbsG+5WFTcjMJzCM= github.com/valyala/quicktemplate v1.7.0 h1:LUPTJmlVcb46OOUY3IeD9DojFpAVbsG+5WFTcjMJzCM=
@ -1167,8 +1166,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220105145211-5b0dc2dfae98 h1:+6WJMRLHlD7X7frgp7TUZ36RnQzSf9wVVTNakEp+nqY= golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
golang.org/x/net v0.0.0-20220105145211-5b0dc2dfae98/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1295,8 +1294,9 @@ golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1446,8 +1446,8 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr
google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E=
google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
google.golang.org/api v0.64.0 h1:l3pi8ncrQgB9+ncFw3A716L8lWujnXniBYbxWqqy6tE= google.golang.org/api v0.65.0 h1:MTW9c+LIBAbwoS1Gb+YV7NjFBt2f7GtAS5hIzh2NjgQ=
google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM= google.golang.org/api v0.65.0/go.mod h1:ArYhxgGadlWmqO1IqVujw6Cs8IdD33bTmzKo2Sh+cbg=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1524,9 +1524,9 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6
google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20220106181925-4b6d468c965f h1:WNKU7UnZ/38XgsuerGDMsLpi7sjUTvAe9obsduTWXjo= google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0 h1:aCsSLXylHWFno0r4S3joLpiaWayvqd2Mn4iSvx4WZZc=
google.golang.org/genproto v0.0.0-20220106181925-4b6d468c965f/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=

View file

@ -99,16 +99,16 @@ func addTasksLabels(tasks []task, nodesLabels, servicesLabels []map[string]strin
if port.Protocol != "tcp" { if port.Protocol != "tcp" {
continue continue
} }
m := map[string]string{ m := make(map[string]string, len(commonLabels)+2)
"__address__": discoveryutils.JoinHostPort(commonLabels["__meta_dockerswarm_node_address"], port.PublishedPort),
"__meta_dockerswarm_task_port_publish_mode": port.PublishMode,
}
for k, v := range commonLabels { for k, v := range commonLabels {
m[k] = v m[k] = v
} }
m["__address__"] = discoveryutils.JoinHostPort(commonLabels["__meta_dockerswarm_node_address"], port.PublishedPort)
m["__meta_dockerswarm_task_port_publish_mode"] = port.PublishMode
ms = append(ms, m) ms = append(ms, m)
} }
for _, na := range task.NetworksAttachments { for _, na := range task.NetworksAttachments {
networkLabels := networksLabels[na.Network.ID]
for _, address := range na.Addresses { for _, address := range na.Addresses {
ip, _, err := net.ParseCIDR(address) ip, _, err := net.ParseCIDR(address)
if err != nil { if err != nil {
@ -120,29 +120,27 @@ func addTasksLabels(tasks []task, nodesLabels, servicesLabels []map[string]strin
if ep.Protocol != "tcp" { if ep.Protocol != "tcp" {
continue continue
} }
m := map[string]string{ m := make(map[string]string, len(commonLabels)+len(networkLabels)+2)
"__address": discoveryutils.JoinHostPort(ip.String(), ep.PublishedPort),
"__meta_dockerswarm_task_port_publish_mode": ep.PublishMode,
}
for k, v := range commonLabels { for k, v := range commonLabels {
m[k] = v m[k] = v
} }
for k, v := range networksLabels[na.Network.ID] { for k, v := range networkLabels {
m[k] = v m[k] = v
} }
m["__address__"] = discoveryutils.JoinHostPort(ip.String(), ep.PublishedPort)
m["__meta_dockerswarm_task_port_publish_mode"] = ep.PublishMode
ms = append(ms, m) ms = append(ms, m)
added = true added = true
} }
if !added { if !added {
m := map[string]string{ m := make(map[string]string, len(commonLabels)+len(networkLabels)+1)
"__address__": discoveryutils.JoinHostPort(ip.String(), port),
}
for k, v := range commonLabels { for k, v := range commonLabels {
m[k] = v m[k] = v
} }
for k, v := range networksLabels[na.Network.ID] { for k, v := range networkLabels {
m[k] = v m[k] = v
} }
m["__address__"] = discoveryutils.JoinHostPort(ip.String(), port)
ms = append(ms, m) ms = append(ms, m)
} }
} }

View file

@ -167,7 +167,7 @@ func Test_addTasksLabels(t *testing.T) {
}, },
want: [][]prompbmarshal.Label{ want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{ discoveryutils.GetSortedLabels(map[string]string{
"__address__": "172.31.40.97:9100", "__address__": "172.31.40.97:6379",
"__meta_dockerswarm_node_address": "172.31.40.97", "__meta_dockerswarm_node_address": "172.31.40.97",
"__meta_dockerswarm_node_availability": "active", "__meta_dockerswarm_node_availability": "active",
"__meta_dockerswarm_node_engine_version": "19.03.11", "__meta_dockerswarm_node_engine_version": "19.03.11",
@ -298,6 +298,7 @@ func Test_addTasksLabels(t *testing.T) {
Protocol: "tcp", Protocol: "tcp",
Name: "redis", Name: "redis",
PublishMode: "ingress", PublishMode: "ingress",
PublishedPort: 6379,
}, },
}, VirtualIPs: []struct { }, VirtualIPs: []struct {
NetworkID string NetworkID string
@ -315,8 +316,7 @@ func Test_addTasksLabels(t *testing.T) {
}, },
want: [][]prompbmarshal.Label{ want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{ discoveryutils.GetSortedLabels(map[string]string{
"__address": "10.10.15.15:0", "__address__": "10.10.15.15:6379",
"__address__": "172.31.40.97:9100",
"__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1", "__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_dockerswarm_network_ingress": "true", "__meta_dockerswarm_network_ingress": "true",
"__meta_dockerswarm_network_internal": "false", "__meta_dockerswarm_network_internal": "false",

View file

@ -63,6 +63,15 @@ type apiWatcher struct {
func newAPIWatcher(apiServer string, ac *promauth.Config, sdc *SDConfig, swcFunc ScrapeWorkConstructorFunc) *apiWatcher { func newAPIWatcher(apiServer string, ac *promauth.Config, sdc *SDConfig, swcFunc ScrapeWorkConstructorFunc) *apiWatcher {
namespaces := sdc.Namespaces.Names namespaces := sdc.Namespaces.Names
if len(namespaces) == 0 {
if sdc.Namespaces.OwnNamespace {
namespace, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
if err != nil {
logger.Fatalf("cannot determine namespace for the current pod according to `own_namespace: true` option in kubernetes_sd_config: %s", err)
}
namespaces = []string{string(namespace)}
}
}
selectors := sdc.Selectors selectors := sdc.Selectors
proxyURL := sdc.ProxyURL.URL() proxyURL := sdc.ProxyURL.URL()
gw := getGroupWatcher(apiServer, ac, namespaces, selectors, proxyURL) gw := getGroupWatcher(apiServer, ac, namespaces, selectors, proxyURL)

View file

@ -43,6 +43,7 @@ func (sdc *SDConfig) role() string {
// Namespaces represents namespaces for SDConfig // Namespaces represents namespaces for SDConfig
type Namespaces struct { type Namespaces struct {
OwnNamespace bool `yaml:"own_namespace"`
Names []string `yaml:"names"` Names []string `yaml:"names"`
} }
@ -71,7 +72,7 @@ func (sdc *SDConfig) GetScrapeWorkObjects() ([]interface{}, error) {
// MustStart initializes sdc before its usage. // MustStart initializes sdc before its usage.
// //
// swcFunc is used for constructing such objects. // swcFunc is used for constructing ScrapeWork objects from the given metadata.
func (sdc *SDConfig) MustStart(baseDir string, swcFunc ScrapeWorkConstructorFunc) { func (sdc *SDConfig) MustStart(baseDir string, swcFunc ScrapeWorkConstructorFunc) {
cfg, err := newAPIConfig(sdc, baseDir, swcFunc) cfg, err := newAPIConfig(sdc, baseDir, swcFunc)
if err != nil { if err != nil {

View file

@ -36,7 +36,7 @@ func parseNode(data []byte) (object, error) {
// NodeList represents NodeList from k8s API. // NodeList represents NodeList from k8s API.
// //
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#nodelist-v1-core // See https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/node-v1/#NodeList
type NodeList struct { type NodeList struct {
Metadata ListMeta Metadata ListMeta
Items []*Node Items []*Node
@ -44,20 +44,28 @@ type NodeList struct {
// Node represents Node from k8s API. // Node represents Node from k8s API.
// //
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#node-v1-core // See https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/node-v1/
type Node struct { type Node struct {
Metadata ObjectMeta Metadata ObjectMeta
Status NodeStatus Status NodeStatus
Spec NodeSpec
} }
// NodeStatus represents NodeStatus from k8s API. // NodeStatus represents NodeStatus from k8s API.
// //
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#nodestatus-v1-core // See https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/node-v1/#NodeStatus
type NodeStatus struct { type NodeStatus struct {
Addresses []NodeAddress Addresses []NodeAddress
DaemonEndpoints NodeDaemonEndpoints DaemonEndpoints NodeDaemonEndpoints
} }
// NodeSpec represents NodeSpec from k8s API.
//
// See https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/node-v1/#NodeSpec
type NodeSpec struct {
ProviderID string
}
// NodeAddress represents NodeAddress from k8s API. // NodeAddress represents NodeAddress from k8s API.
// //
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#nodeaddress-v1-core // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#nodeaddress-v1-core
@ -87,6 +95,7 @@ func (n *Node) getTargetLabels(gw *groupWatcher) []map[string]string {
"__address__": addr, "__address__": addr,
"instance": n.Metadata.Name, "instance": n.Metadata.Name,
"__meta_kubernetes_node_name": n.Metadata.Name, "__meta_kubernetes_node_name": n.Metadata.Name,
"__meta_kubernetes_node_provider_id": n.Spec.ProviderID,
} }
n.Metadata.registerLabelsAndAnnotations("__meta_kubernetes_node", m) n.Metadata.registerLabelsAndAnnotations("__meta_kubernetes_node", m)
addrTypesUsed := make(map[string]bool, len(n.Status.Addresses)) addrTypesUsed := make(map[string]bool, len(n.Status.Addresses))

View file

@ -67,7 +67,8 @@ func TestParseNodeListSuccess(t *testing.T) {
"podCIDR": "10.244.0.0/24", "podCIDR": "10.244.0.0/24",
"podCIDRs": [ "podCIDRs": [
"10.244.0.0/24" "10.244.0.0/24"
] ],
"providerID": "aws:///foo-bar/baz"
}, },
"status": { "status": {
"capacity": { "capacity": {
@ -246,6 +247,7 @@ func TestParseNodeListSuccess(t *testing.T) {
"instance": "m01", "instance": "m01",
"__address__": "172.17.0.2:10250", "__address__": "172.17.0.2:10250",
"__meta_kubernetes_node_name": "m01", "__meta_kubernetes_node_name": "m01",
"__meta_kubernetes_node_provider_id": "aws:///foo-bar/baz",
"__meta_kubernetes_node_label_beta_kubernetes_io_arch": "amd64", "__meta_kubernetes_node_label_beta_kubernetes_io_arch": "amd64",
"__meta_kubernetes_node_label_beta_kubernetes_io_os": "linux", "__meta_kubernetes_node_label_beta_kubernetes_io_os": "linux",

View file

@ -316,6 +316,7 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}, globalStopCh <-chan struct{})
// Do not send staleness markers on graceful shutdown as Prometheus does. // Do not send staleness markers on graceful shutdown as Prometheus does.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2013#issuecomment-1006994079 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2013#issuecomment-1006994079
default: default:
// Send staleness markers when the given target disappears.
sw.sendStaleSeries(lastScrape, "", t, true) sw.sendStaleSeries(lastScrape, "", t, true)
} }
if sw.seriesLimiter != nil { if sw.seriesLimiter != nil {
@ -450,7 +451,7 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error
} }
sw.finalizeLastScrape() sw.finalizeLastScrape()
if !mustSwitchToStreamParse { if !mustSwitchToStreamParse {
// Return wc to the pool only if its size is smaller than -promscrape.minResponseSizeForStreamParse // Return body to the pool only if its size is smaller than -promscrape.minResponseSizeForStreamParse
// This should reduce memory usage when scraping targets which return big responses. // This should reduce memory usage when scraping targets which return big responses.
leveledbytebufferpool.Put(body) leveledbytebufferpool.Put(body)
} }
@ -710,10 +711,13 @@ func (sw *scrapeWork) sendStaleSeries(lastScrape, currScrape string, timestamp i
for i := range samples { for i := range samples {
samples[i].Value = decimal.StaleNaN samples[i].Value = decimal.StaleNaN
} }
staleSamplesCreated.Add(len(samples))
} }
sw.pushData(&wc.writeRequest) sw.pushData(&wc.writeRequest)
} }
var staleSamplesCreated = metrics.NewCounter(`promscrape_stale_samples_created_total`)
func (sw *scrapeWork) getLabelsHash(labels []prompbmarshal.Label) uint64 { func (sw *scrapeWork) getLabelsHash(labels []prompbmarshal.Label) uint64 {
// It is OK if there will be hash collisions for distinct sets of labels, // It is OK if there will be hash collisions for distinct sets of labels,
// since the accuracy for `scrape_series_added` metric may be lower than 100%. // since the accuracy for `scrape_series_added` metric may be lower than 100%.

View file

@ -83,7 +83,7 @@ again:
} }
lex.sTail = s[n+1:] lex.sTail = s[n+1:]
goto again goto again
case '{', '}', '[', ']', '(', ')', ',': case '{', '}', '[', ']', '(', ')', ',', '@':
token = s[:1] token = s[:1]
goto tokenFoundLabel goto tokenFoundLabel
} }

View file

@ -70,6 +70,11 @@ func optimizeBinaryOpArgs(be *BinaryOpExpr) *BinaryOpExpr {
} }
func getMetricExprForOptimization(e Expr) *MetricExpr { func getMetricExprForOptimization(e Expr) *MetricExpr {
re, ok := e.(*RollupExpr)
if ok {
// Try optimizing the inner expression in RollupExpr.
return getMetricExprForOptimization(re.Expr)
}
me, ok := e.(*MetricExpr) me, ok := e.(*MetricExpr)
if ok { if ok {
// Ordinary metric expression, i.e. `foo{bar="baz"}` // Ordinary metric expression, i.e. `foo{bar="baz"}`
@ -95,18 +100,13 @@ func getMetricExprForOptimization(e Expr) *MetricExpr {
return nil return nil
} }
if IsRollupFunc(fe.Name) { if IsRollupFunc(fe.Name) {
for _, arg := range fe.Args { argIdx := GetRollupArgIdx(fe)
re, ok := arg.(*RollupExpr) if argIdx >= len(fe.Args) {
if !ok {
continue
}
if me, ok := re.Expr.(*MetricExpr); ok {
// rollup_func(foo{bar="baz"}[d])
return me
}
}
return nil return nil
} }
arg := fe.Args[argIdx]
return getMetricExprForOptimization(arg)
}
if IsTransformFunc(fe.Name) { if IsTransformFunc(fe.Name) {
switch strings.ToLower(fe.Name) { switch strings.ToLower(fe.Name) {
case "absent", "histogram_quantile", "label_join", "label_replace", "scalar", "vector", case "absent", "histogram_quantile", "label_join", "label_replace", "scalar", "vector",

View file

@ -104,6 +104,9 @@ func mustParseWithArgExpr(s string) *withArgExpr {
func removeParensExpr(e Expr) Expr { func removeParensExpr(e Expr) Expr {
if re, ok := e.(*RollupExpr); ok { if re, ok := e.(*RollupExpr); ok {
re.Expr = removeParensExpr(re.Expr) re.Expr = removeParensExpr(re.Expr)
if re.At != nil {
re.At = removeParensExpr(re.At)
}
return re return re
} }
if be, ok := e.(*BinaryOpExpr); ok { if be, ok := e.(*BinaryOpExpr); ok {
@ -144,6 +147,9 @@ func removeParensExpr(e Expr) Expr {
func simplifyConstants(e Expr) Expr { func simplifyConstants(e Expr) Expr {
if re, ok := e.(*RollupExpr); ok { if re, ok := e.(*RollupExpr); ok {
re.Expr = simplifyConstants(re.Expr) re.Expr = simplifyConstants(re.Expr)
if re.At != nil {
re.At = simplifyConstants(re.At)
}
return re return re
} }
if ae, ok := e.(*AggrFuncExpr); ok { if ae, ok := e.(*AggrFuncExpr); ok {
@ -415,13 +421,17 @@ func (p *parser) parseSingleExpr() (Expr, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
if p.lex.Token != "[" && !isOffset(p.lex.Token) { if !isRollupStartToken(p.lex.Token) {
// There is no rollup expression. // There is no rollup expression.
return e, nil return e, nil
} }
return p.parseRollupExpr(e) return p.parseRollupExpr(e)
} }
func isRollupStartToken(token string) bool {
return token == "[" || token == "@" || isOffset(token)
}
func (p *parser) parseSingleExprWithoutRollupSuffix() (Expr, error) { func (p *parser) parseSingleExprWithoutRollupSuffix() (Expr, error) {
if isPositiveDuration(p.lex.Token) { if isPositiveDuration(p.lex.Token) {
return p.parsePositiveDuration() return p.parsePositiveDuration()
@ -659,17 +669,12 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) {
return se, nil return se, nil
} }
} }
be := &BinaryOpExpr{ be := *t
Op: t.Op, be.Left = left
Bool: t.Bool, be.Right = right
GroupModifier: t.GroupModifier,
JoinModifier: t.JoinModifier,
Left: left,
Right: right,
}
be.GroupModifier.Args = groupModifierArgs be.GroupModifier.Args = groupModifierArgs
be.JoinModifier.Args = joinModifierArgs be.JoinModifier.Args = joinModifierArgs
pe := parensExpr{be} pe := parensExpr{&be}
return &pe, nil return &pe, nil
case *FuncExpr: case *FuncExpr:
args, err := expandWithArgs(was, t.Args) args, err := expandWithArgs(was, t.Args)
@ -678,11 +683,9 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) {
} }
wa := getWithArgExpr(was, t.Name) wa := getWithArgExpr(was, t.Name)
if wa == nil { if wa == nil {
fe := &FuncExpr{ fe := *t
Name: t.Name, fe.Args = args
Args: args, return &fe, nil
}
return fe, nil
} }
return expandWithExprExt(was, wa, args) return expandWithExprExt(was, wa, args)
case *AggrFuncExpr: case *AggrFuncExpr:
@ -694,14 +697,10 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
ae := &AggrFuncExpr{ ae := *t
Name: t.Name, ae.Args = args
Args: args,
Modifier: t.Modifier,
Limit: t.Limit,
}
ae.Modifier.Args = modifierArgs ae.Modifier.Args = modifierArgs
return ae, nil return &ae, nil
case *parensExpr: case *parensExpr:
exprs, err := expandWithArgs(was, *t) exprs, err := expandWithArgs(was, *t)
if err != nil { if err != nil {
@ -752,6 +751,13 @@ func expandWithExpr(was []*withArgExpr, e Expr) (Expr, error) {
} }
re := *t re := *t
re.Expr = eNew re.Expr = eNew
if t.At != nil {
atNew, err := expandWithExpr(was, t.At)
if err != nil {
return nil, err
}
re.At = atNew
}
return &re, nil return &re, nil
case *withExpr: case *withExpr:
wasNew := make([]*withArgExpr, 0, len(was)+len(t.Was)) wasNew := make([]*withArgExpr, 0, len(was)+len(t.Was))
@ -1010,9 +1016,20 @@ func (p *parser) parseFuncExpr() (*FuncExpr, error) {
return nil, err return nil, err
} }
fe.Args = args fe.Args = args
if isKeepMetricNames(p.lex.Token) {
fe.KeepMetricNames = true
if err := p.lex.Next(); err != nil {
return nil, err
}
}
return &fe, nil return &fe, nil
} }
func isKeepMetricNames(token string) bool {
token = strings.ToLower(token)
return token == "keep_metric_names"
}
func (p *parser) parseModifierExpr(me *ModifierExpr) error { func (p *parser) parseModifierExpr(me *ModifierExpr) error {
if !isIdentPrefix(p.lex.Token) { if !isIdentPrefix(p.lex.Token) {
return fmt.Errorf(`ModifierExpr: unexpected token %q; want "ident"`, p.lex.Token) return fmt.Errorf(`ModifierExpr: unexpected token %q; want "ident"`, p.lex.Token)
@ -1268,6 +1285,20 @@ func (p *parser) parseWindowAndStep() (*DurationExpr, *DurationExpr, bool, error
return window, step, inheritStep, nil return window, step, inheritStep, nil
} }
func (p *parser) parseAtExpr() (Expr, error) {
if p.lex.Token != "@" {
return nil, fmt.Errorf(`unexpected token %q; want "@"`, p.lex.Token)
}
if err := p.lex.Next(); err != nil {
return nil, err
}
e, err := p.parseSingleExprWithoutRollupSuffix()
if err != nil {
return nil, fmt.Errorf("cannot parse `@` expresion: %w", err)
}
return e, nil
}
func (p *parser) parseOffset() (*DurationExpr, error) { func (p *parser) parseOffset() (*DurationExpr, error) {
if !isOffset(p.lex.Token) { if !isOffset(p.lex.Token) {
return nil, fmt.Errorf(`offset: unexpected token %q; want "offset"`, p.lex.Token) return nil, fmt.Errorf(`offset: unexpected token %q; want "offset"`, p.lex.Token)
@ -1374,11 +1405,11 @@ func (p *parser) parseIdentExpr() (Expr, error) {
return p.parseAggrFuncExpr() return p.parseAggrFuncExpr()
} }
return p.parseFuncExpr() return p.parseFuncExpr()
case "{", "[", ")", ",": case "{", "[", ")", ",", "@":
p.lex.Prev() p.lex.Prev()
return p.parseMetricExpr() return p.parseMetricExpr()
default: default:
return nil, fmt.Errorf(`identExpr: unexpected token %q; want "(", "{", "[", ")", ","`, p.lex.Token) return nil, fmt.Errorf(`identExpr: unexpected token %q; want "(", "{", "[", ")", "," or "@"`, p.lex.Token)
} }
} }
@ -1417,15 +1448,34 @@ func (p *parser) parseRollupExpr(arg Expr) (Expr, error) {
re.Window = window re.Window = window
re.Step = step re.Step = step
re.InheritStep = inheritStep re.InheritStep = inheritStep
if !isOffset(p.lex.Token) { if !isOffset(p.lex.Token) && p.lex.Token != "@" {
return &re, nil return &re, nil
} }
} }
if p.lex.Token == "@" {
at, err := p.parseAtExpr()
if err != nil {
return nil, err
}
re.At = at
}
if isOffset(p.lex.Token) {
offset, err := p.parseOffset() offset, err := p.parseOffset()
if err != nil { if err != nil {
return nil, err return nil, err
} }
re.Offset = offset re.Offset = offset
}
if p.lex.Token == "@" {
if re.At != nil {
return nil, fmt.Errorf("duplicate `@` token")
}
at, err := p.parseAtExpr()
if err != nil {
return nil, err
}
re.At = at
}
return &re, nil return &re, nil
} }
@ -1562,12 +1612,18 @@ type FuncExpr struct {
// Args contains function args. // Args contains function args.
Args []Expr Args []Expr
// If KeepMetricNames is set to true, then the function should keep metric names.
KeepMetricNames bool
} }
// AppendString appends string representation of fe to dst and returns the result. // AppendString appends string representation of fe to dst and returns the result.
func (fe *FuncExpr) AppendString(dst []byte) []byte { func (fe *FuncExpr) AppendString(dst []byte) []byte {
dst = appendEscapedIdent(dst, fe.Name) dst = appendEscapedIdent(dst, fe.Name)
dst = appendStringArgListExpr(dst, fe.Args) dst = appendStringArgListExpr(dst, fe.Args)
if fe.KeepMetricNames {
dst = append(dst, " keep_metric_names"...)
}
return dst return dst
} }
@ -1677,6 +1733,12 @@ type RollupExpr struct {
// If set to true, then `foo[1h:]` would print the same // If set to true, then `foo[1h:]` would print the same
// instead of `foo[1h]`. // instead of `foo[1h]`.
InheritStep bool InheritStep bool
// At contains an optional expression after `@` modifier.
//
// For example, `foo @ end()` or `bar[5m] @ 12345`
// See https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier
At Expr
} }
// ForSubquery returns true if re represents subquery. // ForSubquery returns true if re represents subquery.
@ -1720,6 +1782,17 @@ func (re *RollupExpr) AppendString(dst []byte) []byte {
dst = append(dst, " offset "...) dst = append(dst, " offset "...)
dst = re.Offset.AppendString(dst) dst = re.Offset.AppendString(dst)
} }
if re.At != nil {
dst = append(dst, " @ "...)
_, needAtParens := re.At.(*BinaryOpExpr)
if needAtParens {
dst = append(dst, '(')
}
dst = re.At.AppendString(dst)
if needAtParens {
dst = append(dst, ')')
}
}
return dst return dst
} }

View file

@ -63,6 +63,7 @@ var rollupFuncs = map[string]bool{
"scrape_interval": true, "scrape_interval": true,
"share_gt_over_time": true, "share_gt_over_time": true,
"share_le_over_time": true, "share_le_over_time": true,
"stale_samples_over_time": true,
"stddev_over_time": true, "stddev_over_time": true,
"stdvar_over_time": true, "stdvar_over_time": true,
"sum_over_time": true, "sum_over_time": true,
@ -84,3 +85,23 @@ func IsRollupFunc(funcName string) bool {
s := strings.ToLower(funcName) s := strings.ToLower(funcName)
return rollupFuncs[s] return rollupFuncs[s]
} }
// GetRollupArgIdx returns the argument index for the given fe, which accepts the rollup argument.
//
// -1 is returned if fe isn't a rollup function.
func GetRollupArgIdx(fe *FuncExpr) int {
funcName := fe.Name
funcName = strings.ToLower(funcName)
if !rollupFuncs[funcName] {
return -1
}
switch funcName {
case "quantile_over_time", "aggr_over_time",
"hoeffding_bound_lower", "hoeffding_bound_upper":
return 1
case "quantiles_over_time":
return len(fe.Args) - 1
default:
return 0
}
}

View file

@ -2562,6 +2562,9 @@ var awsPartition = partition{
endpointKey{ endpointKey{
Region: "ap-southeast-2", Region: "ap-southeast-2",
}: endpoint{}, }: endpoint{},
endpointKey{
Region: "ap-southeast-3",
}: endpoint{},
endpointKey{ endpointKey{
Region: "ca-central-1", Region: "ca-central-1",
}: endpoint{}, }: endpoint{},
@ -19560,6 +19563,9 @@ var awsPartition = partition{
endpointKey{ endpointKey{
Region: "ap-southeast-2", Region: "ap-southeast-2",
}: endpoint{}, }: endpoint{},
endpointKey{
Region: "ap-southeast-3",
}: endpoint{},
endpointKey{ endpointKey{
Region: "ca-central-1", Region: "ca-central-1",
}: endpoint{}, }: endpoint{},
@ -26199,7 +26205,12 @@ var awsusgovPartition = partition{
Endpoints: serviceEndpoints{ Endpoints: serviceEndpoints{
endpointKey{ endpointKey{
Region: "us-gov-east-1", Region: "us-gov-east-1",
}: endpoint{}, }: endpoint{
Hostname: "portal.sso.us-gov-east-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-gov-east-1",
},
},
endpointKey{ endpointKey{
Region: "us-gov-west-1", Region: "us-gov-west-1",
}: endpoint{ }: endpoint{
@ -28016,6 +28027,24 @@ var awsisoPartition = partition{
}, },
Deprecated: boxedTrue, Deprecated: boxedTrue,
}, },
endpointKey{
Region: "us-iso-west-1",
}: endpoint{},
endpointKey{
Region: "us-iso-west-1",
Variant: fipsVariant,
}: endpoint{
Hostname: "dms.us-iso-west-1.c2s.ic.gov",
},
endpointKey{
Region: "us-iso-west-1-fips",
}: endpoint{
Hostname: "dms.us-iso-west-1.c2s.ic.gov",
CredentialScope: credentialScope{
Region: "us-iso-west-1",
},
Deprecated: boxedTrue,
},
}, },
}, },
"ds": service{ "ds": service{

View file

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go" const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK // SDKVersion is the version of this SDK
const SDKVersion = "1.42.29" const SDKVersion = "1.42.35"

View file

@ -3,6 +3,7 @@
before: before:
hooks: hooks:
- ./gen.sh - ./gen.sh
- go install mvdan.cc/garble@latest
builds: builds:
- -
@ -31,6 +32,7 @@ builds:
- mips64le - mips64le
goarm: goarm:
- 7 - 7
gobinary: garble
- -
id: "s2d" id: "s2d"
binary: s2d binary: s2d
@ -57,6 +59,7 @@ builds:
- mips64le - mips64le
goarm: goarm:
- 7 - 7
gobinary: garble
- -
id: "s2sx" id: "s2sx"
binary: s2sx binary: s2sx
@ -84,6 +87,7 @@ builds:
- mips64le - mips64le
goarm: goarm:
- 7 - 7
gobinary: garble
archives: archives:
- -

View file

@ -17,6 +17,13 @@ This package provides various compression algorithms.
# changelog # changelog
* Jan 11, 2022 (v1.14.1)
* s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462)
* flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458)
* zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468)
* zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464)
* Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445)
* Aug 30, 2021 (v1.13.5) * Aug 30, 2021 (v1.13.5)
* gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425)
* s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413)
@ -432,6 +439,13 @@ For more information see my blog post on [Fast Linear Time Compression](http://b
This is implemented on Go 1.7 as "Huffman Only" mode, though not exposed for gzip. This is implemented on Go 1.7 as "Huffman Only" mode, though not exposed for gzip.
# Other packages
Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code):
* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression.
* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression.
* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer.
# license # license

View file

@ -6,9 +6,13 @@
package flate package flate
import ( import (
"encoding/binary"
"fmt" "fmt"
"io" "io"
"math" "math"
"math/bits"
comp "github.com/klauspost/compress"
) )
const ( const (
@ -37,15 +41,17 @@ const (
maxMatchLength = 258 // The longest match for the compressor maxMatchLength = 258 // The longest match for the compressor
minOffsetSize = 1 // The shortest offset that makes any sense minOffsetSize = 1 // The shortest offset that makes any sense
// The maximum number of tokens we put into a single flat block, just too // The maximum number of tokens we will encode at the time.
// stop things from getting too large. // Smaller sizes usually creates less optimal blocks.
maxFlateBlockTokens = 1 << 14 // Bigger can make context switching slow.
// We use this for levels 7-9, so we make it big.
maxFlateBlockTokens = 1 << 15
maxStoreBlockSize = 65535 maxStoreBlockSize = 65535
hashBits = 17 // After 17 performance degrades hashBits = 17 // After 17 performance degrades
hashSize = 1 << hashBits hashSize = 1 << hashBits
hashMask = (1 << hashBits) - 1 hashMask = (1 << hashBits) - 1
hashShift = (hashBits + minMatchLength - 1) / minMatchLength hashShift = (hashBits + minMatchLength - 1) / minMatchLength
maxHashOffset = 1 << 24 maxHashOffset = 1 << 28
skipNever = math.MaxInt32 skipNever = math.MaxInt32
@ -70,9 +76,9 @@ var levels = []compressionLevel{
{0, 0, 0, 0, 0, 6}, {0, 0, 0, 0, 0, 6},
// Levels 7-9 use increasingly more lazy matching // Levels 7-9 use increasingly more lazy matching
// and increasingly stringent conditions for "good enough". // and increasingly stringent conditions for "good enough".
{8, 8, 24, 16, skipNever, 7}, {6, 10, 12, 16, skipNever, 7},
{10, 16, 24, 64, skipNever, 8}, {10, 24, 32, 64, skipNever, 8},
{32, 258, 258, 4096, skipNever, 9}, {32, 258, 258, 1024, skipNever, 9},
} }
// advancedState contains state for the advanced levels, with bigger hash tables, etc. // advancedState contains state for the advanced levels, with bigger hash tables, etc.
@ -94,6 +100,7 @@ type advancedState struct {
// input window: unprocessed data is window[index:windowEnd] // input window: unprocessed data is window[index:windowEnd]
index int index int
estBitsPerByte int
hashMatch [maxMatchLength + minMatchLength]uint32 hashMatch [maxMatchLength + minMatchLength]uint32
hash uint32 hash uint32
@ -170,7 +177,8 @@ func (d *compressor) writeBlock(tok *tokens, index int, eof bool) error {
window = d.window[d.blockStart:index] window = d.window[d.blockStart:index]
} }
d.blockStart = index d.blockStart = index
d.w.writeBlock(tok, eof, window) //d.w.writeBlock(tok, eof, window)
d.w.writeBlockDynamic(tok, eof, window, d.sync)
return d.w.err return d.w.err
} }
return nil return nil
@ -263,7 +271,7 @@ func (d *compressor) fillWindow(b []byte) {
// Try to find a match starting at index whose length is greater than prevSize. // Try to find a match starting at index whose length is greater than prevSize.
// We only look at chainCount possibilities before giving up. // We only look at chainCount possibilities before giving up.
// pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead // pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead
func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { func (d *compressor) findMatch(pos int, prevHead int, lookahead, bpb int) (length, offset int, ok bool) {
minMatchLook := maxMatchLength minMatchLook := maxMatchLength
if lookahead < minMatchLook { if lookahead < minMatchLook {
minMatchLook = lookahead minMatchLook = lookahead
@ -279,22 +287,28 @@ func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead
// If we've got a match that's good enough, only look in 1/4 the chain. // If we've got a match that's good enough, only look in 1/4 the chain.
tries := d.chain tries := d.chain
length = prevLength length = minMatchLength - 1
if length >= d.good {
tries >>= 2
}
wEnd := win[pos+length] wEnd := win[pos+length]
wPos := win[pos:] wPos := win[pos:]
minIndex := pos - windowSize minIndex := pos - windowSize
if minIndex < 0 {
minIndex = 0
}
offset = 0
// Base is 4 bytes at with an additional cost.
// Matches must be better than this.
cGain := minMatchLength*bpb - 12
for i := prevHead; tries > 0; tries-- { for i := prevHead; tries > 0; tries-- {
if wEnd == win[i+length] { if wEnd == win[i+length] {
n := matchLen(win[i:i+minMatchLook], wPos) n := matchLen(win[i:i+minMatchLook], wPos)
if n > length {
if n > length && (n > minMatchLength || pos-i <= 4096) { newGain := n*bpb - bits.Len32(uint32(pos-i))
if newGain > cGain {
length = n length = n
offset = pos - i offset = pos - i
cGain = newGain
ok = true ok = true
if n >= nice { if n >= nice {
// The match is good enough that we don't try to find a better one. // The match is good enough that we don't try to find a better one.
@ -303,12 +317,13 @@ func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead
wEnd = win[pos+n] wEnd = win[pos+n]
} }
} }
if i == minIndex { }
if i <= minIndex {
// hashPrev[i & windowMask] has already been overwritten, so stop now. // hashPrev[i & windowMask] has already been overwritten, so stop now.
break break
} }
i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
if i < minIndex || i < 0 { if i < minIndex {
break break
} }
} }
@ -327,8 +342,7 @@ func (d *compressor) writeStoredBlock(buf []byte) error {
// of the supplied slice. // of the supplied slice.
// The caller must ensure that len(b) >= 4. // The caller must ensure that len(b) >= 4.
func hash4(b []byte) uint32 { func hash4(b []byte) uint32 {
b = b[:4] return hash4u(binary.LittleEndian.Uint32(b), hashBits)
return hash4u(uint32(b[3])|uint32(b[2])<<8|uint32(b[1])<<16|uint32(b[0])<<24, hashBits)
} }
// bulkHash4 will compute hashes using the same // bulkHash4 will compute hashes using the same
@ -337,11 +351,12 @@ func bulkHash4(b []byte, dst []uint32) {
if len(b) < 4 { if len(b) < 4 {
return return
} }
hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 hb := binary.LittleEndian.Uint32(b)
dst[0] = hash4u(hb, hashBits) dst[0] = hash4u(hb, hashBits)
end := len(b) - 4 + 1 end := len(b) - 4 + 1
for i := 1; i < end; i++ { for i := 1; i < end; i++ {
hb = (hb << 8) | uint32(b[i+3]) hb = (hb >> 8) | uint32(b[i+3])<<24
dst[i] = hash4u(hb, hashBits) dst[i] = hash4u(hb, hashBits)
} }
} }
@ -374,10 +389,15 @@ func (d *compressor) deflateLazy() {
if d.windowEnd-s.index < minMatchLength+maxMatchLength && !d.sync { if d.windowEnd-s.index < minMatchLength+maxMatchLength && !d.sync {
return return
} }
s.estBitsPerByte = 8
if !d.sync {
s.estBitsPerByte = comp.ShannonEntropyBits(d.window[s.index:d.windowEnd])
s.estBitsPerByte = int(1 + float64(s.estBitsPerByte)/float64(d.windowEnd-s.index))
}
s.maxInsertIndex = d.windowEnd - (minMatchLength - 1) s.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
if s.index < s.maxInsertIndex { if s.index < s.maxInsertIndex {
s.hash = hash4(d.window[s.index : s.index+minMatchLength]) s.hash = hash4(d.window[s.index:])
} }
for { for {
@ -410,7 +430,7 @@ func (d *compressor) deflateLazy() {
} }
if s.index < s.maxInsertIndex { if s.index < s.maxInsertIndex {
// Update the hash // Update the hash
s.hash = hash4(d.window[s.index : s.index+minMatchLength]) s.hash = hash4(d.window[s.index:])
ch := s.hashHead[s.hash&hashMask] ch := s.hashHead[s.hash&hashMask]
s.chainHead = int(ch) s.chainHead = int(ch)
s.hashPrev[s.index&windowMask] = ch s.hashPrev[s.index&windowMask] = ch
@ -426,12 +446,37 @@ func (d *compressor) deflateLazy() {
} }
if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy {
if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, minMatchLength-1, lookahead); ok { if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, lookahead, s.estBitsPerByte); ok {
s.length = newLength s.length = newLength
s.offset = newOffset s.offset = newOffset
} }
} }
if prevLength >= minMatchLength && s.length <= prevLength { if prevLength >= minMatchLength && s.length <= prevLength {
// Check for better match at end...
//
// checkOff must be >=2 since we otherwise risk checking s.index
// Offset of 2 seems to yield best results.
const checkOff = 2
prevIndex := s.index - 1
if prevIndex+prevLength+checkOff < s.maxInsertIndex {
end := lookahead
if lookahead > maxMatchLength {
end = maxMatchLength
}
end += prevIndex
idx := prevIndex + prevLength - (4 - checkOff)
h := hash4(d.window[idx:])
ch2 := int(s.hashHead[h&hashMask]) - s.hashOffset - prevLength + (4 - checkOff)
if ch2 > minIndex {
length := matchLen(d.window[prevIndex:end], d.window[ch2:])
// It seems like a pure length metric is best.
if length > prevLength {
prevLength = length
prevOffset = prevIndex - ch2
}
}
}
// There was a match at the previous step, and the current match is // There was a match at the previous step, and the current match is
// not better. Output the previous match. // not better. Output the previous match.
d.tokens.AddMatch(uint32(prevLength-3), uint32(prevOffset-minOffsetSize)) d.tokens.AddMatch(uint32(prevLength-3), uint32(prevOffset-minOffsetSize))
@ -479,6 +524,7 @@ func (d *compressor) deflateLazy() {
} }
d.tokens.Reset() d.tokens.Reset()
} }
s.ii = 0
} else { } else {
// Reset, if we got a match this run. // Reset, if we got a match this run.
if s.length >= minMatchLength { if s.length >= minMatchLength {
@ -498,13 +544,12 @@ func (d *compressor) deflateLazy() {
// If we have a long run of no matches, skip additional bytes // If we have a long run of no matches, skip additional bytes
// Resets when s.ii overflows after 64KB. // Resets when s.ii overflows after 64KB.
if s.ii > 31 { if n := int(s.ii) - d.chain; n > 0 {
n := int(s.ii >> 5) n = 1 + int(n>>6)
for j := 0; j < n; j++ { for j := 0; j < n; j++ {
if s.index >= d.windowEnd-1 { if s.index >= d.windowEnd-1 {
break break
} }
d.tokens.AddLiteral(d.window[s.index-1]) d.tokens.AddLiteral(d.window[s.index-1])
if d.tokens.n == maxFlateBlockTokens { if d.tokens.n == maxFlateBlockTokens {
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
@ -512,6 +557,14 @@ func (d *compressor) deflateLazy() {
} }
d.tokens.Reset() d.tokens.Reset()
} }
// Index...
if s.index < s.maxInsertIndex {
h := hash4(d.window[s.index:])
ch := s.hashHead[h]
s.chainHead = int(ch)
s.hashPrev[s.index&windowMask] = ch
s.hashHead[h] = uint32(s.index + s.hashOffset)
}
s.index++ s.index++
} }
// Flush last byte // Flush last byte
@ -611,7 +664,9 @@ func (d *compressor) write(b []byte) (n int, err error) {
} }
n = len(b) n = len(b)
for len(b) > 0 { for len(b) > 0 {
if d.windowEnd == len(d.window) || d.sync {
d.step(d) d.step(d)
}
b = b[d.fill(d, b):] b = b[d.fill(d, b):]
if d.err != nil { if d.err != nil {
return 0, d.err return 0, d.err
@ -652,13 +707,13 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
level = 5 level = 5
fallthrough fallthrough
case level >= 1 && level <= 6: case level >= 1 && level <= 6:
d.w.logNewTablePenalty = 8 d.w.logNewTablePenalty = 7
d.fast = newFastEnc(level) d.fast = newFastEnc(level)
d.window = make([]byte, maxStoreBlockSize) d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillBlock d.fill = (*compressor).fillBlock
d.step = (*compressor).storeFast d.step = (*compressor).storeFast
case 7 <= level && level <= 9: case 7 <= level && level <= 9:
d.w.logNewTablePenalty = 10 d.w.logNewTablePenalty = 8
d.state = &advancedState{} d.state = &advancedState{}
d.compressionLevel = levels[level] d.compressionLevel = levels[level]
d.initDeflate() d.initDeflate()

View file

@ -213,11 +213,9 @@ func (e *fastGen) Reset() {
// matchLen returns the maximum length. // matchLen returns the maximum length.
// 'a' must be the shortest of the two. // 'a' must be the shortest of the two.
func matchLen(a, b []byte) int { func matchLen(a, b []byte) int {
b = b[:len(a)]
var checked int var checked int
for len(a) >= 8 { for len(a) >= 8 {
b = b[:len(a)]
if diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b); diff != 0 { if diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b); diff != 0 {
return checked + (bits.TrailingZeros64(diff) >> 3) return checked + (bits.TrailingZeros64(diff) >> 3)
} }

View file

@ -155,37 +155,33 @@ func (w *huffmanBitWriter) reset(writer io.Writer) {
w.lastHuffMan = false w.lastHuffMan = false
} }
func (w *huffmanBitWriter) canReuse(t *tokens) (offsets, lits bool) { func (w *huffmanBitWriter) canReuse(t *tokens) (ok bool) {
offsets, lits = true, true
a := t.offHist[:offsetCodeCount] a := t.offHist[:offsetCodeCount]
b := w.offsetFreq[:len(a)] b := w.offsetEncoding.codes
for i := range a { b = b[:len(a)]
if b[i] == 0 && a[i] != 0 { for i, v := range a {
offsets = false if v != 0 && b[i].len == 0 {
break return false
} }
} }
a = t.extraHist[:literalCount-256] a = t.extraHist[:literalCount-256]
b = w.literalFreq[256:literalCount] b = w.literalEncoding.codes[256:literalCount]
b = b[:len(a)] b = b[:len(a)]
for i := range a { for i, v := range a {
if b[i] == 0 && a[i] != 0 { if v != 0 && b[i].len == 0 {
lits = false return false
break
} }
} }
if lits {
a = t.litHist[:] a = t.litHist[:256]
b = w.literalFreq[:len(a)] b = w.literalEncoding.codes[:len(a)]
for i := range a { for i, v := range a {
if b[i] == 0 && a[i] != 0 { if v != 0 && b[i].len == 0 {
lits = false return false
break
} }
} }
} return true
return
} }
func (w *huffmanBitWriter) flush() { func (w *huffmanBitWriter) flush() {
@ -222,7 +218,7 @@ func (w *huffmanBitWriter) write(b []byte) {
} }
func (w *huffmanBitWriter) writeBits(b int32, nb uint16) { func (w *huffmanBitWriter) writeBits(b int32, nb uint16) {
w.bits |= uint64(b) << w.nbits w.bits |= uint64(b) << (w.nbits & 63)
w.nbits += nb w.nbits += nb
if w.nbits >= 48 { if w.nbits >= 48 {
w.writeOutBits() w.writeOutBits()
@ -423,7 +419,7 @@ func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) {
func (w *huffmanBitWriter) writeCode(c hcode) { func (w *huffmanBitWriter) writeCode(c hcode) {
// The function does not get inlined if we "& 63" the shift. // The function does not get inlined if we "& 63" the shift.
w.bits |= uint64(c.code) << w.nbits w.bits |= uint64(c.code) << (w.nbits & 63)
w.nbits += c.len w.nbits += c.len
if w.nbits >= 48 { if w.nbits >= 48 {
w.writeOutBits() w.writeOutBits()
@ -566,7 +562,7 @@ func (w *huffmanBitWriter) writeBlock(tokens *tokens, eof bool, input []byte) {
w.lastHeader = 0 w.lastHeader = 0
} }
numLiterals, numOffsets := w.indexTokens(tokens, false) numLiterals, numOffsets := w.indexTokens(tokens, false)
w.generate(tokens) w.generate()
var extraBits int var extraBits int
storedSize, storable := w.storedSize(input) storedSize, storable := w.storedSize(input)
if storable { if storable {
@ -595,7 +591,7 @@ func (w *huffmanBitWriter) writeBlock(tokens *tokens, eof bool, input []byte) {
} }
// Stored bytes? // Stored bytes?
if storable && storedSize < size { if storable && storedSize <= size {
w.writeStoredHeader(len(input), eof) w.writeStoredHeader(len(input), eof)
w.writeBytes(input) w.writeBytes(input)
return return
@ -634,22 +630,39 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
w.lastHeader = 0 w.lastHeader = 0
w.lastHuffMan = false w.lastHuffMan = false
} }
if !sync {
tokens.Fill() // fillReuse enables filling of empty values.
// This will make encodings always reusable without testing.
// However, this does not appear to benefit on most cases.
const fillReuse = false
// Check if we can reuse...
if !fillReuse && w.lastHeader > 0 && !w.canReuse(tokens) {
w.writeCode(w.literalEncoding.codes[endBlockMarker])
w.lastHeader = 0
} }
numLiterals, numOffsets := w.indexTokens(tokens, !sync) numLiterals, numOffsets := w.indexTokens(tokens, !sync)
extraBits := 0
ssize, storable := w.storedSize(input)
const usePrefs = true
if storable || w.lastHeader > 0 {
extraBits = w.extraBitSize()
}
var size int var size int
// Check if we should reuse. // Check if we should reuse.
if w.lastHeader > 0 { if w.lastHeader > 0 {
// Estimate size for using a new table. // Estimate size for using a new table.
// Use the previous header size as the best estimate. // Use the previous header size as the best estimate.
newSize := w.lastHeader + tokens.EstimatedBits() newSize := w.lastHeader + tokens.EstimatedBits()
newSize += newSize >> w.logNewTablePenalty newSize += int(w.literalEncoding.codes[endBlockMarker].len) + newSize>>w.logNewTablePenalty
// The estimated size is calculated as an optimal table. // The estimated size is calculated as an optimal table.
// We add a penalty to make it more realistic and re-use a bit more. // We add a penalty to make it more realistic and re-use a bit more.
reuseSize := w.dynamicReuseSize(w.literalEncoding, w.offsetEncoding) + w.extraBitSize() reuseSize := w.dynamicReuseSize(w.literalEncoding, w.offsetEncoding) + extraBits
// Check if a new table is better. // Check if a new table is better.
if newSize < reuseSize { if newSize < reuseSize {
@ -660,35 +673,79 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
} else { } else {
size = reuseSize size = reuseSize
} }
if preSize := w.fixedSize(extraBits) + 7; usePrefs && preSize < size {
// Check if we get a reasonable size decrease. // Check if we get a reasonable size decrease.
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { if storable && ssize <= size {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
w.writeFixedHeader(eof)
if !sync {
tokens.AddEOB()
}
w.writeTokens(tokens.Slice(), fixedLiteralEncoding.codes, fixedOffsetEncoding.codes)
return
}
// Check if we get a reasonable size decrease.
if storable && ssize <= size {
w.writeStoredHeader(len(input), eof) w.writeStoredHeader(len(input), eof)
w.writeBytes(input) w.writeBytes(input)
w.lastHeader = 0
return return
} }
} }
// We want a new block/table // We want a new block/table
if w.lastHeader == 0 { if w.lastHeader == 0 {
w.generate(tokens) if fillReuse && !sync {
w.fillTokens()
numLiterals, numOffsets = maxNumLit, maxNumDist
} else {
w.literalFreq[endBlockMarker] = 1
}
w.generate()
// Generate codegen and codegenFrequencies, which indicates how to encode // Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding. // the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
w.codegenEncoding.generate(w.codegenFreq[:], 7) w.codegenEncoding.generate(w.codegenFreq[:], 7)
var numCodegens int var numCodegens int
size, numCodegens = w.dynamicSize(w.literalEncoding, w.offsetEncoding, w.extraBitSize()) if fillReuse && !sync {
// Store bytes, if we don't get a reasonable improvement. // Reindex for accurate size...
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { w.indexTokens(tokens, true)
}
size, numCodegens = w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits)
// Store predefined, if we don't get a reasonable improvement.
if preSize := w.fixedSize(extraBits); usePrefs && preSize <= size {
// Store bytes, if we don't get an improvement.
if storable && ssize <= preSize {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
w.writeFixedHeader(eof)
if !sync {
tokens.AddEOB()
}
w.writeTokens(tokens.Slice(), fixedLiteralEncoding.codes, fixedOffsetEncoding.codes)
return
}
if storable && ssize <= size {
// Store bytes, if we don't get an improvement.
w.writeStoredHeader(len(input), eof) w.writeStoredHeader(len(input), eof)
w.writeBytes(input) w.writeBytes(input)
w.lastHeader = 0
return return
} }
// Write Huffman table. // Write Huffman table.
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
if !sync {
w.lastHeader, _ = w.headerSize() w.lastHeader, _ = w.headerSize()
}
w.lastHuffMan = false w.lastHuffMan = false
} }
@ -699,6 +756,19 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
w.writeTokens(tokens.Slice(), w.literalEncoding.codes, w.offsetEncoding.codes) w.writeTokens(tokens.Slice(), w.literalEncoding.codes, w.offsetEncoding.codes)
} }
func (w *huffmanBitWriter) fillTokens() {
for i, v := range w.literalFreq[:literalCount] {
if v == 0 {
w.literalFreq[i] = 1
}
}
for i, v := range w.offsetFreq[:offsetCodeCount] {
if v == 0 {
w.offsetFreq[i] = 1
}
}
}
// indexTokens indexes a slice of tokens, and updates // indexTokens indexes a slice of tokens, and updates
// literalFreq and offsetFreq, and generates literalEncoding // literalFreq and offsetFreq, and generates literalEncoding
// and offsetEncoding. // and offsetEncoding.
@ -733,7 +803,7 @@ func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, num
return return
} }
func (w *huffmanBitWriter) generate(t *tokens) { func (w *huffmanBitWriter) generate() {
w.literalEncoding.generate(w.literalFreq[:literalCount], 15) w.literalEncoding.generate(w.literalFreq[:literalCount], 15)
w.offsetEncoding.generate(w.offsetFreq[:offsetCodeCount], 15) w.offsetEncoding.generate(w.offsetFreq[:offsetCodeCount], 15)
} }
@ -768,7 +838,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
if t < matchType { if t < matchType {
//w.writeCode(lits[t.literal()]) //w.writeCode(lits[t.literal()])
c := lits[t.literal()] c := lits[t.literal()]
bits |= uint64(c.code) << nbits bits |= uint64(c.code) << (nbits & 63)
nbits += c.len nbits += c.len
if nbits >= 48 { if nbits >= 48 {
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
@ -796,7 +866,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
} else { } else {
// inlined // inlined
c := lengths[lengthCode&31] c := lengths[lengthCode&31]
bits |= uint64(c.code) << nbits bits |= uint64(c.code) << (nbits & 63)
nbits += c.len nbits += c.len
if nbits >= 48 { if nbits >= 48 {
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
@ -819,7 +889,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
if extraLengthBits > 0 { if extraLengthBits > 0 {
//w.writeBits(extraLength, extraLengthBits) //w.writeBits(extraLength, extraLengthBits)
extraLength := int32(length - lengthBase[lengthCode&31]) extraLength := int32(length - lengthBase[lengthCode&31])
bits |= uint64(extraLength) << nbits bits |= uint64(extraLength) << (nbits & 63)
nbits += extraLengthBits nbits += extraLengthBits
if nbits >= 48 { if nbits >= 48 {
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
@ -846,7 +916,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
} else { } else {
// inlined // inlined
c := offs[offsetCode] c := offs[offsetCode]
bits |= uint64(c.code) << nbits bits |= uint64(c.code) << (nbits & 63)
nbits += c.len nbits += c.len
if nbits >= 48 { if nbits >= 48 {
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
@ -867,7 +937,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
offsetComb := offsetCombined[offsetCode] offsetComb := offsetCombined[offsetCode]
if offsetComb > 1<<16 { if offsetComb > 1<<16 {
//w.writeBits(extraOffset, extraOffsetBits) //w.writeBits(extraOffset, extraOffsetBits)
bits |= uint64(offset&matchOffsetOnlyMask-(offsetComb&0xffff)) << nbits bits |= uint64(offset-(offsetComb&0xffff)) << (nbits & 63)
nbits += uint16(offsetComb >> 16) nbits += uint16(offsetComb >> 16)
if nbits >= 48 { if nbits >= 48 {
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
@ -996,10 +1066,41 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
encoding := w.literalEncoding.codes[:256] encoding := w.literalEncoding.codes[:256]
// Go 1.16 LOVES having these on stack. At least 1.5x the speed. // Go 1.16 LOVES having these on stack. At least 1.5x the speed.
bits, nbits, nbytes := w.bits, w.nbits, w.nbytes bits, nbits, nbytes := w.bits, w.nbits, w.nbytes
// Unroll, write 3 codes/loop.
// Fastest number of unrolls.
for len(input) > 3 {
// We must have at least 48 bits free.
if nbits >= 8 {
n := nbits >> 3
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
bits >>= (n * 8) & 63
nbits -= n * 8
nbytes += uint8(n)
}
if nbytes >= bufferFlushSize {
if w.err != nil {
nbytes = 0
return
}
_, w.err = w.writer.Write(w.bytes[:nbytes])
nbytes = 0
}
a, b := encoding[input[0]], encoding[input[1]]
bits |= uint64(a.code) << (nbits & 63)
bits |= uint64(b.code) << ((nbits + a.len) & 63)
c := encoding[input[2]]
nbits += b.len + a.len
bits |= uint64(c.code) << (nbits & 63)
nbits += c.len
input = input[3:]
}
// Remaining...
for _, t := range input { for _, t := range input {
// Bitwriting inlined, ~30% speedup // Bitwriting inlined, ~30% speedup
c := encoding[t] c := encoding[t]
bits |= uint64(c.code) << nbits bits |= uint64(c.code) << (nbits & 63)
nbits += c.len nbits += c.len
if debugDeflate { if debugDeflate {
count += int(c.len) count += int(c.len)

View file

@ -328,11 +328,17 @@ func (f *decompressor) nextBlock() {
switch typ { switch typ {
case 0: case 0:
f.dataBlock() f.dataBlock()
if debugDecode {
fmt.Println("stored block")
}
case 1: case 1:
// compressed, fixed Huffman tables // compressed, fixed Huffman tables
f.hl = &fixedHuffmanDecoder f.hl = &fixedHuffmanDecoder
f.hd = nil f.hd = nil
f.huffmanBlockDecoder()() f.huffmanBlockDecoder()()
if debugDecode {
fmt.Println("predefinied huffman block")
}
case 2: case 2:
// compressed, dynamic Huffman tables // compressed, dynamic Huffman tables
if f.err = f.readHuffman(); f.err != nil { if f.err = f.readHuffman(); f.err != nil {
@ -341,6 +347,9 @@ func (f *decompressor) nextBlock() {
f.hl = &f.h1 f.hl = &f.h1
f.hd = &f.h2 f.hd = &f.h2
f.huffmanBlockDecoder()() f.huffmanBlockDecoder()()
if debugDecode {
fmt.Println("dynamic huffman block")
}
default: default:
// 3 is reserved. // 3 is reserved.
if debugDecode { if debugDecode {

View file

@ -20,7 +20,7 @@ type dEntrySingle struct {
// double-symbols decoding // double-symbols decoding
type dEntryDouble struct { type dEntryDouble struct {
seq uint16 seq [4]byte
nBits uint8 nBits uint8
len uint8 len uint8
} }
@ -753,23 +753,21 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
br[stream2].fillFast() br[stream2].fillFast()
val := br[stream].peekBitsFast(d.actualTableLog) val := br[stream].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
br[stream].advance(uint8(v.entry))
buf[off+bufoff*stream] = uint8(v.entry >> 8)
val2 := br[stream2].peekBitsFast(d.actualTableLog) val2 := br[stream2].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
v2 := single[val2&tlMask] v2 := single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry)) br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream] = uint8(v.entry >> 8)
buf[off+bufoff*stream2] = uint8(v2.entry >> 8) buf[off+bufoff*stream2] = uint8(v2.entry >> 8)
val = br[stream].peekBitsFast(d.actualTableLog) val = br[stream].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
br[stream].advance(uint8(v.entry))
buf[off+bufoff*stream+1] = uint8(v.entry >> 8)
val2 = br[stream2].peekBitsFast(d.actualTableLog) val2 = br[stream2].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
v2 = single[val2&tlMask] v2 = single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry)) br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream+1] = uint8(v.entry >> 8)
buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8) buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8)
} }
@ -780,23 +778,21 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
br[stream2].fillFast() br[stream2].fillFast()
val := br[stream].peekBitsFast(d.actualTableLog) val := br[stream].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
br[stream].advance(uint8(v.entry))
buf[off+bufoff*stream] = uint8(v.entry >> 8)
val2 := br[stream2].peekBitsFast(d.actualTableLog) val2 := br[stream2].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
v2 := single[val2&tlMask] v2 := single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry)) br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream] = uint8(v.entry >> 8)
buf[off+bufoff*stream2] = uint8(v2.entry >> 8) buf[off+bufoff*stream2] = uint8(v2.entry >> 8)
val = br[stream].peekBitsFast(d.actualTableLog) val = br[stream].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
br[stream].advance(uint8(v.entry))
buf[off+bufoff*stream+1] = uint8(v.entry >> 8)
val2 = br[stream2].peekBitsFast(d.actualTableLog) val2 = br[stream2].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
v2 = single[val2&tlMask] v2 = single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry)) br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream+1] = uint8(v.entry >> 8)
buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8) buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8)
} }
@ -914,7 +910,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
out := dst out := dst
dstEvery := (dstSize + 3) / 4 dstEvery := (dstSize + 3) / 4
shift := (8 - d.actualTableLog) & 7 shift := (56 + (8 - d.actualTableLog)) & 63
const tlSize = 1 << 8 const tlSize = 1 << 8
single := d.dt.single[:tlSize] single := d.dt.single[:tlSize]
@ -935,79 +931,91 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
// Interleave 2 decodes. // Interleave 2 decodes.
const stream = 0 const stream = 0
const stream2 = 1 const stream2 = 1
br[stream].fillFast() br1 := &br[stream]
br[stream2].fillFast() br2 := &br[stream2]
br1.fillFast()
br2.fillFast()
v := single[br[stream].peekByteFast()>>shift].entry v := single[uint8(br1.value>>shift)].entry
v2 := single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8) buf[off+bufoff*stream] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 := single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2] = uint8(v2 >> 8) buf[off+bufoff*stream2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8) buf[off+bufoff*stream+1] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+1] = uint8(v2 >> 8) buf[off+bufoff*stream2+1] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8) buf[off+bufoff*stream+2] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+2] = uint8(v2 >> 8) buf[off+bufoff*stream2+2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br1.value>>shift)].entry
buf[off+bufoff*stream+3] = uint8(v >> 8) v2 = single[uint8(br2.value>>shift)].entry
br[stream].advance(uint8(v)) br1.bitsRead += uint8(v)
br1.value <<= v & 63
v2 = single[br[stream2].peekByteFast()>>shift].entry br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream2+3] = uint8(v2 >> 8) buf[off+bufoff*stream2+3] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2)) buf[off+bufoff*stream+3] = uint8(v >> 8)
} }
{ {
const stream = 2 const stream = 2
const stream2 = 3 const stream2 = 3
br[stream].fillFast() br1 := &br[stream]
br[stream2].fillFast() br2 := &br[stream2]
br1.fillFast()
br2.fillFast()
v := single[br[stream].peekByteFast()>>shift].entry v := single[uint8(br1.value>>shift)].entry
v2 := single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8) buf[off+bufoff*stream] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 := single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2] = uint8(v2 >> 8) buf[off+bufoff*stream2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8) buf[off+bufoff*stream+1] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+1] = uint8(v2 >> 8) buf[off+bufoff*stream2+1] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8) buf[off+bufoff*stream+2] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+2] = uint8(v2 >> 8) buf[off+bufoff*stream2+2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br1.value>>shift)].entry
buf[off+bufoff*stream+3] = uint8(v >> 8) v2 = single[uint8(br2.value>>shift)].entry
br[stream].advance(uint8(v)) br1.bitsRead += uint8(v)
br1.value <<= v & 63
v2 = single[br[stream2].peekByteFast()>>shift].entry br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream2+3] = uint8(v2 >> 8) buf[off+bufoff*stream2+3] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2)) buf[off+bufoff*stream+3] = uint8(v >> 8)
} }
off += 4 off += 4
@ -1073,7 +1081,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
} }
// Read value and increment offset. // Read value and increment offset.
v := single[br.peekByteFast()>>shift].entry v := single[uint8(br.value>>shift)].entry
nBits := uint8(v) nBits := uint8(v)
br.advance(nBits) br.advance(nBits)
bitsLeft -= int(nBits) bitsLeft -= int(nBits)
@ -1121,7 +1129,7 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
out := dst out := dst
dstEvery := (dstSize + 3) / 4 dstEvery := (dstSize + 3) / 4
const shift = 0 const shift = 56
const tlSize = 1 << 8 const tlSize = 1 << 8
const tlMask = tlSize - 1 const tlMask = tlSize - 1
single := d.dt.single[:tlSize] single := d.dt.single[:tlSize]
@ -1145,37 +1153,41 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
br[stream].fillFast() br[stream].fillFast()
br[stream2].fillFast() br[stream2].fillFast()
v := single[br[stream].peekByteFast()>>shift].entry v := single[uint8(br[stream].value>>shift)].entry
v2 := single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8) buf[off+bufoff*stream] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 := single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2] = uint8(v2 >> 8) buf[off+bufoff*stream2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8) buf[off+bufoff*stream+1] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+1] = uint8(v2 >> 8) buf[off+bufoff*stream2+1] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8) buf[off+bufoff*stream+2] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+2] = uint8(v2 >> 8) buf[off+bufoff*stream2+2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+3] = uint8(v >> 8) buf[off+bufoff*stream+3] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+3] = uint8(v2 >> 8) buf[off+bufoff*stream2+3] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
} }
{ {
@ -1184,37 +1196,41 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
br[stream].fillFast() br[stream].fillFast()
br[stream2].fillFast() br[stream2].fillFast()
v := single[br[stream].peekByteFast()>>shift].entry v := single[uint8(br[stream].value>>shift)].entry
v2 := single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8) buf[off+bufoff*stream] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 := single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2] = uint8(v2 >> 8) buf[off+bufoff*stream2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8) buf[off+bufoff*stream+1] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+1] = uint8(v2 >> 8) buf[off+bufoff*stream2+1] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8) buf[off+bufoff*stream+2] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+2] = uint8(v2 >> 8) buf[off+bufoff*stream2+2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+3] = uint8(v >> 8) buf[off+bufoff*stream+3] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+3] = uint8(v2 >> 8) buf[off+bufoff*stream2+3] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
} }
off += 4 off += 4
@ -1280,7 +1296,7 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
} }
// Read value and increment offset. // Read value and increment offset.
v := single[br.peekByteFast()>>shift].entry v := single[br.peekByteFast()].entry
nBits := uint8(v) nBits := uint8(v)
br.advance(nBits) br.advance(nBits)
bitsLeft -= int(nBits) bitsLeft -= int(nBits)

View file

@ -50,16 +50,23 @@ func (b *bitReader) getBits(n uint8) int {
if n == 0 /*|| b.bitsRead >= 64 */ { if n == 0 /*|| b.bitsRead >= 64 */ {
return 0 return 0
} }
return b.getBitsFast(n) return int(b.get32BitsFast(n))
} }
// getBitsFast requires that at least one bit is requested every time. // get32BitsFast requires that at least one bit is requested every time.
// There are no checks if the buffer is filled. // There are no checks if the buffer is filled.
func (b *bitReader) getBitsFast(n uint8) int { func (b *bitReader) get32BitsFast(n uint8) uint32 {
const regMask = 64 - 1 const regMask = 64 - 1
v := uint32((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) v := uint32((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask))
b.bitsRead += n b.bitsRead += n
return int(v) return v
}
func (b *bitReader) get16BitsFast(n uint8) uint16 {
const regMask = 64 - 1
v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask))
b.bitsRead += n
return v
} }
// fillFast() will make sure at least 32 bits are available. // fillFast() will make sure at least 32 bits are available.

View file

@ -38,7 +38,7 @@ func (b *bitWriter) addBits16NC(value uint16, bits uint8) {
b.nBits += bits b.nBits += bits
} }
// addBits32NC will add up to 32 bits. // addBits32NC will add up to 31 bits.
// It will not check if there is space for them, // It will not check if there is space for them,
// so the caller must ensure that it has flushed recently. // so the caller must ensure that it has flushed recently.
func (b *bitWriter) addBits32NC(value uint32, bits uint8) { func (b *bitWriter) addBits32NC(value uint32, bits uint8) {
@ -46,6 +46,26 @@ func (b *bitWriter) addBits32NC(value uint32, bits uint8) {
b.nBits += bits b.nBits += bits
} }
// addBits64NC will add up to 64 bits.
// There must be space for 32 bits.
func (b *bitWriter) addBits64NC(value uint64, bits uint8) {
if bits <= 31 {
b.addBits32Clean(uint32(value), bits)
return
}
b.addBits32Clean(uint32(value), 32)
b.flush32()
b.addBits32Clean(uint32(value>>32), bits-32)
}
// addBits32Clean will add up to 32 bits.
// It will not check if there is space for them.
// The input must not contain more bits than specified.
func (b *bitWriter) addBits32Clean(value uint32, bits uint8) {
b.bitContainer |= uint64(value) << (b.nBits & 63)
b.nBits += bits
}
// addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. // addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated.
// It will not check if there is space for them, so the caller must ensure that it has flushed recently. // It will not check if there is space for them, so the caller must ensure that it has flushed recently.
func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { func (b *bitWriter) addBits16Clean(value uint16, bits uint8) {

View file

@ -51,7 +51,7 @@ func (b *blockEnc) init() {
if cap(b.literals) < maxCompressedBlockSize { if cap(b.literals) < maxCompressedBlockSize {
b.literals = make([]byte, 0, maxCompressedBlockSize) b.literals = make([]byte, 0, maxCompressedBlockSize)
} }
const defSeqs = 200 const defSeqs = 2000
if cap(b.sequences) < defSeqs { if cap(b.sequences) < defSeqs {
b.sequences = make([]seq, 0, defSeqs) b.sequences = make([]seq, 0, defSeqs)
} }
@ -426,7 +426,7 @@ func fuzzFseEncoder(data []byte) int {
return 0 return 0
} }
enc := fseEncoder{} enc := fseEncoder{}
hist := enc.Histogram()[:256] hist := enc.Histogram()
maxSym := uint8(0) maxSym := uint8(0)
for i, v := range data { for i, v := range data {
v = v & 63 v = v & 63
@ -722,22 +722,47 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error {
println("Encoded seq", seq, s, "codes:", s.llCode, s.mlCode, s.ofCode, "states:", ll.state, ml.state, of.state, "bits:", llB, mlB, ofB) println("Encoded seq", seq, s, "codes:", s.llCode, s.mlCode, s.ofCode, "states:", ll.state, ml.state, of.state, "bits:", llB, mlB, ofB)
} }
seq-- seq--
if llEnc.maxBits+mlEnc.maxBits+ofEnc.maxBits <= 32 { // Store sequences in reverse...
// No need to flush (common)
for seq >= 0 { for seq >= 0 {
s = b.sequences[seq] s = b.sequences[seq]
wr.flush32()
llB, ofB, mlB := llTT[s.llCode], ofTT[s.ofCode], mlTT[s.mlCode]
// tabelog max is 8 for all.
of.encode(ofB)
ml.encode(mlB)
ll.encode(llB)
wr.flush32()
// We checked that all can stay within 32 bits ofB := ofTT[s.ofCode]
wr.addBits32NC(s.litLen, llB.outBits) wr.flush32() // tablelog max is below 8 for each, so it will fill max 24 bits.
wr.addBits32NC(s.matchLen, mlB.outBits) //of.encode(ofB)
wr.addBits32NC(s.offset, ofB.outBits) nbBitsOut := (uint32(of.state) + ofB.deltaNbBits) >> 16
dstState := int32(of.state>>(nbBitsOut&15)) + int32(ofB.deltaFindState)
wr.addBits16NC(of.state, uint8(nbBitsOut))
of.state = of.stateTable[dstState]
// Accumulate extra bits.
outBits := ofB.outBits & 31
extraBits := uint64(s.offset & bitMask32[outBits])
extraBitsN := outBits
mlB := mlTT[s.mlCode]
//ml.encode(mlB)
nbBitsOut = (uint32(ml.state) + mlB.deltaNbBits) >> 16
dstState = int32(ml.state>>(nbBitsOut&15)) + int32(mlB.deltaFindState)
wr.addBits16NC(ml.state, uint8(nbBitsOut))
ml.state = ml.stateTable[dstState]
outBits = mlB.outBits & 31
extraBits = extraBits<<outBits | uint64(s.matchLen&bitMask32[outBits])
extraBitsN += outBits
llB := llTT[s.llCode]
//ll.encode(llB)
nbBitsOut = (uint32(ll.state) + llB.deltaNbBits) >> 16
dstState = int32(ll.state>>(nbBitsOut&15)) + int32(llB.deltaFindState)
wr.addBits16NC(ll.state, uint8(nbBitsOut))
ll.state = ll.stateTable[dstState]
outBits = llB.outBits & 31
extraBits = extraBits<<outBits | uint64(s.litLen&bitMask32[outBits])
extraBitsN += outBits
wr.flush32()
wr.addBits64NC(extraBits, extraBitsN)
if debugSequences { if debugSequences {
println("Encoded seq", seq, s) println("Encoded seq", seq, s)
@ -745,30 +770,6 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error {
seq-- seq--
} }
} else {
for seq >= 0 {
s = b.sequences[seq]
wr.flush32()
llB, ofB, mlB := llTT[s.llCode], ofTT[s.ofCode], mlTT[s.mlCode]
// tabelog max is below 8 for each.
of.encode(ofB)
ml.encode(mlB)
ll.encode(llB)
wr.flush32()
// ml+ll = max 32 bits total
wr.addBits32NC(s.litLen, llB.outBits)
wr.addBits32NC(s.matchLen, mlB.outBits)
wr.flush32()
wr.addBits32NC(s.offset, ofB.outBits)
if debugSequences {
println("Encoded seq", seq, s)
}
seq--
}
}
ml.flush(mlEnc.actualTableLog) ml.flush(mlEnc.actualTableLog)
of.flush(ofEnc.actualTableLog) of.flush(ofEnc.actualTableLog)
ll.flush(llEnc.actualTableLog) ll.flush(llEnc.actualTableLog)
@ -801,14 +802,13 @@ func (b *blockEnc) genCodes() {
// nothing to do // nothing to do
return return
} }
if len(b.sequences) > math.MaxUint16 { if len(b.sequences) > math.MaxUint16 {
panic("can only encode up to 64K sequences") panic("can only encode up to 64K sequences")
} }
// No bounds checks after here: // No bounds checks after here:
llH := b.coders.llEnc.Histogram()[:256] llH := b.coders.llEnc.Histogram()
ofH := b.coders.ofEnc.Histogram()[:256] ofH := b.coders.ofEnc.Histogram()
mlH := b.coders.mlEnc.Histogram()[:256] mlH := b.coders.mlEnc.Histogram()
for i := range llH { for i := range llH {
llH[i] = 0 llH[i] = 0
} }
@ -820,7 +820,8 @@ func (b *blockEnc) genCodes() {
} }
var llMax, ofMax, mlMax uint8 var llMax, ofMax, mlMax uint8
for i, seq := range b.sequences { for i := range b.sequences {
seq := &b.sequences[i]
v := llCode(seq.litLen) v := llCode(seq.litLen)
seq.llCode = v seq.llCode = v
llH[v]++ llH[v]++
@ -844,7 +845,6 @@ func (b *blockEnc) genCodes() {
panic(fmt.Errorf("mlMax > maxMatchLengthSymbol (%d), matchlen: %d", mlMax, seq.matchLen)) panic(fmt.Errorf("mlMax > maxMatchLengthSymbol (%d), matchlen: %d", mlMax, seq.matchLen))
} }
} }
b.sequences[i] = seq
} }
maxCount := func(a []uint32) int { maxCount := func(a []uint32) int {
var max uint32 var max uint32

View file

@ -108,11 +108,6 @@ func (e *fastBase) UseBlock(enc *blockEnc) {
e.blk = enc e.blk = enc
} }
func (e *fastBase) matchlenNoHist(s, t int32, src []byte) int32 {
// Extend the match to be as long as possible.
return int32(matchLen(src[s:], src[t:]))
}
func (e *fastBase) matchlen(s, t int32, src []byte) int32 { func (e *fastBase) matchlen(s, t int32, src []byte) int32 {
if debugAsserts { if debugAsserts {
if s < 0 { if s < 0 {
@ -131,9 +126,24 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 {
panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize)) panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize))
} }
} }
a := src[s:]
b := src[t:]
b = b[:len(a)]
end := int32((len(a) >> 3) << 3)
for i := int32(0); i < end; i += 8 {
if diff := load6432(a, i) ^ load6432(b, i); diff != 0 {
return i + int32(bits.TrailingZeros64(diff)>>3)
}
}
// Extend the match to be as long as possible. a = a[end:]
return int32(matchLen(src[s:], src[t:])) b = b[end:]
for i := range a {
if a[i] != b[i] {
return int32(i) + end
}
}
return int32(len(a)) + end
} }
// Reset the encoding table. // Reset the encoding table.

View file

@ -6,8 +6,6 @@ package zstd
import ( import (
"fmt" "fmt"
"math"
"math/bits"
) )
const ( const (
@ -136,20 +134,7 @@ encodeLoop:
// Consider history as well. // Consider history as well.
var seq seq var seq seq
var length int32 var length int32
// length = 4 + e.matchlen(s+6, repIndex+4, src) length = 4 + e.matchlen(s+6, repIndex+4, src)
{
a := src[s+6:]
b := src[repIndex+4:]
endI := len(a) & (math.MaxInt32 - 7)
length = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
length = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
seq.matchLen = uint32(length - zstdMinMatch) seq.matchLen = uint32(length - zstdMinMatch)
// We might be able to match backwards. // We might be able to match backwards.
@ -236,20 +221,7 @@ encodeLoop:
} }
// Extend the 4-byte match as long as possible. // Extend the 4-byte match as long as possible.
//l := e.matchlen(s+4, t+4, src) + 4 l := e.matchlen(s+4, t+4, src) + 4
var l int32
{
a := src[s+4:]
b := src[t+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
// Extend backwards // Extend backwards
tMin := s - e.maxMatchOff tMin := s - e.maxMatchOff
@ -286,20 +258,7 @@ encodeLoop:
if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) { if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) {
// We have at least 4 byte match. // We have at least 4 byte match.
// No need to check backwards. We come straight from a match // No need to check backwards. We come straight from a match
//l := 4 + e.matchlen(s+4, o2+4, src) l := 4 + e.matchlen(s+4, o2+4, src)
var l int32
{
a := src[s+4:]
b := src[o2+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
// Store this, since we have it. // Store this, since we have it.
nextHash := hashLen(cv, hashLog, tableFastHashLen) nextHash := hashLen(cv, hashLog, tableFastHashLen)
@ -418,21 +377,7 @@ encodeLoop:
if len(blk.sequences) > 2 && load3232(src, repIndex) == uint32(cv>>16) { if len(blk.sequences) > 2 && load3232(src, repIndex) == uint32(cv>>16) {
// Consider history as well. // Consider history as well.
var seq seq var seq seq
// length := 4 + e.matchlen(s+6, repIndex+4, src) length := 4 + e.matchlen(s+6, repIndex+4, src)
// length := 4 + int32(matchLen(src[s+6:], src[repIndex+4:]))
var length int32
{
a := src[s+6:]
b := src[repIndex+4:]
endI := len(a) & (math.MaxInt32 - 7)
length = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
length = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
seq.matchLen = uint32(length - zstdMinMatch) seq.matchLen = uint32(length - zstdMinMatch)
@ -522,21 +467,7 @@ encodeLoop:
panic(fmt.Sprintf("t (%d) < 0 ", t)) panic(fmt.Sprintf("t (%d) < 0 ", t))
} }
// Extend the 4-byte match as long as possible. // Extend the 4-byte match as long as possible.
//l := e.matchlenNoHist(s+4, t+4, src) + 4 l := e.matchlen(s+4, t+4, src) + 4
// l := int32(matchLen(src[s+4:], src[t+4:])) + 4
var l int32
{
a := src[s+4:]
b := src[t+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
// Extend backwards // Extend backwards
tMin := s - e.maxMatchOff tMin := s - e.maxMatchOff
@ -573,21 +504,7 @@ encodeLoop:
if o2 := s - offset2; len(blk.sequences) > 2 && load3232(src, o2) == uint32(cv) { if o2 := s - offset2; len(blk.sequences) > 2 && load3232(src, o2) == uint32(cv) {
// We have at least 4 byte match. // We have at least 4 byte match.
// No need to check backwards. We come straight from a match // No need to check backwards. We come straight from a match
//l := 4 + e.matchlenNoHist(s+4, o2+4, src) l := 4 + e.matchlen(s+4, o2+4, src)
// l := 4 + int32(matchLen(src[s+4:], src[o2+4:]))
var l int32
{
a := src[s+4:]
b := src[o2+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
// Store this, since we have it. // Store this, since we have it.
nextHash := hashLen(cv, hashLog, tableFastHashLen) nextHash := hashLen(cv, hashLog, tableFastHashLen)
@ -731,19 +648,7 @@ encodeLoop:
// Consider history as well. // Consider history as well.
var seq seq var seq seq
var length int32 var length int32
// length = 4 + e.matchlen(s+6, repIndex+4, src) length = 4 + e.matchlen(s+6, repIndex+4, src)
{
a := src[s+6:]
b := src[repIndex+4:]
endI := len(a) & (math.MaxInt32 - 7)
length = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
length = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
seq.matchLen = uint32(length - zstdMinMatch) seq.matchLen = uint32(length - zstdMinMatch)
@ -831,20 +736,7 @@ encodeLoop:
} }
// Extend the 4-byte match as long as possible. // Extend the 4-byte match as long as possible.
//l := e.matchlen(s+4, t+4, src) + 4 l := e.matchlen(s+4, t+4, src) + 4
var l int32
{
a := src[s+4:]
b := src[t+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
// Extend backwards // Extend backwards
tMin := s - e.maxMatchOff tMin := s - e.maxMatchOff
@ -881,20 +773,7 @@ encodeLoop:
if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) { if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) {
// We have at least 4 byte match. // We have at least 4 byte match.
// No need to check backwards. We come straight from a match // No need to check backwards. We come straight from a match
//l := 4 + e.matchlen(s+4, o2+4, src) l := 4 + e.matchlen(s+4, o2+4, src)
var l int32
{
a := src[s+4:]
b := src[o2+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
// Store this, since we have it. // Store this, since we have it.
nextHash := hashLen(cv, hashLog, tableFastHashLen) nextHash := hashLen(cv, hashLog, tableFastHashLen)

View file

@ -379,7 +379,7 @@ func (s decSymbol) final() (int, uint8) {
// This can only be used if no symbols are 0 bits. // This can only be used if no symbols are 0 bits.
// At least tablelog bits must be available in the bit reader. // At least tablelog bits must be available in the bit reader.
func (s *fseState) nextFast(br *bitReader) (uint32, uint8) { func (s *fseState) nextFast(br *bitReader) (uint32, uint8) {
lowBits := uint16(br.getBitsFast(s.state.nbBits())) lowBits := br.get16BitsFast(s.state.nbBits())
s.state = s.dt[s.state.newState()+lowBits] s.state = s.dt[s.state.newState()+lowBits]
return s.state.baseline(), s.state.addBits() return s.state.baseline(), s.state.addBits()
} }

View file

@ -62,9 +62,8 @@ func (s symbolTransform) String() string {
// To indicate that you have populated the histogram call HistogramFinished // To indicate that you have populated the histogram call HistogramFinished
// with the value of the highest populated symbol, as well as the number of entries // with the value of the highest populated symbol, as well as the number of entries
// in the most populated entry. These are accepted at face value. // in the most populated entry. These are accepted at face value.
// The returned slice will always be length 256. func (s *fseEncoder) Histogram() *[256]uint32 {
func (s *fseEncoder) Histogram() []uint32 { return &s.count
return s.count[:]
} }
// HistogramFinished can be called to indicate that the histogram has been populated. // HistogramFinished can be called to indicate that the histogram has been populated.

View file

@ -0,0 +1,189 @@
// +build gc,!purego
#include "textflag.h"
// Register allocation.
#define digest R1
#define h R2 // Return value.
#define p R3 // Input pointer.
#define len R4
#define nblocks R5 // len / 32.
#define prime1 R7
#define prime2 R8
#define prime3 R9
#define prime4 R10
#define prime5 R11
#define v1 R12
#define v2 R13
#define v3 R14
#define v4 R15
#define x1 R20
#define x2 R21
#define x3 R22
#define x4 R23
#define round(acc, x) \
MADD prime2, acc, x, acc \
ROR $64-31, acc \
MUL prime1, acc \
// x = round(0, x).
#define round0(x) \
MUL prime2, x \
ROR $64-31, x \
MUL prime1, x \
#define mergeRound(x) \
round0(x) \
EOR x, h \
MADD h, prime4, prime1, h \
// Update v[1-4] with 32-byte blocks. Assumes len >= 32.
#define blocksLoop() \
LSR $5, len, nblocks \
PCALIGN $16 \
loop: \
LDP.P 32(p), (x1, x2) \
round(v1, x1) \
LDP -16(p), (x3, x4) \
round(v2, x2) \
SUB $1, nblocks \
round(v3, x3) \
round(v4, x4) \
CBNZ nblocks, loop \
// The primes are repeated here to ensure that they're stored
// in a contiguous array, so we can load them with LDP.
DATA primes<> +0(SB)/8, $11400714785074694791
DATA primes<> +8(SB)/8, $14029467366897019727
DATA primes<>+16(SB)/8, $1609587929392839161
DATA primes<>+24(SB)/8, $9650029242287828579
DATA primes<>+32(SB)/8, $2870177450012600261
GLOBL primes<>(SB), NOPTR+RODATA, $40
// func Sum64(b []byte) uint64
TEXT ·Sum64(SB), NOFRAME+NOSPLIT, $0-32
LDP b_base+0(FP), (p, len)
LDP primes<> +0(SB), (prime1, prime2)
LDP primes<>+16(SB), (prime3, prime4)
MOVD primes<>+32(SB), prime5
CMP $32, len
CSEL LO, prime5, ZR, h // if len < 32 { h = prime5 } else { h = 0 }
BLO afterLoop
ADD prime1, prime2, v1
MOVD prime2, v2
MOVD $0, v3
NEG prime1, v4
blocksLoop()
ROR $64-1, v1, x1
ROR $64-7, v2, x2
ADD x1, x2
ROR $64-12, v3, x3
ROR $64-18, v4, x4
ADD x3, x4
ADD x2, x4, h
mergeRound(v1)
mergeRound(v2)
mergeRound(v3)
mergeRound(v4)
afterLoop:
ADD len, h
TBZ $4, len, try8
LDP.P 16(p), (x1, x2)
round0(x1)
ROR $64-27, h
EOR x1 @> 64-27, h, h
MADD h, prime4, prime1, h
round0(x2)
ROR $64-27, h
EOR x2 @> 64-27, h
MADD h, prime4, prime1, h
try8:
TBZ $3, len, try4
MOVD.P 8(p), x1
round0(x1)
ROR $64-27, h
EOR x1 @> 64-27, h
MADD h, prime4, prime1, h
try4:
TBZ $2, len, try2
MOVWU.P 4(p), x2
MUL prime1, x2
ROR $64-23, h
EOR x2 @> 64-23, h
MADD h, prime3, prime2, h
try2:
TBZ $1, len, try1
MOVHU.P 2(p), x3
AND $255, x3, x1
LSR $8, x3, x2
MUL prime5, x1
ROR $64-11, h
EOR x1 @> 64-11, h
MUL prime1, h
MUL prime5, x2
ROR $64-11, h
EOR x2 @> 64-11, h
MUL prime1, h
try1:
TBZ $0, len, end
MOVBU (p), x4
MUL prime5, x4
ROR $64-11, h
EOR x4 @> 64-11, h
MUL prime1, h
end:
EOR h >> 33, h
MUL prime2, h
EOR h >> 29, h
MUL prime3, h
EOR h >> 32, h
MOVD h, ret+24(FP)
RET
// func writeBlocks(d *Digest, b []byte) int
//
// Assumes len(b) >= 32.
TEXT ·writeBlocks(SB), NOFRAME+NOSPLIT, $0-40
LDP primes<>(SB), (prime1, prime2)
// Load state. Assume v[1-4] are stored contiguously.
MOVD d+0(FP), digest
LDP 0(digest), (v1, v2)
LDP 16(digest), (v3, v4)
LDP b_base+8(FP), (p, len)
blocksLoop()
// Store updated state.
STP (v1, v2), 0(digest)
STP (v3, v4), 16(digest)
BIC $31, len
MOVD len, ret+32(FP)
RET

View file

@ -1,5 +1,8 @@
//go:build !appengine && gc && !purego //go:build (amd64 || arm64) && !appengine && gc && !purego
// +build !appengine,gc,!purego // +build amd64 arm64
// +build !appengine
// +build gc
// +build !purego
package xxhash package xxhash

View file

@ -1,5 +1,5 @@
//go:build !amd64 || appengine || !gc || purego //go:build (!amd64 && !arm64) || appengine || !gc || purego
// +build !amd64 appengine !gc purego // +build !amd64,!arm64 appengine !gc purego
package xxhash package xxhash

View file

@ -278,7 +278,7 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
mlState = mlTable[mlState.newState()&maxTableMask] mlState = mlTable[mlState.newState()&maxTableMask]
ofState = ofTable[ofState.newState()&maxTableMask] ofState = ofTable[ofState.newState()&maxTableMask]
} else { } else {
bits := br.getBitsFast(nBits) bits := br.get32BitsFast(nBits)
lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31)) lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31))
llState = llTable[(llState.newState()+lowBits)&maxTableMask] llState = llTable[(llState.newState()+lowBits)&maxTableMask]
@ -326,7 +326,7 @@ func (s *sequenceDecs) updateAlt(br *bitReader) {
s.offsets.state.state = s.offsets.state.dt[c.newState()] s.offsets.state.state = s.offsets.state.dt[c.newState()]
return return
} }
bits := br.getBitsFast(nBits) bits := br.get32BitsFast(nBits)
lowBits := uint16(bits >> ((c.nbBits() + b.nbBits()) & 31)) lowBits := uint16(bits >> ((c.nbBits() + b.nbBits()) & 31))
s.litLengths.state.state = s.litLengths.state.dt[a.newState()+lowBits] s.litLengths.state.state = s.litLengths.state.dt[a.newState()+lowBits]

View file

@ -3,8 +3,8 @@ GOARCH ?= $(shell go env GOARCH)
GOOS_GOARCH := $(GOOS)_$(GOARCH) GOOS_GOARCH := $(GOOS)_$(GOARCH)
GOOS_GOARCH_NATIVE := $(shell go env GOHOSTOS)_$(shell go env GOHOSTARCH) GOOS_GOARCH_NATIVE := $(shell go env GOHOSTOS)_$(shell go env GOHOSTARCH)
LIBZSTD_NAME := libzstd_$(GOOS_GOARCH).a LIBZSTD_NAME := libzstd_$(GOOS_GOARCH).a
ZSTD_VERSION ?= v1.5.0 ZSTD_VERSION ?= v1.5.1
MUSL_BUILDER_IMAGE=golang:1.17.1-alpine MUSL_BUILDER_IMAGE=golang:1.17.6-alpine
BUILDER_IMAGE := local/builder_musl:2.0.0-$(shell echo $(MUSL_BUILDER_IMAGE) | tr : _) BUILDER_IMAGE := local/builder_musl:2.0.0-$(shell echo $(MUSL_BUILDER_IMAGE) | tr : _)
.PHONY: libzstd.a .PHONY: libzstd.a

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -46,7 +46,7 @@ extern "C" {
* *
* Zstd can use dictionaries to improve compression ratio of small data. * Zstd can use dictionaries to improve compression ratio of small data.
* Traditionally small files don't compress well because there is very little * Traditionally small files don't compress well because there is very little
* repetion in a single sample, since it is small. But, if you are compressing * repetition in a single sample, since it is small. But, if you are compressing
* many similar files, like a bunch of JSON records that share the same * many similar files, like a bunch of JSON records that share the same
* structure, you can train a dictionary on ahead of time on some samples of * structure, you can train a dictionary on ahead of time on some samples of
* these files. Then, zstd can use the dictionary to find repetitions that are * these files. Then, zstd can use the dictionary to find repetitions that are
@ -132,7 +132,7 @@ extern "C" {
* *
* # Benchmark levels 1-3 without a dictionary * # Benchmark levels 1-3 without a dictionary
* zstd -b1e3 -r /path/to/my/files * zstd -b1e3 -r /path/to/my/files
* # Benchmark levels 1-3 with a dictioanry * # Benchmark levels 1-3 with a dictionary
* zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary
* *
* When should I retrain a dictionary? * When should I retrain a dictionary?
@ -237,7 +237,6 @@ typedef struct {
* is presumed that the most profitable content is at the end of the dictionary, * is presumed that the most profitable content is at the end of the dictionary,
* since that is the cheapest to reference. * since that is the cheapest to reference.
* *
* `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN). * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
* *
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`), * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
@ -272,8 +271,9 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
* Use them only in association with static linking. * Use them only in association with static linking.
* ==================================================================================== */ * ==================================================================================== */
#define ZDICT_CONTENTSIZE_MIN 128
#define ZDICT_DICTSIZE_MIN 256 #define ZDICT_DICTSIZE_MIN 256
/* Deprecated: Remove in v1.6.0 */
#define ZDICT_CONTENTSIZE_MIN 128
/*! ZDICT_cover_params_t: /*! ZDICT_cover_params_t:
* k and d are the only required parameters. * k and d are the only required parameters.

View file

@ -20,19 +20,21 @@ extern "C" {
/* ===== ZSTDLIB_API : control library symbols visibility ===== */ /* ===== ZSTDLIB_API : control library symbols visibility ===== */
#ifndef ZSTDLIB_VISIBILITY #ifndef ZSTDLIB_VISIBLE
# if defined(__GNUC__) && (__GNUC__ >= 4) # if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) # define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
# else # else
# define ZSTDLIB_VISIBILITY # define ZSTDLIB_VISIBLE
# define ZSTDLIB_HIDDEN
# endif # endif
#endif #endif
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ # define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else #else
# define ZSTDLIB_API ZSTDLIB_VISIBILITY # define ZSTDLIB_API ZSTDLIB_VISIBLE
#endif #endif
@ -72,7 +74,7 @@ extern "C" {
/*------ Version ------*/ /*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 5 #define ZSTD_VERSION_MINOR 5
#define ZSTD_VERSION_RELEASE 0 #define ZSTD_VERSION_RELEASE 1
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
/*! ZSTD_versionNumber() : /*! ZSTD_versionNumber() :
@ -247,7 +249,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
* *
* It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
* *
* This API supercedes all other "advanced" API entry points in the experimental section. * This API supersedes all other "advanced" API entry points in the experimental section.
* In the future, we expect to remove from experimental API entry points which are redundant with this API. * In the future, we expect to remove from experimental API entry points which are redundant with this API.
*/ */
@ -417,7 +419,7 @@ typedef enum {
* ZSTD_c_stableOutBuffer * ZSTD_c_stableOutBuffer
* ZSTD_c_blockDelimiters * ZSTD_c_blockDelimiters
* ZSTD_c_validateSequences * ZSTD_c_validateSequences
* ZSTD_c_splitBlocks * ZSTD_c_useBlockSplitter
* ZSTD_c_useRowMatchFinder * ZSTD_c_useRowMatchFinder
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly; * note : never ever use experimentalParam? names directly;
@ -932,7 +934,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
* Advanced dictionary and prefix API (Requires v1.4.0+) * Advanced dictionary and prefix API (Requires v1.4.0+)
* *
* This API allows dictionaries to be used with ZSTD_compress2(), * This API allows dictionaries to be used with ZSTD_compress2(),
* ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
* only reset with the context is reset with ZSTD_reset_parameters or * only reset with the context is reset with ZSTD_reset_parameters or
* ZSTD_reset_session_and_parameters. Prefixes are single-use. * ZSTD_reset_session_and_parameters. Prefixes are single-use.
******************************************************************************/ ******************************************************************************/
@ -1073,25 +1075,36 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
/* This can be overridden externally to hide static symbols. */
#ifndef ZSTDLIB_STATIC_API
# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE
# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE
# else
# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE
# endif
#endif
/* Deprecation warnings : /* Deprecation warnings :
* Should these warnings be a problem, it is generally possible to disable them, * Should these warnings be a problem, it is generally possible to disable them,
* typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
* Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
*/ */
#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS #ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
# define ZSTD_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */ # define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API /* disable deprecation warnings */
#else #else
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API # define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_STATIC_API
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) # elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
# define ZSTD_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message))) # define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3) # elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZSTD_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated)) # define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated))
# elif defined(_MSC_VER) # elif defined(_MSC_VER)
# define ZSTD_DEPRECATED(message) ZSTDLIB_API __declspec(deprecated(message)) # define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __declspec(deprecated(message))
# else # else
# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") # pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
# define ZSTD_DEPRECATED(message) ZSTDLIB_API # define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API
# endif # endif
#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ #endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
@ -1157,9 +1170,6 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
#define ZSTD_SRCSIZEHINT_MIN 0 #define ZSTD_SRCSIZEHINT_MIN 0
#define ZSTD_SRCSIZEHINT_MAX INT_MAX #define ZSTD_SRCSIZEHINT_MAX INT_MAX
/* internal */
#define ZSTD_HASHLOG3_MAX 17
/* --- Advanced types --- */ /* --- Advanced types --- */
@ -1302,10 +1312,14 @@ typedef enum {
} ZSTD_literalCompressionMode_e; } ZSTD_literalCompressionMode_e;
typedef enum { typedef enum {
ZSTD_urm_auto = 0, /* Automatically determine whether or not we use row matchfinder */ /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
ZSTD_urm_disableRowMatchFinder = 1, /* Never use row matchfinder */ * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
ZSTD_urm_enableRowMatchFinder = 2 /* Always use row matchfinder when applicable */ * or ZSTD_ps_disable allow for a force enable/disable the feature.
} ZSTD_useRowMatchFinderMode_e; */
ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */
ZSTD_ps_enable = 1, /* Force-enable the feature */
ZSTD_ps_disable = 2 /* Do not use the feature */
} ZSTD_paramSwitch_e;
/*************************************** /***************************************
* Frame size functions * Frame size functions
@ -1332,7 +1346,7 @@ typedef enum {
* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
* read each contained frame header. This is fast as most of the data is skipped, * read each contained frame header. This is fast as most of the data is skipped,
* however it does mean that all frame data must be present and valid. */ * however it does mean that all frame data must be present and valid. */
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_decompressBound() : /*! ZSTD_decompressBound() :
* `src` should point to the start of a series of ZSTD encoded and/or skippable frames * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
@ -1347,13 +1361,13 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
* note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
* upper-bound = # blocks * min(128 KB, Window_Size) * upper-bound = # blocks * min(128 KB, Window_Size)
*/ */
ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
/*! ZSTD_frameHeaderSize() : /*! ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
* @return : size of the Frame Header, * @return : size of the Frame Header,
* or an error code (if srcSize is too small) */ * or an error code (if srcSize is too small) */
ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
typedef enum { typedef enum {
ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
@ -1376,7 +1390,7 @@ typedef enum {
* @return : number of sequences generated * @return : number of sequences generated
*/ */
ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
size_t outSeqsSize, const void* src, size_t srcSize); size_t outSeqsSize, const void* src, size_t srcSize);
/*! ZSTD_mergeBlockDelimiters() : /*! ZSTD_mergeBlockDelimiters() :
@ -1390,7 +1404,7 @@ ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
* setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
* @return : number of sequences left after merging * @return : number of sequences left after merging
*/ */
ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
/*! ZSTD_compressSequences() : /*! ZSTD_compressSequences() :
* Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
@ -1420,7 +1434,7 @@ ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t se
* and cannot emit an RLE block that disagrees with the repcode history * and cannot emit an RLE block that disagrees with the repcode history
* @return : final compressed size or a ZSTD error. * @return : final compressed size or a ZSTD error.
*/ */
ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize, const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize); const void* src, size_t srcSize);
@ -1438,9 +1452,29 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size
* *
* @return : number of bytes written or a ZSTD error. * @return : number of bytes written or a ZSTD error.
*/ */
ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
const void* src, size_t srcSize, unsigned magicVariant); const void* src, size_t srcSize, unsigned magicVariant);
/*! ZSTD_readSkippableFrame() :
* Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
*
* The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
* in the magicVariant.
*
* Returns an error if destination buffer is not large enough, or if the frame is not skippable.
*
* @return : number of bytes written or a ZSTD error.
*/
ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
const void* src, size_t srcSize);
/*! ZSTD_isSkippableFrame() :
* Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
*/
ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
/*************************************** /***************************************
* Memory management * Memory management
@ -1469,10 +1503,10 @@ ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
* Note 2 : only single-threaded compression is supported. * Note 2 : only single-threaded compression is supported.
* ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
*/ */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
/*! ZSTD_estimateCStreamSize() : /*! ZSTD_estimateCStreamSize() :
* ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
@ -1487,20 +1521,20 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
* Note : if streaming is init with function ZSTD_init?Stream_usingDict(), * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
* an internal ?Dict will be created, which additional size is not estimated here. * an internal ?Dict will be created, which additional size is not estimated here.
* In this case, get total size by adding ZSTD_estimate?DictSize */ * In this case, get total size by adding ZSTD_estimate?DictSize */
ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
/*! ZSTD_estimate?DictSize() : /*! ZSTD_estimate?DictSize() :
* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
* ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
* Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
*/ */
ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
/*! ZSTD_initStatic*() : /*! ZSTD_initStatic*() :
* Initialize an object using a pre-allocated fixed-size buffer. * Initialize an object using a pre-allocated fixed-size buffer.
@ -1523,20 +1557,20 @@ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e
* Limitation 2 : static cctx currently not compatible with multi-threading. * Limitation 2 : static cctx currently not compatible with multi-threading.
* Limitation 3 : static dctx is incompatible with legacy support. * Limitation 3 : static dctx is incompatible with legacy support.
*/ */
ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict(
void* workspace, size_t workspaceSize, void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType, ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams); ZSTD_compressionParameters cParams);
ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict(
void* workspace, size_t workspaceSize, void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
@ -1557,12 +1591,12 @@ __attribute__((__unused__))
#endif #endif
ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType, ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams, ZSTD_compressionParameters cParams,
@ -1579,22 +1613,22 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
* ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
*/ */
typedef struct POOL_ctx_s ZSTD_threadPool; typedef struct POOL_ctx_s ZSTD_threadPool;
ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */
ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
/* /*
* This API is temporary and is expected to change or disappear in the future! * This API is temporary and is expected to change or disappear in the future!
*/ */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2(
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType, ZSTD_dictContentType_e dictContentType,
const ZSTD_CCtx_params* cctxParams, const ZSTD_CCtx_params* cctxParams,
ZSTD_customMem customMem); ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced( ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced(
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType, ZSTD_dictContentType_e dictContentType,
@ -1611,22 +1645,22 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(
* As a consequence, `dictBuffer` **must** outlive CDict, * As a consequence, `dictBuffer` **must** outlive CDict,
* and its content must remain unmodified throughout the lifetime of CDict. * and its content must remain unmodified throughout the lifetime of CDict.
* note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_getCParams() : /*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
* `estimatedSrcSize` value is optional, select 0 if not known */ * `estimatedSrcSize` value is optional, select 0 if not known */
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
/*! ZSTD_getParams() : /*! ZSTD_getParams() :
* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
* All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
/*! ZSTD_checkCParams() : /*! ZSTD_checkCParams() :
* Ensure param values remain within authorized range. * Ensure param values remain within authorized range.
* @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
/*! ZSTD_adjustCParams() : /*! ZSTD_adjustCParams() :
* optimize params for a given `srcSize` and `dictSize`. * optimize params for a given `srcSize` and `dictSize`.
@ -1634,7 +1668,7 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
* `dictSize` must be `0` when there is no dictionary. * `dictSize` must be `0` when there is no dictionary.
* cPar can be invalid : all parameters will be clamped within valid range in the @return struct. * cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
* This function never fails (wide contract) */ * This function never fails (wide contract) */
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
/*! ZSTD_compress_advanced() : /*! ZSTD_compress_advanced() :
* Note : this function is now DEPRECATED. * Note : this function is now DEPRECATED.
@ -1662,18 +1696,18 @@ size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
/*! ZSTD_CCtx_loadDictionary_byReference() : /*! ZSTD_CCtx_loadDictionary_byReference() :
* Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
* It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
/*! ZSTD_CCtx_loadDictionary_advanced() : /*! ZSTD_CCtx_loadDictionary_advanced() :
* Same as ZSTD_CCtx_loadDictionary(), but gives finer control over * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
* how to load the dictionary (by copy ? by reference ?) * how to load the dictionary (by copy ? by reference ?)
* and how to interpret it (automatic ? force raw mode ? full mode only ?) */ * and how to interpret it (automatic ? force raw mode ? full mode only ?) */
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
/*! ZSTD_CCtx_refPrefix_advanced() : /*! ZSTD_CCtx_refPrefix_advanced() :
* Same as ZSTD_CCtx_refPrefix(), but gives finer control over * Same as ZSTD_CCtx_refPrefix(), but gives finer control over
* how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
/* === experimental parameters === */ /* === experimental parameters === */
/* these parameters can be used with ZSTD_setParameter() /* these parameters can be used with ZSTD_setParameter()
@ -1712,9 +1746,15 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
* See the comments on that enum for an explanation of the feature. */ * See the comments on that enum for an explanation of the feature. */
#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 #define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
/* Controls how the literals are compressed (default is auto). /* Controlled with ZSTD_paramSwitch_e enum.
* The value must be of type ZSTD_literalCompressionMode_e. * Default is ZSTD_ps_auto.
* See ZSTD_literalCompressionMode_e enum definition for details. * Set to ZSTD_ps_disable to never compress literals.
* Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
* may still be emitted if huffman is not beneficial to use.)
*
* By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
* literals compression based on the compression parameters - specifically,
* negative compression levels do not use literal compression.
*/ */
#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
@ -1777,7 +1817,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
* *
* Note that this means that the CDict tables can no longer be copied into the * Note that this means that the CDict tables can no longer be copied into the
* CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
* useable. The dictionary can only be attached or reloaded. * usable. The dictionary can only be attached or reloaded.
* *
* In general, you should expect compression to be faster--sometimes very much * In general, you should expect compression to be faster--sometimes very much
* so--and CDict creation to be slightly slower. Eventually, we will probably * so--and CDict creation to be slightly slower. Eventually, we will probably
@ -1866,23 +1906,26 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
*/ */
#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 #define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
/* ZSTD_c_splitBlocks /* ZSTD_c_useBlockSplitter
* Default is 0 == disabled. Set to 1 to enable block splitting. * Controlled with ZSTD_paramSwitch_e enum.
* Default is ZSTD_ps_auto.
* Set to ZSTD_ps_disable to never use block splitter.
* Set to ZSTD_ps_enable to always use block splitter.
* *
* Will attempt to split blocks in order to improve compression ratio at the cost of speed. * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
* block splitting based on the compression parameters.
*/ */
#define ZSTD_c_splitBlocks ZSTD_c_experimentalParam13 #define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13
/* ZSTD_c_useRowMatchFinder /* ZSTD_c_useRowMatchFinder
* Default is ZSTD_urm_auto. * Controlled with ZSTD_paramSwitch_e enum.
* Controlled with ZSTD_useRowMatchFinderMode_e enum. * Default is ZSTD_ps_auto.
* Set to ZSTD_ps_disable to never use row-based matchfinder.
* Set to ZSTD_ps_enable to force usage of row-based matchfinder.
* *
* By default, in ZSTD_urm_auto, when finalizing the compression parameters, the library * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
* will decide at runtime whether to use the row-based matchfinder based on support for SIMD * the row-based matchfinder based on support for SIMD instructions and the window log.
* instructions as well as the windowLog. * Note that this only pertains to compression strategies: greedy, lazy, and lazy2
*
* Set to ZSTD_urm_disableRowMatchFinder to never use row-based matchfinder.
* Set to ZSTD_urm_enableRowMatchFinder to force usage of row-based matchfinder.
*/ */
#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 #define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
@ -1911,7 +1954,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
* and store it into int* value. * and store it into int* value.
* @return : 0, or an error code (which can be tested with ZSTD_isError()). * @return : 0, or an error code (which can be tested with ZSTD_isError()).
*/ */
ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
/*! ZSTD_CCtx_params : /*! ZSTD_CCtx_params :
@ -1931,25 +1974,25 @@ ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter
* This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
* for static allocation of CCtx for single-threaded compression. * for static allocation of CCtx for single-threaded compression.
*/ */
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */
/*! ZSTD_CCtxParams_reset() : /*! ZSTD_CCtxParams_reset() :
* Reset params to default values. * Reset params to default values.
*/ */
ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
/*! ZSTD_CCtxParams_init() : /*! ZSTD_CCtxParams_init() :
* Initializes the compression parameters of cctxParams according to * Initializes the compression parameters of cctxParams according to
* compression level. All other parameters are reset to their default values. * compression level. All other parameters are reset to their default values.
*/ */
ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
/*! ZSTD_CCtxParams_init_advanced() : /*! ZSTD_CCtxParams_init_advanced() :
* Initializes the compression and frame parameters of cctxParams according to * Initializes the compression and frame parameters of cctxParams according to
* params. All other parameters are reset to their default values. * params. All other parameters are reset to their default values.
*/ */
ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ /*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+
* Similar to ZSTD_CCtx_setParameter. * Similar to ZSTD_CCtx_setParameter.
@ -1959,14 +2002,14 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, Z
* @result : a code representing success or failure (which can be tested with * @result : a code representing success or failure (which can be tested with
* ZSTD_isError()). * ZSTD_isError()).
*/ */
ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
/*! ZSTD_CCtxParams_getParameter() : /*! ZSTD_CCtxParams_getParameter() :
* Similar to ZSTD_CCtx_getParameter. * Similar to ZSTD_CCtx_getParameter.
* Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
* @result : 0, or an error code (which can be tested with ZSTD_isError()). * @result : 0, or an error code (which can be tested with ZSTD_isError()).
*/ */
ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
/*! ZSTD_CCtx_setParametersUsingCCtxParams() : /*! ZSTD_CCtx_setParametersUsingCCtxParams() :
* Apply a set of ZSTD_CCtx_params to the compression context. * Apply a set of ZSTD_CCtx_params to the compression context.
@ -1975,7 +2018,7 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params,
* if nbWorkers>=1, new parameters will be picked up at next job, * if nbWorkers>=1, new parameters will be picked up at next job,
* with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
*/ */
ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
/*! ZSTD_compressStream2_simpleArgs() : /*! ZSTD_compressStream2_simpleArgs() :
@ -1984,7 +2027,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
* This variant might be helpful for binders from dynamic languages * This variant might be helpful for binders from dynamic languages
* which have troubles handling structures containing memory pointers. * which have troubles handling structures containing memory pointers.
*/ */
ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
ZSTD_CCtx* cctx, ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, size_t* dstPos, void* dst, size_t dstCapacity, size_t* dstPos,
const void* src, size_t srcSize, size_t* srcPos, const void* src, size_t srcSize, size_t* srcPos,
@ -2000,33 +2043,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs (
* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
* Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
* Note 3 : Skippable Frame Identifiers are considered valid. */ * Note 3 : Skippable Frame Identifiers are considered valid. */
ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
/*! ZSTD_createDDict_byReference() : /*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, ready to start decompression operation without startup delay. * Create a digested dictionary, ready to start decompression operation without startup delay.
* Dictionary content is referenced, and therefore stays in dictBuffer. * Dictionary content is referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives DDict, * It is important that dictBuffer outlives DDict,
* it must remain read accessible throughout the lifetime of DDict */ * it must remain read accessible throughout the lifetime of DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
/*! ZSTD_DCtx_loadDictionary_byReference() : /*! ZSTD_DCtx_loadDictionary_byReference() :
* Same as ZSTD_DCtx_loadDictionary(), * Same as ZSTD_DCtx_loadDictionary(),
* but references `dict` content instead of copying it into `dctx`. * but references `dict` content instead of copying it into `dctx`.
* This saves memory if `dict` remains around., * This saves memory if `dict` remains around.,
* However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
/*! ZSTD_DCtx_loadDictionary_advanced() : /*! ZSTD_DCtx_loadDictionary_advanced() :
* Same as ZSTD_DCtx_loadDictionary(), * Same as ZSTD_DCtx_loadDictionary(),
* but gives direct control over * but gives direct control over
* how to load the dictionary (by copy ? by reference ?) * how to load the dictionary (by copy ? by reference ?)
* and how to interpret it (automatic ? force raw mode ? full mode only ?). */ * and how to interpret it (automatic ? force raw mode ? full mode only ?). */
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
/*! ZSTD_DCtx_refPrefix_advanced() : /*! ZSTD_DCtx_refPrefix_advanced() :
* Same as ZSTD_DCtx_refPrefix(), but gives finer control over * Same as ZSTD_DCtx_refPrefix(), but gives finer control over
* how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
/*! ZSTD_DCtx_setMaxWindowSize() : /*! ZSTD_DCtx_setMaxWindowSize() :
* Refuses allocating internal buffers for frames requiring a window size larger than provided limit. * Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
@ -2035,14 +2078,14 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre
* By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
* @return : 0, or an error code (which can be tested using ZSTD_isError()). * @return : 0, or an error code (which can be tested using ZSTD_isError()).
*/ */
ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
/*! ZSTD_DCtx_getParameter() : /*! ZSTD_DCtx_getParameter() :
* Get the requested decompression parameter value, selected by enum ZSTD_dParameter, * Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
* and store it into int* value. * and store it into int* value.
* @return : 0, or an error code (which can be tested with ZSTD_isError()). * @return : 0, or an error code (which can be tested with ZSTD_isError()).
*/ */
ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
/* ZSTD_d_format /* ZSTD_d_format
* experimental parameter, * experimental parameter,
@ -2131,7 +2174,7 @@ size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
* This can be helpful for binders from dynamic languages * This can be helpful for binders from dynamic languages
* which have troubles handling structures containing memory pointers. * which have troubles handling structures containing memory pointers.
*/ */
ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
ZSTD_DCtx* dctx, ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, size_t* dstPos, void* dst, size_t dstCapacity, size_t* dstPos,
const void* src, size_t srcSize, size_t* srcPos); const void* src, size_t srcSize, size_t* srcPos);
@ -2270,7 +2313,7 @@ typedef struct {
* Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
* Aggregates progression inside active worker threads. * Aggregates progression inside active worker threads.
*/ */
ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
/*! ZSTD_toFlushNow() : /*! ZSTD_toFlushNow() :
* Tell how many bytes are ready to be flushed immediately. * Tell how many bytes are ready to be flushed immediately.
@ -2285,7 +2328,7 @@ ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx
* therefore flush speed is limited by production speed of oldest job * therefore flush speed is limited by production speed of oldest job
* irrespective of the speed of concurrent (and newer) jobs. * irrespective of the speed of concurrent (and newer) jobs.
*/ */
ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
/*===== Advanced Streaming decompression functions =====*/ /*===== Advanced Streaming decompression functions =====*/
@ -2299,7 +2342,7 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
* note: no dictionary will be used if dict == NULL or dictSize < 8 * note: no dictionary will be used if dict == NULL or dictSize < 8
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/ */
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
/*! /*!
* This function is deprecated, and is equivalent to: * This function is deprecated, and is equivalent to:
@ -2310,7 +2353,7 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic
* note : ddict is referenced, it must outlive decompression session * note : ddict is referenced, it must outlive decompression session
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/ */
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
/*! /*!
* This function is deprecated, and is equivalent to: * This function is deprecated, and is equivalent to:
@ -2320,7 +2363,7 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDi
* re-use decompression parameters from previous init; saves dictionary loading * re-use decompression parameters from previous init; saves dictionary loading
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
*/ */
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
/********************************************************************* /*********************************************************************
@ -2362,13 +2405,13 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
*/ */
/*===== Buffer-less streaming compression functions =====*/ /*===== Buffer-less streaming compression functions =====*/
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ /* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
ZSTD_DEPRECATED("use advanced API to access custom parameters") ZSTD_DEPRECATED("use advanced API to access custom parameters")
@ -2465,24 +2508,24 @@ typedef struct {
* @return : 0, `zfhPtr` is correctly filled, * @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount, * >0, `srcSize` is too small, value is wanted `srcSize` amount,
* or an error code, which can be tested using ZSTD_isError() */ * or an error code, which can be tested using ZSTD_isError() */
ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
/*! ZSTD_getFrameHeader_advanced() : /*! ZSTD_getFrameHeader_advanced() :
* same as ZSTD_getFrameHeader(), * same as ZSTD_getFrameHeader(),
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */ * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/* misc */ /* misc */
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
@ -2519,10 +2562,10 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
*/ */
/*===== Raw zstd block functions =====*/ /*===== Raw zstd block functions =====*/
ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */

View file

@ -741,6 +741,7 @@ const (
ETH_P_QINQ2 = 0x9200 ETH_P_QINQ2 = 0x9200
ETH_P_QINQ3 = 0x9300 ETH_P_QINQ3 = 0x9300
ETH_P_RARP = 0x8035 ETH_P_RARP = 0x8035
ETH_P_REALTEK = 0x8899
ETH_P_SCA = 0x6007 ETH_P_SCA = 0x6007
ETH_P_SLOW = 0x8809 ETH_P_SLOW = 0x8809
ETH_P_SNAP = 0x5 ETH_P_SNAP = 0x5
@ -810,10 +811,12 @@ const (
FAN_EPIDFD = -0x2 FAN_EPIDFD = -0x2
FAN_EVENT_INFO_TYPE_DFID = 0x3 FAN_EVENT_INFO_TYPE_DFID = 0x3
FAN_EVENT_INFO_TYPE_DFID_NAME = 0x2 FAN_EVENT_INFO_TYPE_DFID_NAME = 0x2
FAN_EVENT_INFO_TYPE_ERROR = 0x5
FAN_EVENT_INFO_TYPE_FID = 0x1 FAN_EVENT_INFO_TYPE_FID = 0x1
FAN_EVENT_INFO_TYPE_PIDFD = 0x4 FAN_EVENT_INFO_TYPE_PIDFD = 0x4
FAN_EVENT_METADATA_LEN = 0x18 FAN_EVENT_METADATA_LEN = 0x18
FAN_EVENT_ON_CHILD = 0x8000000 FAN_EVENT_ON_CHILD = 0x8000000
FAN_FS_ERROR = 0x8000
FAN_MARK_ADD = 0x1 FAN_MARK_ADD = 0x1
FAN_MARK_DONT_FOLLOW = 0x4 FAN_MARK_DONT_FOLLOW = 0x4
FAN_MARK_FILESYSTEM = 0x100 FAN_MARK_FILESYSTEM = 0x100
@ -1827,6 +1830,8 @@ const (
PERF_MEM_BLK_DATA = 0x2 PERF_MEM_BLK_DATA = 0x2
PERF_MEM_BLK_NA = 0x1 PERF_MEM_BLK_NA = 0x1
PERF_MEM_BLK_SHIFT = 0x28 PERF_MEM_BLK_SHIFT = 0x28
PERF_MEM_HOPS_0 = 0x1
PERF_MEM_HOPS_SHIFT = 0x2b
PERF_MEM_LOCK_LOCKED = 0x2 PERF_MEM_LOCK_LOCKED = 0x2
PERF_MEM_LOCK_NA = 0x1 PERF_MEM_LOCK_NA = 0x1
PERF_MEM_LOCK_SHIFT = 0x18 PERF_MEM_LOCK_SHIFT = 0x18
@ -1986,6 +1991,9 @@ const (
PR_SCHED_CORE_CREATE = 0x1 PR_SCHED_CORE_CREATE = 0x1
PR_SCHED_CORE_GET = 0x0 PR_SCHED_CORE_GET = 0x0
PR_SCHED_CORE_MAX = 0x4 PR_SCHED_CORE_MAX = 0x4
PR_SCHED_CORE_SCOPE_PROCESS_GROUP = 0x2
PR_SCHED_CORE_SCOPE_THREAD = 0x0
PR_SCHED_CORE_SCOPE_THREAD_GROUP = 0x1
PR_SCHED_CORE_SHARE_FROM = 0x3 PR_SCHED_CORE_SHARE_FROM = 0x3
PR_SCHED_CORE_SHARE_TO = 0x2 PR_SCHED_CORE_SHARE_TO = 0x2
PR_SET_CHILD_SUBREAPER = 0x24 PR_SET_CHILD_SUBREAPER = 0x24
@ -2167,12 +2175,23 @@ const (
RTCF_NAT = 0x800000 RTCF_NAT = 0x800000
RTCF_VALVE = 0x200000 RTCF_VALVE = 0x200000
RTC_AF = 0x20 RTC_AF = 0x20
RTC_BSM_DIRECT = 0x1
RTC_BSM_DISABLED = 0x0
RTC_BSM_LEVEL = 0x2
RTC_BSM_STANDBY = 0x3
RTC_FEATURE_ALARM = 0x0 RTC_FEATURE_ALARM = 0x0
RTC_FEATURE_ALARM_RES_2S = 0x3
RTC_FEATURE_ALARM_RES_MINUTE = 0x1 RTC_FEATURE_ALARM_RES_MINUTE = 0x1
RTC_FEATURE_CNT = 0x3 RTC_FEATURE_BACKUP_SWITCH_MODE = 0x6
RTC_FEATURE_CNT = 0x7
RTC_FEATURE_CORRECTION = 0x5
RTC_FEATURE_NEED_WEEK_DAY = 0x2 RTC_FEATURE_NEED_WEEK_DAY = 0x2
RTC_FEATURE_UPDATE_INTERRUPT = 0x4
RTC_IRQF = 0x80 RTC_IRQF = 0x80
RTC_MAX_FREQ = 0x2000 RTC_MAX_FREQ = 0x2000
RTC_PARAM_BACKUP_SWITCH_MODE = 0x2
RTC_PARAM_CORRECTION = 0x1
RTC_PARAM_FEATURES = 0x0
RTC_PF = 0x40 RTC_PF = 0x40
RTC_UF = 0x10 RTC_UF = 0x10
RTF_ADDRCLASSMASK = 0xf8000000 RTF_ADDRCLASSMASK = 0xf8000000
@ -2532,6 +2551,8 @@ const (
SO_VM_SOCKETS_BUFFER_MIN_SIZE = 0x1 SO_VM_SOCKETS_BUFFER_MIN_SIZE = 0x1
SO_VM_SOCKETS_BUFFER_SIZE = 0x0 SO_VM_SOCKETS_BUFFER_SIZE = 0x0
SO_VM_SOCKETS_CONNECT_TIMEOUT = 0x6 SO_VM_SOCKETS_CONNECT_TIMEOUT = 0x6
SO_VM_SOCKETS_CONNECT_TIMEOUT_NEW = 0x8
SO_VM_SOCKETS_CONNECT_TIMEOUT_OLD = 0x6
SO_VM_SOCKETS_NONBLOCK_TXRX = 0x7 SO_VM_SOCKETS_NONBLOCK_TXRX = 0x7
SO_VM_SOCKETS_PEER_HOST_VM_ID = 0x3 SO_VM_SOCKETS_PEER_HOST_VM_ID = 0x3
SO_VM_SOCKETS_TRUSTED = 0x5 SO_VM_SOCKETS_TRUSTED = 0x5

Some files were not shown because too many files have changed in this diff Show more