From 68bad22fd26d1436ad0236b1f3ced8604c5d851c Mon Sep 17 00:00:00 2001 From: Hui Wang Date: Tue, 29 Oct 2024 23:30:39 +0800 Subject: [PATCH] vmalert: integrate with victorialogs (#7255) address https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6706. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/vmalert-support-vlog-ds/docs/VictoriaLogs/vmalert.md. Related fix https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7254. Note: in this pull request, vmalert doesn't support [backfilling](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/vmalert-support-vlog-ds/docs/VictoriaLogs/vmalert.md#rules-backfilling) for rules with a customized time filter. It might be added in the future, see [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7289) for details. Feature can be tested with image `victoriametrics/vmalert:heads-vmalert-support-vlog-ds-0-g420629c-scratch`. --------- Signed-off-by: hagen1778 Co-authored-by: hagen1778 --- app/vmalert/config/config.go | 9 +- app/vmalert/config/config_test.go | 38 +- .../testdata/rules/vlog-rules0-bad.rules | 10 + .../testdata/rules/vlog-rules0-good.rules | 29 + app/vmalert/config/types.go | 19 +- app/vmalert/datasource/client.go | 333 +++++++++ ...{vm_graphite_api.go => client_graphite.go} | 2 +- .../{vm_prom_api.go => client_prom.go} | 26 +- .../datasource/{vm_test.go => client_test.go} | 305 ++++++-- app/vmalert/datasource/client_vlogs.go | 61 ++ app/vmalert/datasource/datasource.go | 14 +- app/vmalert/datasource/init.go | 3 +- app/vmalert/datasource/vm.go | 272 ------- app/vmalert/main.go | 2 +- app/vmalert/remoteread/init.go | 2 +- app/vmalert/rule/alerting.go | 11 +- app/vmalert/rule/group.go | 1 - app/vmalert/rule/recording.go | 19 +- app/vmalert/rule/recording_test.go | 22 + deployment/docker/README.md | 14 +- deployment/docker/auth-mixed-datasource.yml | 9 + deployment/docker/docker-compose-cluster.yml | 8 +- .../docker/docker-compose-victorialogs.yml | 41 +- deployment/docker/docker-compose.yml | 8 +- .../docker/{ => rules}/alerts-cluster.yml | 0 .../docker/{ => rules}/alerts-health.yml | 0 .../docker/{ => rules}/alerts-vmagent.yml | 0 .../docker/{ => rules}/alerts-vmalert.yml | 0 .../docker/{ => rules}/alerts-vmauth.yml | 0 deployment/docker/{ => rules}/alerts.yml | 0 .../docker/rules/vlogs-example-alerts.yml | 13 + docs/VictoriaLogs/vmalert.md | 207 ++++++ .../vmalert_victorialogs.excalidraw | 687 ++++++++++++++++++ docs/VictoriaLogs/vmalert_victorialogs.webp | Bin 0 -> 41810 bytes docs/changelog/CHANGELOG.md | 1 + docs/vmalert.md | 19 +- docs/vmauth.md | 2 +- lib/logstorage/parser.go | 32 + lib/logstorage/parser_test.go | 25 + 39 files changed, 1819 insertions(+), 425 deletions(-) create mode 100644 app/vmalert/config/testdata/rules/vlog-rules0-bad.rules create mode 100644 app/vmalert/config/testdata/rules/vlog-rules0-good.rules create mode 100644 app/vmalert/datasource/client.go rename app/vmalert/datasource/{vm_graphite_api.go => client_graphite.go} (95%) rename app/vmalert/datasource/{vm_prom_api.go => client_prom.go} (92%) rename app/vmalert/datasource/{vm_test.go => client_test.go} (64%) create mode 100644 app/vmalert/datasource/client_vlogs.go delete mode 100644 app/vmalert/datasource/vm.go create mode 100644 deployment/docker/auth-mixed-datasource.yml rename deployment/docker/{ => rules}/alerts-cluster.yml (100%) rename deployment/docker/{ => rules}/alerts-health.yml (100%) rename deployment/docker/{ => rules}/alerts-vmagent.yml (100%) rename deployment/docker/{ => rules}/alerts-vmalert.yml (100%) rename deployment/docker/{ => rules}/alerts-vmauth.yml (100%) rename deployment/docker/{ => rules}/alerts.yml (100%) create mode 100644 deployment/docker/rules/vlogs-example-alerts.yml create mode 100644 docs/VictoriaLogs/vmalert.md create mode 100644 docs/VictoriaLogs/vmalert_victorialogs.excalidraw create mode 100644 docs/VictoriaLogs/vmalert_victorialogs.webp diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index 31e16a115..353424b75 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -3,6 +3,7 @@ package config import ( "bytes" "crypto/md5" + "flag" "fmt" "hash/fnv" "io" @@ -17,6 +18,10 @@ import ( "gopkg.in/yaml.v2" ) +var ( + defaultRuleType = flag.String("rule.defaultRuleType", "prometheus", `Default type for rule expressions, can be overridden via "type" parameter on the group level, see https://docs.victoriametrics.com/vmalert/#groups. Supported values: "graphite", "prometheus" and "vlogs".`) +) + // Group contains list of Rules grouped into // entity with one name and evaluation interval type Group struct { @@ -59,11 +64,9 @@ func (g *Group) UnmarshalYAML(unmarshal func(any) error) error { if err != nil { return fmt.Errorf("failed to marshal group configuration for checksum: %w", err) } - // change default value to prometheus datasource. if g.Type.Get() == "" { - g.Type.Set(NewPrometheusType()) + g.Type = NewRawType(*defaultRuleType) } - h := md5.New() h.Write(b) g.Checksum = fmt.Sprintf("%x", h.Sum(nil)) diff --git a/app/vmalert/config/config_test.go b/app/vmalert/config/config_test.go index 1aa06d582..85207d53a 100644 --- a/app/vmalert/config/config_test.go +++ b/app/vmalert/config/config_test.go @@ -122,6 +122,7 @@ func TestParse_Failure(t *testing.T) { f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set") f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set") f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr") + f([]string{"testdata/rules/vlog-rules0-bad.rules"}, "bad LogsQL expr") f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header") f([]string{"testdata/rules/rules-multi-doc-bad.rules"}, "unknown fields") f([]string{"testdata/rules/rules-multi-doc-duplicates-bad.rules"}, "duplicate") @@ -240,7 +241,7 @@ func TestGroupValidate_Failure(t *testing.T) { }, false, "duplicate") f(&Group{ - Name: "test graphite prometheus bad expr", + Name: "test graphite with prometheus expr", Type: NewGraphiteType(), Rules: []Rule{ { @@ -267,6 +268,20 @@ func TestGroupValidate_Failure(t *testing.T) { }, }, false, "either `record` or `alert` must be set") + f(&Group{ + Name: "test vlogs with prometheus expr", + Type: NewVLogsType(), + Rules: []Rule{ + { + Expr: "sum(up == 0 ) by (host)", + For: promutils.NewDuration(10 * time.Millisecond), + }, + { + Expr: "sumSeries(time('foo.bar',10))", + }, + }, + }, false, "invalid rule") + // validate expressions f(&Group{ Name: "test", @@ -297,6 +312,16 @@ func TestGroupValidate_Failure(t *testing.T) { }}, }, }, true, "bad graphite expr") + + f(&Group{ + Name: "test vlogs", + Type: NewVLogsType(), + Rules: []Rule{ + {Alert: "alert", Expr: "stats count(*) as requests", Labels: map[string]string{ + "description": "some-description", + }}, + }, + }, true, "bad LogsQL expr") } func TestGroupValidate_Success(t *testing.T) { @@ -336,7 +361,7 @@ func TestGroupValidate_Success(t *testing.T) { }, }, false, false) - // validate annotiations + // validate annotations f(&Group{ Name: "test", Rules: []Rule{ @@ -363,6 +388,15 @@ func TestGroupValidate_Success(t *testing.T) { }}, }, }, false, true) + f(&Group{ + Name: "test victorialogs", + Type: NewVLogsType(), + Rules: []Rule{ + {Alert: "alert", Expr: " _time: 1m | stats count(*) as requests", Labels: map[string]string{ + "description": "{{ value|query }}", + }}, + }, + }, false, true) } func TestHashRule_NotEqual(t *testing.T) { diff --git a/app/vmalert/config/testdata/rules/vlog-rules0-bad.rules b/app/vmalert/config/testdata/rules/vlog-rules0-bad.rules new file mode 100644 index 000000000..91c248ae7 --- /dev/null +++ b/app/vmalert/config/testdata/rules/vlog-rules0-bad.rules @@ -0,0 +1,10 @@ +groups: + - name: InvalidStatsLogsql + type: vlogs + interval: 5m + rules: + - record: MissingFilter + expr: 'stats count(*) as requests' + - record: MissingStatsPipe + expr: 'service: "nginx"' + diff --git a/app/vmalert/config/testdata/rules/vlog-rules0-good.rules b/app/vmalert/config/testdata/rules/vlog-rules0-good.rules new file mode 100644 index 000000000..a41e44de1 --- /dev/null +++ b/app/vmalert/config/testdata/rules/vlog-rules0-good.rules @@ -0,0 +1,29 @@ +groups: + - name: RequestCount + type: vlogs + interval: 5m + rules: + - record: nginxRequestCount + expr: 'env: "test" AND service: "nginx" | stats count(*) as requests' + annotations: + description: "Service nginx on env test accepted {{$labels.requests}} requests in the last 5 minutes" + - record: prodRequestCount + expr: 'env: "prod" | stats by (service) count(*) as requests' + annotations: + description: "Service {{$labels.service}} on env prod accepted {{$labels.requests}} requests in the last 5 minutes" + - name: ServiceLog + type: vlogs + interval: 5m + rules: + - alert: HasErrorLog + expr: 'env: "prod" AND status:~"error|warn" | stats by (service) count(*) as errorLog | filter errorLog:>0' + annotations: + description: "Service {{$labels.service}} generated {{$labels.errorLog}} error logs in the last 5 minutes" + - name: ServiceRequest + type: vlogs + interval: 10m + rules: + - alert: TooManyFailedRequest + expr: '* | extract "ip= " | extract "status_code=;" | stats by (ip) count() if (code:!~200) as failed, count() as total| math failed / total as failed_percentage| filter failed_percentage :> 0.01 | fields ip,failed_percentage' + annotations: + description: "Connection from address {{$labels.ip}} has {{$value}} failed requests ratio in last 10 minutes" diff --git a/app/vmalert/config/types.go b/app/vmalert/config/types.go index cf16d0d8b..b71325ec3 100644 --- a/app/vmalert/config/types.go +++ b/app/vmalert/config/types.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphiteql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage" "github.com/VictoriaMetrics/metricsql" ) @@ -27,6 +28,13 @@ func NewGraphiteType() Type { } } +// NewVLogsType returns victorialogs datasource type +func NewVLogsType() Type { + return Type{ + Name: "vlogs", + } +} + // NewRawType returns datasource type from raw string // without validation. func NewRawType(d string) Type { @@ -62,6 +70,10 @@ func (t *Type) ValidateExpr(expr string) error { if _, err := metricsql.Parse(expr); err != nil { return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err) } + case "vlogs": + if _, err := logstorage.ParseStatsQuery(expr); err != nil { + return fmt.Errorf("bad LogsQL expr: %q, err: %w", expr, err) + } default: return fmt.Errorf("unknown datasource type=%q", t.Name) } @@ -74,13 +86,10 @@ func (t *Type) UnmarshalYAML(unmarshal func(any) error) error { if err := unmarshal(&s); err != nil { return err } - if s == "" { - s = "prometheus" - } switch s { - case "graphite", "prometheus": + case "graphite", "prometheus", "vlogs": default: - return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, "prometheus", "graphite") + return fmt.Errorf("unknown datasource type=%q, want prometheus, graphite or vlogs", s) } t.Name = s return nil diff --git a/app/vmalert/datasource/client.go b/app/vmalert/datasource/client.go new file mode 100644 index 000000000..4fa884824 --- /dev/null +++ b/app/vmalert/datasource/client.go @@ -0,0 +1,333 @@ +package datasource + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" +) + +type datasourceType string + +const ( + datasourcePrometheus datasourceType = "prometheus" + datasourceGraphite datasourceType = "graphite" + datasourceVLogs datasourceType = "vlogs" +) + +func toDatasourceType(s string) datasourceType { + switch s { + case string(datasourcePrometheus): + return datasourcePrometheus + case string(datasourceGraphite): + return datasourceGraphite + case string(datasourceVLogs): + return datasourceVLogs + default: + logger.Panicf("BUG: unknown datasource type %q", s) + } + return "" +} + +// Client is a datasource entity for reading data, +// supported clients are enumerated in datasourceType. +// WARN: when adding a new field, remember to check if Clone() method needs to be updated. +type Client struct { + c *http.Client + authCfg *promauth.Config + datasourceURL string + appendTypePrefix bool + queryStep time.Duration + dataSourceType datasourceType + // ApplyIntervalAsTimeFilter is only valid for vlogs datasource. + // Set to true if there is no [timeFilter](https://docs.victoriametrics.com/victorialogs/logsql/#time-filter) in the rule expression, + // and we will add evaluation interval as an additional timeFilter when querying. + applyIntervalAsTimeFilter bool + + // evaluationInterval will help setting request's `step` param, + // or adding time filter for LogsQL expression. + evaluationInterval time.Duration + // extraParams contains params to be attached to each HTTP request + extraParams url.Values + // extraHeaders are headers to be attached to each HTTP request + extraHeaders []keyValue + + // whether to print additional log messages + // for each sent request + debug bool +} + +type keyValue struct { + key string + value string +} + +// Clone clones shared http client and other configuration to the new client. +func (c *Client) Clone() *Client { + ns := &Client{ + c: c.c, + authCfg: c.authCfg, + datasourceURL: c.datasourceURL, + appendTypePrefix: c.appendTypePrefix, + queryStep: c.queryStep, + + dataSourceType: c.dataSourceType, + evaluationInterval: c.evaluationInterval, + + // init map so it can be populated below + extraParams: url.Values{}, + + debug: c.debug, + } + if len(c.extraHeaders) > 0 { + ns.extraHeaders = make([]keyValue, len(c.extraHeaders)) + copy(ns.extraHeaders, c.extraHeaders) + } + for k, v := range c.extraParams { + ns.extraParams[k] = v + } + + return ns +} + +// ApplyParams - changes given querier params. +func (c *Client) ApplyParams(params QuerierParams) *Client { + if params.DataSourceType != "" { + c.dataSourceType = toDatasourceType(params.DataSourceType) + } + c.evaluationInterval = params.EvaluationInterval + c.applyIntervalAsTimeFilter = params.ApplyIntervalAsTimeFilter + if params.QueryParams != nil { + if c.extraParams == nil { + c.extraParams = url.Values{} + } + for k, vl := range params.QueryParams { + // custom query params are prior to default ones + if c.extraParams.Has(k) { + c.extraParams.Del(k) + } + for _, v := range vl { + // don't use .Set() instead of Del/Add since it is allowed + // for GET params to be duplicated + // see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4908 + c.extraParams.Add(k, v) + } + } + } + if params.Headers != nil { + for key, value := range params.Headers { + kv := keyValue{key: key, value: value} + c.extraHeaders = append(c.extraHeaders, kv) + } + } + c.debug = params.Debug + return c +} + +// BuildWithParams - implements interface. +func (c *Client) BuildWithParams(params QuerierParams) Querier { + return c.Clone().ApplyParams(params) +} + +// NewPrometheusClient returns a new prometheus datasource client. +func NewPrometheusClient(baseURL string, authCfg *promauth.Config, appendTypePrefix bool, c *http.Client) *Client { + return &Client{ + c: c, + authCfg: authCfg, + datasourceURL: strings.TrimSuffix(baseURL, "/"), + appendTypePrefix: appendTypePrefix, + queryStep: *queryStep, + dataSourceType: datasourcePrometheus, + extraParams: url.Values{}, + } +} + +// Query executes the given query and returns parsed response +func (c *Client) Query(ctx context.Context, query string, ts time.Time) (Result, *http.Request, error) { + req, err := c.newQueryRequest(ctx, query, ts) + if err != nil { + return Result{}, nil, err + } + resp, err := c.do(req) + if err != nil { + if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) { + // Return unexpected error to the caller. + return Result{}, nil, err + } + // Something in the middle between client and datasource might be closing + // the connection. So we do a one more attempt in hope request will succeed. + req, err = c.newQueryRequest(ctx, query, ts) + if err != nil { + return Result{}, nil, fmt.Errorf("second attempt: %w", err) + } + resp, err = c.do(req) + if err != nil { + return Result{}, nil, fmt.Errorf("second attempt: %w", err) + } + } + + // Process the received response. + var parseFn func(req *http.Request, resp *http.Response) (Result, error) + switch c.dataSourceType { + case datasourcePrometheus: + parseFn = parsePrometheusResponse + case datasourceGraphite: + parseFn = parseGraphiteResponse + case datasourceVLogs: + parseFn = parseVLogsResponse + default: + logger.Panicf("BUG: unsupported datasource type %q to parse query response", c.dataSourceType) + } + result, err := parseFn(req, resp) + _ = resp.Body.Close() + return result, req, err +} + +// QueryRange executes the given query on the given time range. +// For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries +// Graphite type isn't supported. +func (c *Client) QueryRange(ctx context.Context, query string, start, end time.Time) (res Result, err error) { + if c.dataSourceType == datasourceGraphite { + return res, fmt.Errorf("%q is not supported for QueryRange", c.dataSourceType) + } + // TODO: disable range query LogsQL with time filter now + if c.dataSourceType == datasourceVLogs && !c.applyIntervalAsTimeFilter { + return res, fmt.Errorf("range query is not supported for LogsQL expression %q because it contains time filter. Remove time filter from the expression and try again", query) + } + if start.IsZero() { + return res, fmt.Errorf("start param is missing") + } + if end.IsZero() { + return res, fmt.Errorf("end param is missing") + } + req, err := c.newQueryRangeRequest(ctx, query, start, end) + if err != nil { + return res, err + } + resp, err := c.do(req) + if err != nil { + if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) { + // Return unexpected error to the caller. + return res, err + } + // Something in the middle between client and datasource might be closing + // the connection. So we do a one more attempt in hope request will succeed. + req, err = c.newQueryRangeRequest(ctx, query, start, end) + if err != nil { + return res, fmt.Errorf("second attempt: %w", err) + } + resp, err = c.do(req) + if err != nil { + return res, fmt.Errorf("second attempt: %w", err) + } + } + + // Process the received response. + var parseFn func(req *http.Request, resp *http.Response) (Result, error) + switch c.dataSourceType { + case datasourcePrometheus: + parseFn = parsePrometheusResponse + case datasourceVLogs: + parseFn = parseVLogsResponse + default: + logger.Panicf("BUG: unsupported datasource type %q to parse query range response", c.dataSourceType) + } + res, err = parseFn(req, resp) + _ = resp.Body.Close() + return res, err +} + +func (c *Client) do(req *http.Request) (*http.Response, error) { + ru := req.URL.Redacted() + if *showDatasourceURL { + ru = req.URL.String() + } + if c.debug { + logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, ru) + } + resp, err := c.c.Do(req) + if err != nil { + return nil, fmt.Errorf("error getting response from %s: %w", ru, err) + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body) + } + return resp, nil +} + +func (c *Client) newQueryRangeRequest(ctx context.Context, query string, start, end time.Time) (*http.Request, error) { + req, err := c.newRequest(ctx) + if err != nil { + return nil, fmt.Errorf("cannot create query_range request to datasource %q: %w", c.datasourceURL, err) + } + switch c.dataSourceType { + case datasourcePrometheus: + c.setPrometheusRangeReqParams(req, query, start, end) + case datasourceVLogs: + c.setVLogsRangeReqParams(req, query, start, end) + default: + logger.Panicf("BUG: unsupported datasource type %q to create range query request", c.dataSourceType) + } + return req, nil +} + +func (c *Client) newQueryRequest(ctx context.Context, query string, ts time.Time) (*http.Request, error) { + req, err := c.newRequest(ctx) + if err != nil { + return nil, fmt.Errorf("cannot create query request to datasource %q: %w", c.datasourceURL, err) + } + switch c.dataSourceType { + case datasourcePrometheus: + c.setPrometheusInstantReqParams(req, query, ts) + case datasourceGraphite: + c.setGraphiteReqParams(req, query) + case datasourceVLogs: + c.setVLogsInstantReqParams(req, query, ts) + default: + logger.Panicf("BUG: unsupported datasource type %q to create query request", c.dataSourceType) + } + return req, nil +} + +func (c *Client) newRequest(ctx context.Context) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.datasourceURL, nil) + if err != nil { + logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.datasourceURL, err) + } + req.Header.Set("Content-Type", "application/json") + if c.authCfg != nil { + err = c.authCfg.SetHeaders(req, true) + if err != nil { + return nil, err + } + } + for _, h := range c.extraHeaders { + req.Header.Set(h.key, h.value) + } + return req, nil +} + +// setReqParams adds query and other extra params for the request. +func (c *Client) setReqParams(r *http.Request, query string) { + q := r.URL.Query() + for k, vs := range c.extraParams { + if q.Has(k) { // extraParams are prior to params in URL + q.Del(k) + } + for _, v := range vs { + q.Add(k, v) + } + } + q.Set("query", query) + r.URL.RawQuery = q.Encode() +} diff --git a/app/vmalert/datasource/vm_graphite_api.go b/app/vmalert/datasource/client_graphite.go similarity index 95% rename from app/vmalert/datasource/vm_graphite_api.go rename to app/vmalert/datasource/client_graphite.go index 699919e77..2200f6ea5 100644 --- a/app/vmalert/datasource/vm_graphite_api.go +++ b/app/vmalert/datasource/client_graphite.go @@ -46,7 +46,7 @@ const ( graphitePrefix = "/graphite" ) -func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string) { +func (s *Client) setGraphiteReqParams(r *http.Request, query string) { if s.appendTypePrefix { r.URL.Path += graphitePrefix } diff --git a/app/vmalert/datasource/vm_prom_api.go b/app/vmalert/datasource/client_prom.go similarity index 92% rename from app/vmalert/datasource/vm_prom_api.go rename to app/vmalert/datasource/client_prom.go index f2b4bddee..4747e8c51 100644 --- a/app/vmalert/datasource/vm_prom_api.go +++ b/app/vmalert/datasource/client_prom.go @@ -14,7 +14,7 @@ import ( ) var ( - disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path "+ + disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' or '/select/logsql/stats_query' path "+ "to the configured -datasource.url and -remoteRead.url") disableStepParam = flag.Bool("datasource.disableStepParam", false, "Whether to disable adding 'step' param to the issued instant queries. "+ "This might be useful when using vmalert with datasources that do not support 'step' param for instant queries, like Google Managed Prometheus. "+ @@ -171,7 +171,7 @@ const ( func parsePrometheusResponse(req *http.Request, resp *http.Response) (res Result, err error) { r := &promResponse{} if err = json.NewDecoder(resp.Body).Decode(r); err != nil { - return res, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL.Redacted(), err) + return res, fmt.Errorf("error parsing response from %s: %w", req.URL.Redacted(), err) } if r.Status == statusError { return res, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL.Redacted(), r.ErrorType, r.Error) @@ -218,7 +218,7 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) (res Result return res, nil } -func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) { +func (s *Client) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) { if s.appendTypePrefix { r.URL.Path += "/prometheus" } @@ -238,10 +238,10 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string, q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds()))) } r.URL.RawQuery = q.Encode() - s.setPrometheusReqParams(r, query) + s.setReqParams(r, query) } -func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) { +func (s *Client) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) { if s.appendTypePrefix { r.URL.Path += "/prometheus" } @@ -257,19 +257,5 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds()))) } r.URL.RawQuery = q.Encode() - s.setPrometheusReqParams(r, query) -} - -func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) { - q := r.URL.Query() - for k, vs := range s.extraParams { - if q.Has(k) { // extraParams are prior to params in URL - q.Del(k) - } - for _, v := range vs { - q.Add(k, v) - } - } - q.Set("query", query) - r.URL.RawQuery = q.Encode() + s.setReqParams(r, query) } diff --git a/app/vmalert/datasource/vm_test.go b/app/vmalert/datasource/client_test.go similarity index 64% rename from app/vmalert/datasource/vm_test.go rename to app/vmalert/datasource/client_test.go index 9c3519a24..35da2279a 100644 --- a/app/vmalert/datasource/vm_test.go +++ b/app/vmalert/datasource/client_test.go @@ -24,8 +24,10 @@ var ( Username: basicAuthName, Password: promauth.NewSecret(basicAuthPass), } - query = "vm_rows" - queryRender = "constantLine(10)" + vmQuery = "vm_rows" + queryRender = "constantLine(10)" + vlogsQuery = "_time: 5m | stats by (foo) count() total" + vlogsRangeQuery = "* | stats by (foo) count() total" ) func TestVMInstantQuery(t *testing.T) { @@ -42,8 +44,8 @@ func TestVMInstantQuery(t *testing.T) { if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass { t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass) } - if r.URL.Query().Get("query") != query { - t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query")) + if r.URL.Query().Get("query") != vmQuery { + t.Fatalf("expected %s in query param, got %s", vmQuery, r.URL.Query().Get("query")) } timeParam := r.URL.Query().Get("time") if timeParam == "" { @@ -78,6 +80,31 @@ func TestVMInstantQuery(t *testing.T) { w.Write([]byte(`[{"target":"constantLine(10)","tags":{"name":"constantLine(10)"},"datapoints":[[10,1611758343],[10,1611758373],[10,1611758403]]}]`)) } }) + mux.HandleFunc("/select/logsql/stats_query", func(w http.ResponseWriter, r *http.Request) { + c++ + if r.Method != http.MethodPost { + t.Fatalf("expected POST method got %s", r.Method) + } + if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass { + t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass) + } + if r.URL.Query().Get("query") != vlogsQuery { + t.Fatalf("expected %s in query param, got %s", vlogsQuery, r.URL.Query().Get("query")) + } + timeParam := r.URL.Query().Get("time") + if timeParam == "" { + t.Fatalf("expected 'time' in query param, got nil instead") + } + if _, err := time.Parse(time.RFC3339, timeParam); err != nil { + t.Fatalf("failed to parse 'time' query param %q: %s", timeParam, err) + } + switch c { + case 9: + w.Write([]byte("[]")) + case 10: + w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"total","foo":"bar"},"value":[1583786142,"13763"]},{"metric":{"__name__":"total","foo":"baz"},"value":[1583786140,"2000"]}]}}`)) + } + }) srv := httptest.NewServer(mux) defer srv.Close() @@ -86,13 +113,13 @@ func TestVMInstantQuery(t *testing.T) { if err != nil { t.Fatalf("unexpected: %s", err) } - s := NewVMStorage(srv.URL, authCfg, 0, false, srv.Client()) + s := NewPrometheusClient(srv.URL, authCfg, false, srv.Client()) p := datasourcePrometheus pq := s.BuildWithParams(QuerierParams{DataSourceType: string(p), EvaluationInterval: 15 * time.Second}) ts := time.Now() - expErr := func(err string) { + expErr := func(query, err string) { _, _, gotErr := pq.Query(ctx, query, ts) if gotErr == nil { t.Fatalf("expected %q got nil", err) @@ -102,13 +129,13 @@ func TestVMInstantQuery(t *testing.T) { } } - expErr("500") // 0 - expErr("error parsing prometheus metrics") // 1 - expErr("response error") // 2 - expErr("unknown status") // 3 - expErr("unexpected end of JSON input") // 4 + expErr(vmQuery, "500") // 0 + expErr(vmQuery, "error parsing response") // 1 + expErr(vmQuery, "response error") // 2 + expErr(vmQuery, "unknown status") // 3 + expErr(vmQuery, "unexpected end of JSON input") // 4 - res, _, err := pq.Query(ctx, query, ts) // 5 - vector + res, _, err := pq.Query(ctx, vmQuery, ts) // 5 - vector if err != nil { t.Fatalf("unexpected %s", err) } @@ -129,7 +156,7 @@ func TestVMInstantQuery(t *testing.T) { } metricsEqual(t, res.Data, expected) - res, req, err := pq.Query(ctx, query, ts) // 6 - scalar + res, req, err := pq.Query(ctx, vmQuery, ts) // 6 - scalar if err != nil { t.Fatalf("unexpected %s", err) } @@ -154,7 +181,7 @@ func TestVMInstantQuery(t *testing.T) { res.SeriesFetched) } - res, _, err = pq.Query(ctx, query, ts) // 7 - scalar with stats + res, _, err = pq.Query(ctx, vmQuery, ts) // 7 - scalar with stats if err != nil { t.Fatalf("unexpected %s", err) } @@ -175,6 +202,7 @@ func TestVMInstantQuery(t *testing.T) { *res.SeriesFetched) } + // test graphite gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)}) res, _, err = gq.Query(ctx, queryRender, ts) // 8 - graphite @@ -192,6 +220,33 @@ func TestVMInstantQuery(t *testing.T) { }, } metricsEqual(t, res.Data, exp) + + // test victorialogs + vlogs := datasourceVLogs + pq = s.BuildWithParams(QuerierParams{DataSourceType: string(vlogs), EvaluationInterval: 15 * time.Second}) + + expErr(vlogsQuery, "error parsing response") // 9 + + res, _, err = pq.Query(ctx, vlogsQuery, ts) // 10 + if err != nil { + t.Fatalf("unexpected %s", err) + } + if len(res.Data) != 2 { + t.Fatalf("expected 2 metrics got %d in %+v", len(res.Data), res.Data) + } + expected = []Metric{ + { + Labels: []Label{{Value: "total", Name: "stats_result"}, {Value: "bar", Name: "foo"}}, + Timestamps: []int64{1583786142}, + Values: []float64{13763}, + }, + { + Labels: []Label{{Value: "total", Name: "stats_result"}, {Value: "baz", Name: "foo"}}, + Timestamps: []int64{1583786140}, + Values: []float64{2000}, + }, + } + metricsEqual(t, res.Data, expected) } func TestVMInstantQueryWithRetry(t *testing.T) { @@ -202,8 +257,8 @@ func TestVMInstantQueryWithRetry(t *testing.T) { c := -1 mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) { c++ - if r.URL.Query().Get("query") != query { - t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query")) + if r.URL.Query().Get("query") != vmQuery { + t.Fatalf("expected %s in query param, got %s", vmQuery, r.URL.Query().Get("query")) } switch c { case 0: @@ -225,11 +280,11 @@ func TestVMInstantQueryWithRetry(t *testing.T) { srv := httptest.NewServer(mux) defer srv.Close() - s := NewVMStorage(srv.URL, nil, 0, false, srv.Client()) + s := NewPrometheusClient(srv.URL, nil, false, srv.Client()) pq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourcePrometheus)}) expErr := func(err string) { - _, _, gotErr := pq.Query(ctx, query, time.Now()) + _, _, gotErr := pq.Query(ctx, vmQuery, time.Now()) if gotErr == nil { t.Fatalf("expected %q got nil", err) } @@ -239,7 +294,7 @@ func TestVMInstantQueryWithRetry(t *testing.T) { } expValue := func(v float64) { - res, _, err := pq.Query(ctx, query, time.Now()) + res, _, err := pq.Query(ctx, vmQuery, time.Now()) if err != nil { t.Fatalf("unexpected %s", err) } @@ -300,8 +355,8 @@ func TestVMRangeQuery(t *testing.T) { if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass { t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass) } - if r.URL.Query().Get("query") != query { - t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query")) + if r.URL.Query().Get("query") != vmQuery { + t.Fatalf("expected %s in query param, got %s", vmQuery, r.URL.Query().Get("query")) } startTS := r.URL.Query().Get("start") if startTS == "" { @@ -326,6 +381,40 @@ func TestVMRangeQuery(t *testing.T) { w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"vm_rows"},"values":[[1583786142,"13763"]]}]}}`)) } }) + mux.HandleFunc("/select/logsql/stats_query_range", func(w http.ResponseWriter, r *http.Request) { + c++ + if r.Method != http.MethodPost { + t.Fatalf("expected POST method got %s", r.Method) + } + if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass { + t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass) + } + if r.URL.Query().Get("query") != vlogsRangeQuery { + t.Fatalf("expected %s in query param, got %s", vmQuery, r.URL.Query().Get("query")) + } + startTS := r.URL.Query().Get("start") + if startTS == "" { + t.Fatalf("expected 'start' in query param, got nil instead") + } + if _, err := time.Parse(time.RFC3339, startTS); err != nil { + t.Fatalf("failed to parse 'start' query param: %s", err) + } + endTS := r.URL.Query().Get("end") + if endTS == "" { + t.Fatalf("expected 'end' in query param, got nil instead") + } + if _, err := time.Parse(time.RFC3339, endTS); err != nil { + t.Fatalf("failed to parse 'end' query param: %s", err) + } + step := r.URL.Query().Get("step") + if step != "60s" { + t.Fatalf("expected 'step' query param to be 60s; got %q instead", step) + } + switch c { + case 1: + w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"total"},"values":[[1583786142,"10"]]}]}}`)) + } + }) srv := httptest.NewServer(mux) defer srv.Close() @@ -334,19 +423,19 @@ func TestVMRangeQuery(t *testing.T) { if err != nil { t.Fatalf("unexpected: %s", err) } - s := NewVMStorage(srv.URL, authCfg, *queryStep, false, srv.Client()) + s := NewPrometheusClient(srv.URL, authCfg, false, srv.Client()) pq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourcePrometheus), EvaluationInterval: 15 * time.Second}) - _, err = pq.QueryRange(ctx, query, time.Now(), time.Time{}) + _, err = pq.QueryRange(ctx, vmQuery, time.Now(), time.Time{}) expectError(t, err, "is missing") - _, err = pq.QueryRange(ctx, query, time.Time{}, time.Now()) + _, err = pq.QueryRange(ctx, vmQuery, time.Time{}, time.Now()) expectError(t, err, "is missing") start, end := time.Now().Add(-time.Minute), time.Now() - res, err := pq.QueryRange(ctx, query, start, end) + res, err := pq.QueryRange(ctx, vmQuery, start, end) if err != nil { t.Fatalf("unexpected %s", err) } @@ -363,33 +452,66 @@ func TestVMRangeQuery(t *testing.T) { t.Fatalf("unexpected metric %+v want %+v", m[0], expected) } + // test unsupported graphite gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)}) _, err = gq.QueryRange(ctx, queryRender, start, end) expectError(t, err, "is not supported") + + // unsupported logsql + gq = s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceVLogs), EvaluationInterval: 60 * time.Second}) + + res, err = gq.QueryRange(ctx, vlogsRangeQuery, start, end) + expectError(t, err, "is not supported") + + // supported logsql + gq = s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceVLogs), EvaluationInterval: 60 * time.Second, ApplyIntervalAsTimeFilter: true}) + res, err = gq.QueryRange(ctx, vlogsRangeQuery, start, end) + if err != nil { + t.Fatalf("unexpected %s", err) + } + m = res.Data + if len(m) != 1 { + t.Fatalf("expected 1 metric got %d in %+v", len(m), m) + } + expected = Metric{ + Labels: []Label{{Value: "total", Name: "stats_result"}}, + Timestamps: []int64{1583786142}, + Values: []float64{10}, + } + if !reflect.DeepEqual(m[0], expected) { + t.Fatalf("unexpected metric %+v want %+v", m[0], expected) + } } func TestRequestParams(t *testing.T) { query := "up" + vlogsQuery := "_time: 5m | stats count() total" timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC) - f := func(isQueryRange bool, vm *VMStorage, checkFn func(t *testing.T, r *http.Request)) { + f := func(isQueryRange bool, c *Client, checkFn func(t *testing.T, r *http.Request)) { t.Helper() - req, err := vm.newRequest(ctx) + req, err := c.newRequest(ctx) if err != nil { t.Fatalf("error in newRequest: %s", err) } - switch vm.dataSourceType { - case "", datasourcePrometheus: + switch c.dataSourceType { + case datasourcePrometheus: if isQueryRange { - vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp) + c.setPrometheusRangeReqParams(req, query, timestamp, timestamp) } else { - vm.setPrometheusInstantReqParams(req, query, timestamp) + c.setPrometheusInstantReqParams(req, query, timestamp) } case datasourceGraphite: - vm.setGraphiteReqParams(req, query) + c.setGraphiteReqParams(req, query) + case datasourceVLogs: + if isQueryRange { + c.setVLogsRangeReqParams(req, vlogsQuery, timestamp, timestamp) + } else { + c.setVLogsInstantReqParams(req, vlogsQuery, timestamp) + } } checkFn(t, req) @@ -399,19 +521,19 @@ func TestRequestParams(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %s", err) } - storage := VMStorage{ + storage := Client{ extraParams: url.Values{"round_digits": {"10"}}, } // prometheus path - f(false, &VMStorage{ + f(false, &Client{ dataSourceType: datasourcePrometheus, }, func(t *testing.T, r *http.Request) { checkEqualString(t, "/api/v1/query", r.URL.Path) }) // prometheus prefix - f(false, &VMStorage{ + f(false, &Client{ dataSourceType: datasourcePrometheus, appendTypePrefix: true, }, func(t *testing.T, r *http.Request) { @@ -419,14 +541,14 @@ func TestRequestParams(t *testing.T) { }) // prometheus range path - f(true, &VMStorage{ + f(true, &Client{ dataSourceType: datasourcePrometheus, }, func(t *testing.T, r *http.Request) { checkEqualString(t, "/api/v1/query_range", r.URL.Path) }) // prometheus range prefix - f(true, &VMStorage{ + f(true, &Client{ dataSourceType: datasourcePrometheus, appendTypePrefix: true, }, func(t *testing.T, r *http.Request) { @@ -434,14 +556,14 @@ func TestRequestParams(t *testing.T) { }) // graphite path - f(false, &VMStorage{ + f(false, &Client{ dataSourceType: datasourceGraphite, }, func(t *testing.T, r *http.Request) { checkEqualString(t, graphitePath, r.URL.Path) }) // graphite prefix - f(false, &VMStorage{ + f(false, &Client{ dataSourceType: datasourceGraphite, appendTypePrefix: true, }, func(t *testing.T, r *http.Request) { @@ -449,21 +571,27 @@ func TestRequestParams(t *testing.T) { }) // default params - f(false, &VMStorage{}, func(t *testing.T, r *http.Request) { + f(false, &Client{dataSourceType: datasourcePrometheus}, func(t *testing.T, r *http.Request) { + exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}} + checkEqualString(t, exp.Encode(), r.URL.RawQuery) + }) + + f(false, &Client{dataSourceType: datasourcePrometheus, applyIntervalAsTimeFilter: true}, func(t *testing.T, r *http.Request) { exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}} checkEqualString(t, exp.Encode(), r.URL.RawQuery) }) // default range params - f(true, &VMStorage{}, func(t *testing.T, r *http.Request) { + f(true, &Client{dataSourceType: datasourcePrometheus}, func(t *testing.T, r *http.Request) { ts := timestamp.Format(time.RFC3339) exp := url.Values{"query": {query}, "start": {ts}, "end": {ts}} checkEqualString(t, exp.Encode(), r.URL.RawQuery) }) // basic auth - f(false, &VMStorage{ - authCfg: authCfg, + f(false, &Client{ + dataSourceType: datasourcePrometheus, + authCfg: authCfg, }, func(t *testing.T, r *http.Request) { u, p, _ := r.BasicAuth() checkEqualString(t, "foo", u) @@ -471,8 +599,9 @@ func TestRequestParams(t *testing.T) { }) // basic auth range - f(true, &VMStorage{ - authCfg: authCfg, + f(true, &Client{ + dataSourceType: datasourcePrometheus, + authCfg: authCfg, }, func(t *testing.T, r *http.Request) { u, p, _ := r.BasicAuth() checkEqualString(t, "foo", u) @@ -480,7 +609,8 @@ func TestRequestParams(t *testing.T) { }) // evaluation interval - f(false, &VMStorage{ + f(false, &Client{ + dataSourceType: datasourcePrometheus, evaluationInterval: 15 * time.Second, }, func(t *testing.T, r *http.Request) { evalInterval := 15 * time.Second @@ -489,8 +619,9 @@ func TestRequestParams(t *testing.T) { }) // step override - f(false, &VMStorage{ - queryStep: time.Minute, + f(false, &Client{ + dataSourceType: datasourcePrometheus, + queryStep: time.Minute, }, func(t *testing.T, r *http.Request) { exp := url.Values{ "query": {query}, @@ -501,7 +632,8 @@ func TestRequestParams(t *testing.T) { }) // step to seconds - f(false, &VMStorage{ + f(false, &Client{ + dataSourceType: datasourcePrometheus, evaluationInterval: 3 * time.Hour, }, func(t *testing.T, r *http.Request) { evalInterval := 3 * time.Hour @@ -510,15 +642,17 @@ func TestRequestParams(t *testing.T) { }) // prometheus extra params - f(false, &VMStorage{ - extraParams: url.Values{"round_digits": {"10"}}, + f(false, &Client{ + dataSourceType: datasourcePrometheus, + extraParams: url.Values{"round_digits": {"10"}}, }, func(t *testing.T, r *http.Request) { exp := url.Values{"query": {query}, "round_digits": {"10"}, "time": {timestamp.Format(time.RFC3339)}} checkEqualString(t, exp.Encode(), r.URL.RawQuery) }) // prometheus extra params range - f(true, &VMStorage{ + f(true, &Client{ + dataSourceType: datasourcePrometheus, extraParams: url.Values{ "nocache": {"1"}, "max_lookback": {"1h"}, @@ -536,7 +670,8 @@ func TestRequestParams(t *testing.T) { // custom params overrides the original params f(false, storage.Clone().ApplyParams(QuerierParams{ - QueryParams: url.Values{"round_digits": {"2"}}, + DataSourceType: string(datasourcePrometheus), + QueryParams: url.Values{"round_digits": {"2"}}, }), func(t *testing.T, r *http.Request) { exp := url.Values{"query": {query}, "round_digits": {"2"}, "time": {timestamp.Format(time.RFC3339)}} checkEqualString(t, exp.Encode(), r.URL.RawQuery) @@ -544,14 +679,15 @@ func TestRequestParams(t *testing.T) { // allow duplicates in query params f(false, storage.Clone().ApplyParams(QuerierParams{ - QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}}, + DataSourceType: string(datasourcePrometheus), + QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}}, }), func(t *testing.T, r *http.Request) { exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}} checkEqualString(t, exp.Encode(), r.URL.RawQuery) }) // graphite extra params - f(false, &VMStorage{ + f(false, &Client{ dataSourceType: datasourceGraphite, extraParams: url.Values{ "nocache": {"1"}, @@ -563,7 +699,7 @@ func TestRequestParams(t *testing.T) { }) // graphite extra params allows to override from - f(false, &VMStorage{ + f(false, &Client{ dataSourceType: datasourceGraphite, extraParams: url.Values{ "from": {"-10m"}, @@ -572,10 +708,38 @@ func TestRequestParams(t *testing.T) { exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query) checkEqualString(t, exp, r.URL.RawQuery) }) + + // test vlogs + f(false, &Client{ + dataSourceType: datasourceVLogs, + evaluationInterval: time.Minute, + }, func(t *testing.T, r *http.Request) { + exp := url.Values{"query": {vlogsQuery}, "time": {timestamp.Format(time.RFC3339)}} + checkEqualString(t, exp.Encode(), r.URL.RawQuery) + }) + + f(false, &Client{ + dataSourceType: datasourceVLogs, + evaluationInterval: time.Minute, + applyIntervalAsTimeFilter: true, + }, func(t *testing.T, r *http.Request) { + ts := timestamp.Format(time.RFC3339) + exp := url.Values{"query": {vlogsQuery}, "time": {ts}, "start": {timestamp.Add(-time.Minute).Format(time.RFC3339)}, "end": {ts}} + checkEqualString(t, exp.Encode(), r.URL.RawQuery) + }) + + f(true, &Client{ + dataSourceType: datasourceVLogs, + evaluationInterval: time.Minute, + }, func(t *testing.T, r *http.Request) { + ts := timestamp.Format(time.RFC3339) + exp := url.Values{"query": {vlogsQuery}, "start": {ts}, "end": {ts}, "step": {"60s"}} + checkEqualString(t, exp.Encode(), r.URL.RawQuery) + }) } func TestHeaders(t *testing.T) { - f := func(vmFn func() *VMStorage, checkFn func(t *testing.T, r *http.Request)) { + f := func(vmFn func() *Client, checkFn func(t *testing.T, r *http.Request)) { t.Helper() vm := vmFn() @@ -587,12 +751,12 @@ func TestHeaders(t *testing.T) { } // basic auth - f(func() *VMStorage { + f(func() *Client { cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", "")) if err != nil { t.Fatalf("Error get auth config: %s", err) } - return &VMStorage{authCfg: cfg} + return NewPrometheusClient("", cfg, false, nil) }, func(t *testing.T, r *http.Request) { u, p, _ := r.BasicAuth() checkEqualString(t, "foo", u) @@ -600,12 +764,12 @@ func TestHeaders(t *testing.T) { }) // bearer auth - f(func() *VMStorage { + f(func() *Client { cfg, err := utils.AuthConfig(utils.WithBearer("foo", "")) if err != nil { t.Fatalf("Error get auth config: %s", err) } - return &VMStorage{authCfg: cfg} + return NewPrometheusClient("", cfg, false, nil) }, func(t *testing.T, r *http.Request) { reqToken := r.Header.Get("Authorization") splitToken := strings.Split(reqToken, "Bearer ") @@ -617,11 +781,13 @@ func TestHeaders(t *testing.T) { }) // custom extraHeaders - f(func() *VMStorage { - return &VMStorage{extraHeaders: []keyValue{ + f(func() *Client { + c := NewPrometheusClient("", nil, false, nil) + c.extraHeaders = []keyValue{ {key: "Foo", value: "bar"}, {key: "Baz", value: "qux"}, - }} + } + return c }, func(t *testing.T, r *http.Request) { h1 := r.Header.Get("Foo") checkEqualString(t, "bar", h1) @@ -630,17 +796,16 @@ func TestHeaders(t *testing.T) { }) // custom header overrides basic auth - f(func() *VMStorage { + f(func() *Client { cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", "")) if err != nil { t.Fatalf("Error get auth config: %s", err) } - return &VMStorage{ - authCfg: cfg, - extraHeaders: []keyValue{ - {key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="}, - }, + c := NewPrometheusClient("", cfg, false, nil) + c.extraHeaders = []keyValue{ + {key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="}, } + return c }, func(t *testing.T, r *http.Request) { u, p, _ := r.BasicAuth() checkEqualString(t, "Aladdin", u) diff --git a/app/vmalert/datasource/client_vlogs.go b/app/vmalert/datasource/client_vlogs.go new file mode 100644 index 000000000..cb6e8892c --- /dev/null +++ b/app/vmalert/datasource/client_vlogs.go @@ -0,0 +1,61 @@ +package datasource + +import ( + "fmt" + "net/http" + "time" +) + +func (s *Client) setVLogsInstantReqParams(r *http.Request, query string, timestamp time.Time) { + // there is no type path prefix in victorialogs APIs right now, ignore appendTypePrefix. + if !*disablePathAppend { + r.URL.Path += "/select/logsql/stats_query" + } + q := r.URL.Query() + // set `time` param explicitly, it will be used as the timestamp of query results. + q.Set("time", timestamp.Format(time.RFC3339)) + // set the `start` and `end` params if applyIntervalAsTimeFilter is enabled(time filter is missing in the rule expr), + // so the query will be executed in time range [timestamp - evaluationInterval, timestamp]. + if s.applyIntervalAsTimeFilter && s.evaluationInterval > 0 { + q.Set("start", timestamp.Add(-s.evaluationInterval).Format(time.RFC3339)) + q.Set("end", timestamp.Format(time.RFC3339)) + } + r.URL.RawQuery = q.Encode() + s.setReqParams(r, query) +} + +func (s *Client) setVLogsRangeReqParams(r *http.Request, query string, start, end time.Time) { + // there is no type path prefix in victorialogs APIs right now, ignore appendTypePrefix. + if !*disablePathAppend { + r.URL.Path += "/select/logsql/stats_query_range" + } + q := r.URL.Query() + q.Add("start", start.Format(time.RFC3339)) + q.Add("end", end.Format(time.RFC3339)) + // set step as evaluationInterval by default + if s.evaluationInterval > 0 { + q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds()))) + } + r.URL.RawQuery = q.Encode() + s.setReqParams(r, query) +} + +func parseVLogsResponse(req *http.Request, resp *http.Response) (res Result, err error) { + res, err = parsePrometheusResponse(req, resp) + if err != nil { + return Result{}, err + } + for i := range res.Data { + m := &res.Data[i] + for j := range m.Labels { + // reserve the stats func result name with a new label `stats_result` instead of dropping it, + // since there could be multiple stats results in a single query, for instance: + // _time:5m | stats quantile(0.5, request_duration_seconds) p50, quantile(0.9, request_duration_seconds) p90 + if m.Labels[j].Name == "__name__" { + m.Labels[j].Name = "stats_result" + break + } + } + } + return +} diff --git a/app/vmalert/datasource/datasource.go b/app/vmalert/datasource/datasource.go index 31e4689c4..97a0f8d49 100644 --- a/app/vmalert/datasource/datasource.go +++ b/app/vmalert/datasource/datasource.go @@ -42,11 +42,15 @@ type QuerierBuilder interface { // QuerierParams params for Querier. type QuerierParams struct { - DataSourceType string - EvaluationInterval time.Duration - QueryParams url.Values - Headers map[string]string - Debug bool + DataSourceType string + // ApplyIntervalAsTimeFilter is only valid for vlogs datasource. + // Set to true if there is no [timeFilter](https://docs.victoriametrics.com/victorialogs/logsql/#time-filter) in the rule expression, + // and we will add evaluation interval as an additional timeFilter when querying. + ApplyIntervalAsTimeFilter bool + EvaluationInterval time.Duration + QueryParams url.Values + Headers map[string]string + Debug bool } // Metric is the basic entity which should be return by datasource diff --git a/app/vmalert/datasource/init.go b/app/vmalert/datasource/init.go index a99f130e8..4e163ab71 100644 --- a/app/vmalert/datasource/init.go +++ b/app/vmalert/datasource/init.go @@ -133,13 +133,12 @@ func Init(extraParams url.Values) (QuerierBuilder, error) { return nil, fmt.Errorf("failed to set request auth header to datasource %q: %w", *addr, err) } - return &VMStorage{ + return &Client{ c: &http.Client{Transport: tr}, authCfg: authCfg, datasourceURL: strings.TrimSuffix(*addr, "/"), appendTypePrefix: *appendTypePrefix, queryStep: *queryStep, - dataSourceType: datasourcePrometheus, extraParams: extraParams, }, nil } diff --git a/app/vmalert/datasource/vm.go b/app/vmalert/datasource/vm.go deleted file mode 100644 index c8258fec4..000000000 --- a/app/vmalert/datasource/vm.go +++ /dev/null @@ -1,272 +0,0 @@ -package datasource - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" -) - -type datasourceType string - -const ( - datasourcePrometheus datasourceType = "prometheus" - datasourceGraphite datasourceType = "graphite" -) - -func toDatasourceType(s string) datasourceType { - if s == string(datasourceGraphite) { - return datasourceGraphite - } - return datasourcePrometheus -} - -// VMStorage represents vmstorage entity with ability to read and write metrics -// WARN: when adding a new field, remember to update Clone() method. -type VMStorage struct { - c *http.Client - authCfg *promauth.Config - datasourceURL string - appendTypePrefix bool - queryStep time.Duration - dataSourceType datasourceType - - // evaluationInterval will help setting request's `step` param. - evaluationInterval time.Duration - // extraParams contains params to be attached to each HTTP request - extraParams url.Values - // extraHeaders are headers to be attached to each HTTP request - extraHeaders []keyValue - - // whether to print additional log messages - // for each sent request - debug bool -} - -type keyValue struct { - key string - value string -} - -// Clone makes clone of VMStorage, shares http client. -func (s *VMStorage) Clone() *VMStorage { - ns := &VMStorage{ - c: s.c, - authCfg: s.authCfg, - datasourceURL: s.datasourceURL, - appendTypePrefix: s.appendTypePrefix, - queryStep: s.queryStep, - - dataSourceType: s.dataSourceType, - evaluationInterval: s.evaluationInterval, - - // init map so it can be populated below - extraParams: url.Values{}, - - debug: s.debug, - } - if len(s.extraHeaders) > 0 { - ns.extraHeaders = make([]keyValue, len(s.extraHeaders)) - copy(ns.extraHeaders, s.extraHeaders) - } - for k, v := range s.extraParams { - ns.extraParams[k] = v - } - - return ns -} - -// ApplyParams - changes given querier params. -func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage { - s.dataSourceType = toDatasourceType(params.DataSourceType) - s.evaluationInterval = params.EvaluationInterval - if params.QueryParams != nil { - if s.extraParams == nil { - s.extraParams = url.Values{} - } - for k, vl := range params.QueryParams { - // custom query params are prior to default ones - if s.extraParams.Has(k) { - s.extraParams.Del(k) - } - for _, v := range vl { - // don't use .Set() instead of Del/Add since it is allowed - // for GET params to be duplicated - // see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4908 - s.extraParams.Add(k, v) - } - } - } - if params.Headers != nil { - for key, value := range params.Headers { - kv := keyValue{key: key, value: value} - s.extraHeaders = append(s.extraHeaders, kv) - } - } - s.debug = params.Debug - return s -} - -// BuildWithParams - implements interface. -func (s *VMStorage) BuildWithParams(params QuerierParams) Querier { - return s.Clone().ApplyParams(params) -} - -// NewVMStorage is a constructor for VMStorage -func NewVMStorage(baseURL string, authCfg *promauth.Config, queryStep time.Duration, appendTypePrefix bool, c *http.Client) *VMStorage { - return &VMStorage{ - c: c, - authCfg: authCfg, - datasourceURL: strings.TrimSuffix(baseURL, "/"), - appendTypePrefix: appendTypePrefix, - queryStep: queryStep, - dataSourceType: datasourcePrometheus, - extraParams: url.Values{}, - } -} - -// Query executes the given query and returns parsed response -func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Result, *http.Request, error) { - req, err := s.newQueryRequest(ctx, query, ts) - if err != nil { - return Result{}, nil, err - } - resp, err := s.do(req) - if err != nil { - if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) { - // Return unexpected error to the caller. - return Result{}, nil, err - } - // Something in the middle between client and datasource might be closing - // the connection. So we do a one more attempt in hope request will succeed. - req, err = s.newQueryRequest(ctx, query, ts) - if err != nil { - return Result{}, nil, fmt.Errorf("second attempt: %w", err) - } - resp, err = s.do(req) - if err != nil { - return Result{}, nil, fmt.Errorf("second attempt: %w", err) - } - } - - // Process the received response. - parseFn := parsePrometheusResponse - if s.dataSourceType != datasourcePrometheus { - parseFn = parseGraphiteResponse - } - result, err := parseFn(req, resp) - _ = resp.Body.Close() - return result, req, err -} - -// QueryRange executes the given query on the given time range. -// For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries -// Graphite type isn't supported. -func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end time.Time) (res Result, err error) { - if s.dataSourceType != datasourcePrometheus { - return res, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType) - } - if start.IsZero() { - return res, fmt.Errorf("start param is missing") - } - if end.IsZero() { - return res, fmt.Errorf("end param is missing") - } - req, err := s.newQueryRangeRequest(ctx, query, start, end) - if err != nil { - return res, err - } - resp, err := s.do(req) - if err != nil { - if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) { - // Return unexpected error to the caller. - return res, err - } - // Something in the middle between client and datasource might be closing - // the connection. So we do a one more attempt in hope request will succeed. - req, err = s.newQueryRangeRequest(ctx, query, start, end) - if err != nil { - return res, fmt.Errorf("second attempt: %w", err) - } - resp, err = s.do(req) - if err != nil { - return res, fmt.Errorf("second attempt: %w", err) - } - } - - // Process the received response. - res, err = parsePrometheusResponse(req, resp) - _ = resp.Body.Close() - return res, err -} - -func (s *VMStorage) do(req *http.Request) (*http.Response, error) { - ru := req.URL.Redacted() - if *showDatasourceURL { - ru = req.URL.String() - } - if s.debug { - logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, ru) - } - resp, err := s.c.Do(req) - if err != nil { - return nil, fmt.Errorf("error getting response from %s: %w", ru, err) - } - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body) - } - return resp, nil -} - -func (s *VMStorage) newQueryRangeRequest(ctx context.Context, query string, start, end time.Time) (*http.Request, error) { - req, err := s.newRequest(ctx) - if err != nil { - return nil, fmt.Errorf("cannot create query_range request to datasource %q: %w", s.datasourceURL, err) - } - s.setPrometheusRangeReqParams(req, query, start, end) - return req, nil -} - -func (s *VMStorage) newQueryRequest(ctx context.Context, query string, ts time.Time) (*http.Request, error) { - req, err := s.newRequest(ctx) - if err != nil { - return nil, fmt.Errorf("cannot create query request to datasource %q: %w", s.datasourceURL, err) - } - switch s.dataSourceType { - case "", datasourcePrometheus: - s.setPrometheusInstantReqParams(req, query, ts) - case datasourceGraphite: - s.setGraphiteReqParams(req, query) - default: - logger.Panicf("BUG: engine not found: %q", s.dataSourceType) - } - return req, nil -} - -func (s *VMStorage) newRequest(ctx context.Context) (*http.Request, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.datasourceURL, nil) - if err != nil { - logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", s.datasourceURL, err) - } - req.Header.Set("Content-Type", "application/json") - if s.authCfg != nil { - err = s.authCfg.SetHeaders(req, true) - if err != nil { - return nil, err - } - } - for _, h := range s.extraHeaders { - req.Header.Set(h.key, h.value) - } - return req, nil -} diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 53af8f210..9e14d03dd 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -66,7 +66,7 @@ absolute path to all .tpl files in root. evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules") validateTemplates = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates") - validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine") + validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions for different types.") externalURL = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier. By default, hostname is used as address.") externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+ diff --git a/app/vmalert/remoteread/init.go b/app/vmalert/remoteread/init.go index 612db0fad..b134dad42 100644 --- a/app/vmalert/remoteread/init.go +++ b/app/vmalert/remoteread/init.go @@ -86,5 +86,5 @@ func Init() (datasource.QuerierBuilder, error) { return nil, fmt.Errorf("failed to configure auth: %w", err) } c := &http.Client{Transport: tr} - return datasource.NewVMStorage(*addr, authCfg, 0, false, c), nil + return datasource.NewPrometheusClient(*addr, authCfg, false, c), nil } diff --git a/app/vmalert/rule/alerting.go b/app/vmalert/rule/alerting.go index 5c9a17393..f0d7101a9 100644 --- a/app/vmalert/rule/alerting.go +++ b/app/vmalert/rule/alerting.go @@ -72,11 +72,12 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule EvalInterval: group.Interval, Debug: cfg.Debug, q: qb.BuildWithParams(datasource.QuerierParams{ - DataSourceType: group.Type.String(), - EvaluationInterval: group.Interval, - QueryParams: group.Params, - Headers: group.Headers, - Debug: cfg.Debug, + DataSourceType: group.Type.String(), + ApplyIntervalAsTimeFilter: setIntervalAsTimeFilter(group.Type.String(), cfg.Expr), + EvaluationInterval: group.Interval, + QueryParams: group.Params, + Headers: group.Headers, + Debug: cfg.Debug, }), alerts: make(map[uint64]*notifier.Alert), metrics: &alertingRuleMetrics{}, diff --git a/app/vmalert/rule/group.go b/app/vmalert/rule/group.go index 5051a6082..5f0e42329 100644 --- a/app/vmalert/rule/group.go +++ b/app/vmalert/rule/group.go @@ -213,7 +213,6 @@ func (g *Group) restore(ctx context.Context, qb datasource.QuerierBuilder, ts ti continue } q := qb.BuildWithParams(datasource.QuerierParams{ - DataSourceType: g.Type.String(), EvaluationInterval: g.Interval, QueryParams: g.Params, Headers: g.Headers, diff --git a/app/vmalert/rule/recording.go b/app/vmalert/rule/recording.go index c015dfe06..3de0524b5 100644 --- a/app/vmalert/rule/recording.go +++ b/app/vmalert/rule/recording.go @@ -10,6 +10,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" ) @@ -64,10 +65,11 @@ func NewRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul File: group.File, metrics: &recordingRuleMetrics{}, q: qb.BuildWithParams(datasource.QuerierParams{ - DataSourceType: group.Type.String(), - EvaluationInterval: group.Interval, - QueryParams: group.Params, - Headers: group.Headers, + DataSourceType: group.Type.String(), + ApplyIntervalAsTimeFilter: setIntervalAsTimeFilter(group.Type.String(), cfg.Expr), + EvaluationInterval: group.Interval, + QueryParams: group.Params, + Headers: group.Headers, }), } @@ -213,3 +215,12 @@ func (rr *RecordingRule) updateWith(r Rule) error { rr.q = nr.q return nil } + +// setIntervalAsTimeFilter returns true if given LogsQL has a time filter. +func setIntervalAsTimeFilter(dType, expr string) bool { + if dType != "vlogs" { + return false + } + q, _ := logstorage.ParseStatsQuery(expr) + return !q.ContainAnyTimeFilter() +} diff --git a/app/vmalert/rule/recording_test.go b/app/vmalert/rule/recording_test.go index 62b9e7265..469b4285a 100644 --- a/app/vmalert/rule/recording_test.go +++ b/app/vmalert/rule/recording_test.go @@ -266,3 +266,25 @@ func TestRecordingRuleExec_Negative(t *testing.T) { t.Fatalf("cannot execute recroding rule: %s", err) } } + +func TestSetIntervalAsTimeFilter(t *testing.T) { + f := func(s, dType string, expected bool) { + t.Helper() + + if setIntervalAsTimeFilter(dType, s) != expected { + t.Fatalf("unexpected result for hasTimeFilter(%q); want %v", s, expected) + } + } + + f(`* | count()`, "prometheus", false) + + f(`* | count()`, "vlogs", true) + f(`error OR _time:5m | count()`, "vlogs", true) + f(`(_time: 5m AND error) OR (_time: 5m AND warn) | count()`, "vlogs", true) + f(`* | error OR _time:5m | count()`, "vlogs", true) + + f(`_time:5m | count()`, "vlogs", false) + f(`_time:2023-04-25T22:45:59Z | count()`, "vlogs", false) + f(`error AND _time:5m | count()`, "vlogs", false) + f(`* | error AND _time:5m | count()`, "vlogs", false) +} diff --git a/deployment/docker/README.md b/deployment/docker/README.md index cd012c09a..d9e136af7 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -105,7 +105,7 @@ vmauth config is available [here](ttps://github.com/VictoriaMetrics/VictoriaMetr ## vmalert -vmalert evaluates alerting rules [alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) +vmalert evaluates alerting rules [alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts.yml) to track VictoriaMetrics health state. It is connected with AlertManager for firing alerts, and with VictoriaMetrics for executing queries and storing alert's state. @@ -153,17 +153,17 @@ make docker-cluster-vm-datasource-down # shutdown cluster See below a list of recommended alerting rules for various VictoriaMetrics components for running in production. Some alerting rules thresholds are just recommendations and could require an adjustment. The list of alerting rules is the following: -* [alerts-health.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml): +* [alerts-health.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml): alerting rules related to all VictoriaMetrics components for tracking their "health" state; -* [alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml): +* [alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts.yml): alerting rules related to [single-server VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) installation; -* [alerts-cluster.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml): +* [alerts-cluster.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-cluster.yml): alerting rules related to [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/cluster-victoriametrics/); -* [alerts-vmagent.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmagent.yml): +* [alerts-vmagent.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmagent.yml): alerting rules related to [vmagent](https://docs.victoriametrics.com/vmagent/) component; -* [alerts-vmalert.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmalert.yml): +* [alerts-vmalert.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml): alerting rules related to [vmalert](https://docs.victoriametrics.com/vmalert/) component; -* [alerts-vmauth.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmauth.yml): +* [alerts-vmauth.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml): alerting rules related to [vmauth](https://docs.victoriametrics.com/vmauth/) component; * [alerts-vlogs.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vlogs.yml): alerting rules related to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/); diff --git a/deployment/docker/auth-mixed-datasource.yml b/deployment/docker/auth-mixed-datasource.yml new file mode 100644 index 000000000..06eb47bbb --- /dev/null +++ b/deployment/docker/auth-mixed-datasource.yml @@ -0,0 +1,9 @@ +# route requests between VictoriaMetrics and VictoriaLogs +unauthorized_user: + url_map: + - src_paths: + - "/api/v1/query.*" + url_prefix: "http://victoriametrics:8428" + - src_paths: + - "/select/logsql/.*" + url_prefix: "http://victorialogs:9428" diff --git a/deployment/docker/docker-compose-cluster.yml b/deployment/docker/docker-compose-cluster.yml index 088bb8681..0ebe2e757 100644 --- a/deployment/docker/docker-compose-cluster.yml +++ b/deployment/docker/docker-compose-cluster.yml @@ -133,10 +133,10 @@ services: ports: - 8880:8880 volumes: - - ./alerts-cluster.yml:/etc/alerts/alerts.yml - - ./alerts-health.yml:/etc/alerts/alerts-health.yml - - ./alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml - - ./alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml + - ./rules/alerts-cluster.yml:/etc/alerts/alerts.yml + - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml + - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml + - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml command: - "--datasource.url=http://vmauth:8427/select/0/prometheus" - "--remoteRead.url=http://vmauth:8427/select/0/prometheus" diff --git a/deployment/docker/docker-compose-victorialogs.yml b/deployment/docker/docker-compose-victorialogs.yml index b18c05718..091c3655d 100644 --- a/deployment/docker/docker-compose-victorialogs.yml +++ b/deployment/docker/docker-compose-victorialogs.yml @@ -26,7 +26,7 @@ services: # and forwards them to VictoriaLogs fluentbit: container_name: fluentbit - image: cr.fluentbit.io/fluent/fluent-bit:2.1.4 + image: fluent/fluent-bit:2.1.4 volumes: - /var/lib/docker/containers:/var/lib/docker/containers:ro - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf @@ -40,7 +40,7 @@ services: # storing logs and serving read queries. victorialogs: container_name: victorialogs - image: docker.io/victoriametrics/victoria-logs:v0.37.0-victorialogs + image: victoriametrics/victoria-logs:v0.37.0-victorialogs command: - "--storageDataPath=/vlogs" - "--httpListenAddr=:9428" @@ -69,29 +69,50 @@ services: - vm_net restart: always - # vmalert executes alerting and recording rules + # vmauth is a router and balancer for HTTP requests. + # It proxies query requests from vmalert to either VictoriaMetrics or VictoriaLogs, + # depending on the requested path. + vmauth: + container_name: vmauth + image: victoriametrics/vmauth:v1.105.0 + depends_on: + - "victoriametrics" + - "victorialogs" + volumes: + - ./auth-mixed-datasource.yml:/etc/auth.yml + command: + - "--auth.config=/etc/auth.yml" + ports: + - 8427:8427 + networks: + - vm_net + restart: always + + # vmalert executes alerting and recording rules according to given rule type. vmalert: container_name: vmalert image: victoriametrics/vmalert:v1.105.0 depends_on: - - "victoriametrics" + - "vmauth" - "alertmanager" + - "victoriametrics" ports: - 8880:8880 volumes: - - ./alerts.yml:/etc/alerts/alerts.yml - - ./alerts-health.yml:/etc/alerts/alerts-health.yml - - ./alerts-vlogs.yml:/etc/alerts/alerts-vlogs.yml - - ./alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml + # disable log-related rules for now, util vmalert supports vlogs type rule + # - ./rules/vlogs-example-alerts.yml:/etc/alerts/vlogs.yml + - ./rules/alerts.yml:/etc/alerts/alerts.yml + - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml + - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml + - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml command: - - "--datasource.url=http://victoriametrics:8428/" + - "--datasource.url=http://vmauth:8427/" - "--remoteRead.url=http://victoriametrics:8428/" - "--remoteWrite.url=http://victoriametrics:8428/" - "--notifier.url=http://alertmanager:9093/" - "--rule=/etc/alerts/*.yml" # display source of alerts in grafana - "--external.url=http://127.0.0.1:3000" #grafana outside container - - '--external.alert.source=explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{.Expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"{{ .ActiveAt.UnixMilli }}","to":"now"}}' networks: - vm_net restart: always diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index 341b3d321..8d279fba1 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -72,10 +72,10 @@ services: ports: - 8880:8880 volumes: - - ./alerts.yml:/etc/alerts/alerts.yml - - ./alerts-health.yml:/etc/alerts/alerts-health.yml - - ./alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml - - ./alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml + - ./rules/alerts.yml:/etc/alerts/alerts.yml + - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml + - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml + - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml command: - "--datasource.url=http://victoriametrics:8428/" - "--remoteRead.url=http://victoriametrics:8428/" diff --git a/deployment/docker/alerts-cluster.yml b/deployment/docker/rules/alerts-cluster.yml similarity index 100% rename from deployment/docker/alerts-cluster.yml rename to deployment/docker/rules/alerts-cluster.yml diff --git a/deployment/docker/alerts-health.yml b/deployment/docker/rules/alerts-health.yml similarity index 100% rename from deployment/docker/alerts-health.yml rename to deployment/docker/rules/alerts-health.yml diff --git a/deployment/docker/alerts-vmagent.yml b/deployment/docker/rules/alerts-vmagent.yml similarity index 100% rename from deployment/docker/alerts-vmagent.yml rename to deployment/docker/rules/alerts-vmagent.yml diff --git a/deployment/docker/alerts-vmalert.yml b/deployment/docker/rules/alerts-vmalert.yml similarity index 100% rename from deployment/docker/alerts-vmalert.yml rename to deployment/docker/rules/alerts-vmalert.yml diff --git a/deployment/docker/alerts-vmauth.yml b/deployment/docker/rules/alerts-vmauth.yml similarity index 100% rename from deployment/docker/alerts-vmauth.yml rename to deployment/docker/rules/alerts-vmauth.yml diff --git a/deployment/docker/alerts.yml b/deployment/docker/rules/alerts.yml similarity index 100% rename from deployment/docker/alerts.yml rename to deployment/docker/rules/alerts.yml diff --git a/deployment/docker/rules/vlogs-example-alerts.yml b/deployment/docker/rules/vlogs-example-alerts.yml new file mode 100644 index 000000000..7ba25c9ea --- /dev/null +++ b/deployment/docker/rules/vlogs-example-alerts.yml @@ -0,0 +1,13 @@ +groups: + - name: TestGroup + type: vlogs + interval: 1m + rules: + - record: logCount + expr: '_time: 1m | stats by (path) count () as total' + annotations: + description: "path {{$labels.path}} generated {{$value}} logs in the last 1 minute" + - alert: tooManyLogs + expr: '_time: 1m | stats by (path) count () as total | filter total:>50' + annotations: + description: "path {{$labels.path}} generated more than 50 log entries in the last 1 minute: {{$value}}" diff --git a/docs/VictoriaLogs/vmalert.md b/docs/VictoriaLogs/vmalert.md new file mode 100644 index 000000000..db1feaa1c --- /dev/null +++ b/docs/VictoriaLogs/vmalert.md @@ -0,0 +1,207 @@ +--- +weight: 10 +title: vmalert +menu: + docs: + parent: "victorialogs" + weight: 10 +aliases: +- /VictoriaLogs/vmalert.html +--- + +_Available from [TODO](https://docs.victoriametrics.com/changelog/#TODO) vmalert version and [v0.36.0](https://docs.victoriametrics.com/victorialogs/changelog/#v0360) VictoriaLogs version._ + +[vmalert](https://docs.victoriametrics.com/vmalert/) integrates with VictoriaLogs via stats APIs [`/select/logsql/stats_query`](https://docs.victoriametrics.com/victorialogs/querying/#querying-log-stats) +and [`/select/logsql/stats_query_range`](https://docs.victoriametrics.com/victorialogs/querying/#querying-log-range-stats). +These endpoints return the log stats in a format compatible with [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries). +It allows using VictoriaLogs as the datasource in vmalert, creating alerting and recording rules via [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). + +_Note: This page provides only integration instructions for vmalert and VictoriaLogs. See the full textbook for vmalert [here](https://docs.victoriametrics.com/vmalert)._ + +## Quick Start + +Run vmalert with `-rule.defaultRuleType=vlogs` cmd-line flag. +``` +./bin/vmalert -rule=alert.rules \ # Path to the files or http url with alerting and/or recording rules in YAML format. + -datasource.url=http://localhost:9428 \ # VictoriaLogs address. + -rule.defaultRuleType=vlogs \ # Set default rules type to VictoriaLogs. + -notifier.url=http://localhost:9093 \ # AlertManager URL (required if alerting rules are used) + -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info (required for recording rules) + -remoteRead.url=http://localhost:8428 \ # Prometheus HTTP API compatible datasource to restore alerts state from +``` + +> See the full list of configuration flags and their descriptions in [configuration](#configuration) section. + +> Each `-rule` file may contain arbitrary number of [groups](https://docs.victoriametrics.com/vmalert/#groups). +See examples in [Groups](#groups) section. + +With configuration example above, vmalert will perform the following interactions: +![vmalert](vmalert_victorialogs.webp) + +1. Rules listed in `-rule` file are executed against VictoriaLogs service configured via `-datasource.url`; +2. Triggered alerting notifications are sent to [Alertmanager](https://github.com/prometheus/alertmanager) service configured via `-notifier.url`; +3. Results of recording rules expressions and alerts state are persisted to Prometheus-compatible remote-write endpoint (i.e. VictoriaMetrics) configured via `-remoteWrite.url`; +4. On vmalert restarts, alerts state [can be restored](https://docs.victoriametrics.com/vmalert/#alerts-state-on-restarts) by querying Prometheus-compatible HTTP API endpoint (i.e. VictoriaMetrics) configured via `-remoteRead.url`. + +## Configuration + +### Flags + +For a complete list of command-line flags, visit https://docs.victoriametrics.com/vmalert/#flags or execute `./vmalert --help` command. +The following are key flags related to integration with VictoriaLogs: + +``` +-datasource.url string + Datasource address supporting log stats APIs, which can be a single VictoriaLogs node or a proxy in front of VictoriaLogs. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. +-notifier.url array + Prometheus Alertmanager URL, e.g. http://127.0.0.1:9093. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability. + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. +-remoteWrite.url string + Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'. +-remoteRead.url string + Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'. +-rule array + Path to the files or http url with alerting and/or recording rules in YAML format. + Supports hierarchical patterns and regexpes. + Examples: + -rule="/path/to/file". Path to a single file with alerting rules. + -rule="http:///path/to/rules". HTTP URL to a page with alerting rules. + -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". + -rule="dir/**/*.yaml". Includes all the .yaml files in "dir" subfolders recursively. + Rule files support YAML multi-document. Files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. + Enterprise version of vmalert supports S3 and GCS paths to rules. + For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules + S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches + all files with prefix rule_ in folder dir. + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. +-rule.defaultRuleType + Default type for rule expressions, can be overridden by type parameter inside the rule group. Supported values: "graphite", "prometheus" and "vlogs". + Default is "prometheus", change it to "vlogs" if all of the rules are written with LogsQL. +-rule.evalDelay time + Adjustment of the time parameter for rule evaluation requests to compensate intentional data delay from the datasource. Normally, should be equal to `-search.latencyOffset` (cm d-line flag configured for VictoriaMetrics single-node or vmselect). + Since there is no intentional search delay in VictoriaLogs, `-rule.evalDelay` can be reduced to a few seconds to accommodate network and ingestion time. +``` + +For more configuration options, such as `notifiers`, visit https://docs.victoriametrics.com/vmalert/#configuration. + +### Groups + +Check the complete group attributes [here](https://docs.victoriametrics.com/vmalert/#groups). + +#### Alerting rules + +Examples: +``` +groups: + - name: ServiceLog + interval: 5m + rules: + - alert: HasErrorLog + expr: 'env: "prod" AND status:~"error|warn" | stats by (service) count() as errorLog | filter errorLog:>0' + annotations: + description: "Service {{$labels.service}} generated {{$labels.errorLog}} error logs in the last 5 minutes" + + - name: ServiceRequest + interval: 5m + rules: + - alert: TooManyFailedRequest + expr: '* | extract "ip= " | extract "status_code=;" | stats by (ip, code) count() if (code:~4.*) as failed, count() as total| math failed / total as failed_percentage| filter failed_percentage :> 0.01 | fields ip,failed_percentage' + annotations: + description: "Connection from address {{$labels.ip}} has {{$value}}% failed requests in last 5 minutes" +``` + +#### Recording rules + +Examples: +``` +groups: + - name: RequestCount + interval: 5m + rules: + - record: nginxRequestCount + expr: 'env: "test" AND service: "nginx" | stats count(*) as requests' + annotations: + description: "Service nginx on env test accepted {{$labels.requests}} requests in the last 5 minutes" + - record: prodRequestCount + expr: 'env: "prod" | stats by (service) count(*) as requests' + annotations: + description: "Service {{$labels.service}} on env prod accepted {{$labels.requests}} requests in the last 5 minutes" +``` + +## Time filter + +It's recommended to omit the [time filter](https://docs.victoriametrics.com/victorialogs/logsql/#time-filter) in rule expression. +By default, vmalert automatically appends the time filter `_time: ` to the expression. +For instance, the rule below will be evaluated every 5 minutes, and will return the result with logs from the last 5 minutes: +``` +groups: + interval: 5m + rules: + - alert: TooManyFailedRequest + expr: '* | extract "ip= " | extract "status_code=;" | stats by (ip, code) count() if (code:~4.*) as failed, count() as total| math failed / total as failed_percentage| filter failed_percentage :> 0.01 | fields ip,failed_percentage' + annotations: "Connection from address {{$labels.ip}} has {{$$value}}% failed requests in last 5 minutes" +``` + +User can also specify a customized time filter if needed. For example, rule below will be evaluated every 5 minutes, +but will calculate result over the logs from the last 10 minutes. +``` +groups: + interval: 5m + rules: + - alert: TooManyFailedRequest + expr: '_time: 10m | extract "ip= " | extract "status_code=;" | stats by (ip, code) count() if (code:~4.*) as failed, count() as total| math failed / total as failed_percentage| filter failed_percentage :> 0.01 | fields ip,failed_percentage' + annotations: "Connection from address {{$labels.ip}} has {{$$value}}% failed requests in last 10 minutes" +``` + +Please note, vmalert doesn't support [backfilling](#rules-backfilling) for rules with a customized time filter now. (Might be added in future) + +## Rules backfilling + +vmalert supports alerting and recording rules backfilling (aka replay) against VictoriaLogs as the datasource. +``` +./bin/vmalert -rule=path/to/your.rules \ # path to files with rules you usually use with vmalert + -datasource.url=http://localhost:9428 \ # VictoriaLogs address. + -rule.defaultRuleType=vlogs \ # Set default rule type to VictoriaLogs. + -remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info + -replay.timeFrom=2021-05-11T07:21:43Z \ # to start replay from + -replay.timeTo=2021-05-29T18:40:43Z # to finish replay by, is optional +``` + +See more details about backfilling [here](https://docs.victoriametrics.com/vmalert/#rules-backfilling). + +## Performance tip + +LogsQL allows users to obtain multiple stats from a single expression. +For instance, the following query calculates 50th, 90th and 99th percentiles for the `request_duration_seconds` field over logs for the last 5 minutes: + +``` +_time:5m | stats + quantile(0.5, request_duration_seconds) p50, + quantile(0.9, request_duration_seconds) p90, + quantile(0.99, request_duration_seconds) p99 +``` + +This expression can also be used in recording rules as follows: +``` +groups: + - name: requestDuration + interval: 5m + rules: + - record: requestDurationQuantile + expr: '_time:5m | stats by (service) quantile(0.5, request_duration_seconds) p50, quantile(0.9, request_duration_seconds) p90, quantile(0.99, request_duration_seconds) p99' +``` +This creates three metrics for each service: +``` +requestDurationQuantile{stats_result="p50", service="service-1"} +requestDurationQuantile{stats_result="p90", service="service-1"} +requestDurationQuantile{stats_result="p99", service="service-1"} + +requestDurationQuantile{stats_result="p50", service="service-2"} +requestDurationQuantile{stats_result="p90", service="service-2"} +requestDurationQuantile{stats_result="p00", service="service-2"} +... +``` + +For additional tips on writing LogsQL, refer to this [doc](https://docs.victoriametrics.com/victorialogs/logsql/#performance-tips). \ No newline at end of file diff --git a/docs/VictoriaLogs/vmalert_victorialogs.excalidraw b/docs/VictoriaLogs/vmalert_victorialogs.excalidraw new file mode 100644 index 000000000..a3bc8bbb5 --- /dev/null +++ b/docs/VictoriaLogs/vmalert_victorialogs.excalidraw @@ -0,0 +1,687 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "type": "rectangle", + "version": 803, + "versionNonce": 1128884469, + "index": "a0", + "isDeleted": false, + "id": "VgBUzo0blGR-Ijd2mQEEf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 422.3502197265625, + "y": 215.55953979492188, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 123.7601318359375, + "height": 72.13211059570312, + "seed": 1194011660, + "groupIds": [ + "iBaXgbpyifSwPplm_GO5b" + ], + "frameId": null, + "roundness": null, + "boundElements": [ + { + "type": "arrow", + "id": "sxEhnxlbT7ldlSsmHDUHp" + }, + { + "id": "wRO0q9xKPHc8e8XPPsQWh", + "type": "arrow" + }, + { + "id": "Bpy5by47XGKB4yS99ZkuA", + "type": "arrow" + } + ], + "updated": 1728889265677, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 660, + "versionNonce": 130510869, + "index": "a1", + "isDeleted": false, + "id": "e9TDm09y-GhPm84XWt0Jv", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 443.89678955078125, + "y": 236.64378356933594, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 82, + "height": 24, + "seed": 327273100, + "groupIds": [ + "iBaXgbpyifSwPplm_GO5b" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1728889112138, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 3, + "text": "vmalert", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": null, + "originalText": "vmalert", + "autoResize": true, + "lineHeight": 1.2 + }, + { + "type": "rectangle", + "version": 2608, + "versionNonce": 1050127035, + "index": "a2", + "isDeleted": false, + "id": "dd52BjHfPMPRji9Tws7U-", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 774.7067312730577, + "y": 231.9532470703125, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 275.7981470513237, + "height": 39.621179787868925, + "seed": 1779959692, + "groupIds": [ + "2Lijjn3PwPQW_8KrcDmdu" + ], + "frameId": null, + "roundness": null, + "boundElements": [ + { + "id": "Bpy5by47XGKB4yS99ZkuA", + "type": "arrow" + } + ], + "updated": 1728889420961, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 1099, + "versionNonce": 499029243, + "index": "a6", + "isDeleted": false, + "id": "8-XFSbd6Zw96EUSJbJXZv", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 371.7434387207031, + "y": 398.50787353515625, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 240.10644531249997, + "height": 44.74725341796875, + "seed": 99322124, + "groupIds": [ + "6obQBPHIfExBKfejeLLVO" + ], + "frameId": null, + "roundness": null, + "boundElements": [ + { + "type": "arrow", + "id": "sxEhnxlbT7ldlSsmHDUHp" + } + ], + "updated": 1728889112138, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 865, + "versionNonce": 316509237, + "index": "a7", + "isDeleted": false, + "id": "GUs816aggGqUSdoEsSmea", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 393.73809814453125, + "y": 410.5976257324219, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 199, + "height": 24, + "seed": 1194745268, + "groupIds": [ + "6obQBPHIfExBKfejeLLVO" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1728889112138, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 3, + "text": "alertmanager:9093", + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "originalText": "alertmanager:9093", + "autoResize": true, + "lineHeight": 1.2 + }, + { + "type": "arrow", + "version": 3377, + "versionNonce": 359177051, + "index": "a8", + "isDeleted": false, + "id": "Bpy5by47XGKB4yS99ZkuA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 556.6860961914062, + "y": 252.95352770712083, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 202.02063508165145, + "height": 0.22881326742660235, + "seed": 357577356, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1728889420962, + "link": null, + "locked": false, + "startBinding": { + "elementId": "VgBUzo0blGR-Ijd2mQEEf", + "focus": 0.0344528515859526, + "gap": 10.57574462890625 + }, + "endBinding": { + "elementId": "dd52BjHfPMPRji9Tws7U-", + "focus": -0.039393828258510157, + "gap": 16 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 202.02063508165145, + -0.22881326742660235 + ] + ] + }, + { + "type": "arrow", + "version": 1460, + "versionNonce": 492906299, + "index": "a9", + "isDeleted": false, + "id": "wRO0q9xKPHc8e8XPPsQWh", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 406.0439244722469, + "y": 246.6775563467225, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 161.00829839007181, + "height": 2.320722012761223, + "seed": 656189364, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1728889313672, + "link": null, + "locked": false, + "startBinding": { + "elementId": "VgBUzo0blGR-Ijd2mQEEf", + "focus": 0.13736472619498497, + "gap": 16.306295254315614 + }, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -161.00829839007181, + -2.320722012761223 + ] + ] + }, + { + "type": "text", + "version": 567, + "versionNonce": 737159899, + "index": "aA", + "isDeleted": false, + "id": "RbVSa4PnOgAMtzoKb-DhW", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 552.4987182617188, + "y": 212.27996826171875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 187.75, + "height": 95, + "seed": 1989838604, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [ + { + "id": "ijEBAhsESSoR3zLPouUVM", + "type": "arrow" + } + ], + "updated": 1728889402055, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "persist alerts state\nand recording rules\n\n\n", + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "persist alerts state\nand recording rules\n\n\n", + "autoResize": true, + "lineHeight": 1.1875 + }, + { + "type": "text", + "version": 830, + "versionNonce": 1996455189, + "index": "aB", + "isDeleted": false, + "id": "ia2QzZNl_tuvfY3ymLjyJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 279.55224609375, + "y": 218.88568115234375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 122, + "height": 19, + "seed": 157304972, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [ + { + "type": "arrow", + "id": "wRO0q9xKPHc8e8XPPsQWh" + } + ], + "updated": 1728889440112, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "execute rules", + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "execute rules", + "autoResize": true, + "lineHeight": 1.1875 + }, + { + "type": "arrow", + "version": 1476, + "versionNonce": 1814378875, + "index": "aC", + "isDeleted": false, + "id": "sxEhnxlbT7ldlSsmHDUHp", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 484.18669893674246, + "y": 302.3424013553929, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 1.0484739253853945, + "height": 84.72775855671654, + "seed": 1818348300, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1728889265678, + "link": null, + "locked": false, + "startBinding": { + "elementId": "VgBUzo0blGR-Ijd2mQEEf", + "focus": 0.010768924644894236, + "gap": 14.650750964767894 + }, + "endBinding": { + "elementId": "8-XFSbd6Zw96EUSJbJXZv", + "focus": -0.051051952959743775, + "gap": 11.437713623046818 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 1.0484739253853945, + 84.72775855671654 + ] + ] + }, + { + "type": "text", + "version": 631, + "versionNonce": 1909410773, + "index": "aD", + "isDeleted": false, + "id": "E9Run6wCm2chQ6JHrmc_y", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 504.27996826171875, + "y": 322.13031005859375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 122, + "height": 38, + "seed": 1836541708, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [ + { + "type": "arrow", + "id": "sxEhnxlbT7ldlSsmHDUHp" + } + ], + "updated": 1728889430719, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "send alert \nnotifications", + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "send alert \nnotifications", + "autoResize": true, + "lineHeight": 1.1875 + }, + { + "type": "text", + "version": 579, + "versionNonce": 326648123, + "index": "aE", + "isDeleted": false, + "id": "ff5OkfgmkKLifS13_TFj3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 591.5895843505859, + "y": 269.2361297607422, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 121.875, + "height": 19, + "seed": 264004620, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [ + { + "type": "arrow", + "id": "wRO0q9xKPHc8e8XPPsQWh" + } + ], + "updated": 1728889436228, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 3, + "text": "restore state", + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "restore state", + "autoResize": true, + "lineHeight": 1.1875 + }, + { + "type": "text", + "version": 1141, + "versionNonce": 39140603, + "index": "aG", + "isDeleted": false, + "id": "J2AqHIHYjG3cvxrBLonQW", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 782.2813415527344, + "y": 238.312045541553, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 254.41375732421875, + "height": 26.05968577236269, + "seed": 254079515, + "groupIds": [ + "fw8b83Mw6tGXQ80jfC5Jx" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1728889417069, + "link": null, + "locked": false, + "fontSize": 21.716404810302244, + "fontFamily": 3, + "text": "victoriametrics:8428", + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "originalText": "victoriametrics:8428", + "autoResize": true, + "lineHeight": 1.2 + }, + { + "type": "rectangle", + "version": 2824, + "versionNonce": 1550880827, + "index": "aH", + "isDeleted": false, + "id": "Whj4hd3Al6CbvGs7cQuWk", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": -11.824915810818197, + "y": 223.79106415879994, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 248.85674080132372, + "height": 40.562586037868925, + "seed": 1519267323, + "groupIds": [ + "skPAIqL9ijNA0WE5GV8Gv" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1728889342561, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1290, + "versionNonce": 1222168987, + "index": "aI", + "isDeleted": false, + "id": "NJgvtn8_Kzy1quxMqyfAr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 8.194007518174999, + "y": 231.4272063800404, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 216.25169372558594, + "height": 26.05968577236269, + "seed": 1311553179, + "groupIds": [ + "3JfeRMxXtVafxucZgxKNy" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1728889339478, + "link": null, + "locked": false, + "fontSize": 21.716404810302244, + "fontFamily": 3, + "text": "victorialogs:9428", + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "originalText": "victorialogs:9428", + "autoResize": true, + "lineHeight": 1.2 + }, + { + "id": "ijEBAhsESSoR3zLPouUVM", + "type": "arrow", + "x": 754.5486716336245, + "y": 263.63184005775634, + "width": 200.78701391878076, + "height": 0.03213913002196023, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aJ", + "roundness": { + "type": 2 + }, + "seed": 1284919637, + "version": 349, + "versionNonce": 186313781, + "isDeleted": false, + "boundElements": null, + "updated": 1728889427809, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -200.78701391878076, + -0.03213913002196023 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "RbVSa4PnOgAMtzoKb-DhW", + "focus": -0.0807019799085118, + "gap": 14.299953371905758, + "fixedPoint": null + }, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + } + ], + "appState": { + "gridSize": 20, + "gridStep": 5, + "gridModeEnabled": false, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} \ No newline at end of file diff --git a/docs/VictoriaLogs/vmalert_victorialogs.webp b/docs/VictoriaLogs/vmalert_victorialogs.webp new file mode 100644 index 0000000000000000000000000000000000000000..f199a054ea6486c9501b1c9cdbc3b5f0e76c50fe GIT binary patch literal 41810 zcmdqHbCjmdvM2hMZQJa!ZJS-TZM(Xv%eKuf+je!?wry8Uf8Rc{&$(yz+_m=1{p)6~ z%!tU0Hx>D2{4&?0Bq=5)RR92}i3%xvSKv^G1^@trf07L(01p@-DJ-lo0`x}&0FC^W zgEIW7v2}J-5*H#=*U%({*aiUoiGK$JBPaWRaQ`y@qxHD>58XN1f2r~RUI=Au;$-wk z<@!$|b^O!$pE05S;Ze>0!BhUl8~%gm`HOdRws-!cqwp8+sG=nNhd2JiQ=0u3-tfQh zM)r<>=|}$2;jyuH`KzzL!e6t4HL+Dy{zD=EBwTN zVA>5h3y9hl5+9fk3pQAY2$Vd41{7dGOS=`}#G`KX3_gg|m(!j95ghZ0@x?&j!%iL7 z&EY5PhxbnNVfQk6lQYYo;AizA@Hz2m_fFCD75e_;cIvWwyF1@c?@Rbo`O55AZ~E)o zN7%ji8^VaZ6MxtT>r2cd|CQd{SEFCg7w5CuL-jF1l-}~!m|yK<^{)J7PEz-dpDkan zAKWL(d;J^2Q}wCe^4H2&=99%2#4Sj_uo)$da8%Wut>|MTpV{T%niE|I@Gdx%(yOk>Iv_`)k5) z;0xjF=;P^_{w8LBYvrr~#(;`Ptbjs)FK&xf<6>c7Yd;T;D-2`26E)+cgJrT*-YoEqA> z4^FQ7hui&&HDE?@O3++9)WG`^y~;Owv|xZ&P%=C}ThthG?!urSG8fnh8UJCfBFVg$KMQX+OlsF#Mw|^KDxI>IvWKh=O2GI1D{p_ ze)W=oT~vN)HY9<97>D^}AX!U>z;Qa6e)VVzT9hAczk42pYFAjENdoLPmQUo;x%>G( z{*rP>f2jAnJI#;j)8u{p)Dp^+NFOuYtX>7&R~H4*rM2^K9U7&q`LB70EdAl@9M%v&vHA3Se}KbcGW;t7H}Vix{%>zhBVys+=fwD;x3iQF6%=SC2GiTZjFKKrP(UENDkOCGcnLU+uc=&PSM zx_fp#s`V|vOILB4KaXjCrpDm_>}3ov^%!b zWUVaYP+hYl3y%ou$sd0Z64<`)BG>8Rpa!x;8%BbDnFI$Q0)msG0is#G*FA}sHp{;DUKNofgL_ zlKL=6eu5aqKvMX?yL?_h6H1$x_UcH360PJLJU{xJL~nuR28hXfkj75}+!f=8rgi|3 zzUlF8=A0&;a)QVsSt<#n(5r>uw5<>p_oy8L%e{DtNPQlnDuyqw^izbarVH}}ygs6x z*_Aas*ed&ksOB4~Kcp3mcMHD&fyeO_prAy2^AmroKRQ(2TTT(68r$1-rFRE=XlK3c z-Dz|zR9E|xhT>qJBHFAKzCq0yYIw`lRlwbZkJ*np1LCJnJXM{jgTQZ6)dW*$K9dS1 zldlLrcyJdsGIy;WS5~2^8s2egiT7nmka&8+sA5TXanW`6q+6G87NUB0E4T+wc%vc& zmU0KG5;%oiHjBVYc^%}b$-o9Fg~RTdkFZOcqFc8)9cR<|;ug}!QCg$-kGywLNSFp1 z=5Ltc(t6vH&5?-uLqkOJ#hoF5qNfInfVNYj z^L;*2>&DNeamhs|Wi7tc>;C@2wDj1z%Jroc1rFpsN`Z92FZFL&r8NjWZ)x<#zLq;W zL>?9Z;+>%QniFZlcDfBPsG(&+z4Yi0SBHt?Q3{zUW6yJ1g;vdn2_jv2R&tqRd9eT% zLV1(m343X+u4gOHy%*eI=n7XTV+kjo0vic9r{a@SJEKfbUV7?n6u>;Ld*zehjSUsE z^W)#)-TzS(t>@F(V|Igg)l01}6*)0X6rjxGJo;H{tNDgLek)}1?7OIsa~%<-lliVR zeli*Ay(Cam95r@bSi>kK8Ds{DD$;$!gMSahRZ)0cEUn+`KT#T`xJzqJ_BfF!6Hkix z4Rms@2?jDHQ2H6mo8ExtgN2{BV$I3cz#{l;aSjr1*uj69O<^Z!GtTQj&McF*I*so) zt#ojCo>B^2=?P*OrCa{u525h#H&rkQ_lJX$$Df(?WjjYvmSFEY+IFIMyOjAh;D$J4 zD0=vZ#L0g1HwfBPtZ>P7UVk4VU5%FK=n4fw85fX>=x6Kvj`_-;gOEvHc?j2q2@JV||8mK$Qt`P3uv zAAExYnL^yc!b+rm;zCjckSf`xXMa~U8-EkgPL}|-sQS&W7jj4|J^@wTUlUAZ z`-mFAyk~8wc{I9Y07it&lZT^3!)*s7pdHh~l#w{1kS7!jjY|0+1c&{9Nzu`$m5uuFD-zWJTZ?TumW7-Ats@Y!$`y6l4F#FSA{w5d=4+R9PiRA07{#K`w zLO6Ba4Ub$!gli;lH=bb;zUpui(i)|GUigA%I1V@eHcdaIVc>D|TPc?G?RRRnpB|RI zv-`SAbwvRTghr={9&GKPx?B1(keIJU__9^K7|qis0U!h_1*472#E);l`U1lXl(#mn zkeyvUsfdgZQalMZov^oK9AMi>#Qz&a6)}?a*MN(y`MCT^U`AxT8_EEv*{ERT%yco0 zjvGXUepX8;(o=c_GXt=$+1x9HCHEb_p>ScWB=MlI$)lE3onO)1<{MViOZ_P5p>ol7XlDh9obZ8ucxrv`D$R9-y_1VZqxc1;`4`(c~ zxUpbAo=oSiIj!iKzT_f0vdbz8ZuIsn#(fI zGu&kfh4M-ruRK-J8P)$sb`A~jGAvkfFOD{#w}MguSFQ5nKbKUT%P@#zinnf%SQ#tt z{2BsmfX-HrE&`xzw)mALrJNlM%sU%?SYgIGj7ZbnOmR#n%{&SO(%l=oDPuGzJOw~h zp`@IUSjdh%im^<5ySGia)cTJj>fc)mN4Y%F>f~syA#`96pX#xDjaK4^GnF9dDt-Cm z(%~LVC#qjCX2yqmNFFl(6J-CZzw$rr`u{}aKgM}-X9fUQBm4+PXe&vQe!hP|`oHn} z|Ef$s2MxO9k|_mKvW|A}H`@O{YzXQlWW(QfzbyFU|FD67bpA^9PrvhTjRsO)8X#XF z7KP1XKJ(v6*EpHLP_~z0B%=CIwc}g4Y3x~t zg3fi+5OeYbd4_-LP}PxY>dA~RhXwJj?fH%KQ-1BxA?BWyVMze=NWUZo_*T#CV$Dn?zPLGCSZ9Cl$_;%$o{^4YmlKNT;~~AxrJw5CecBEZ!C| zrd&~hg%8EnWuiPSE#OE&-M0zVcojU8vQc|)NZ{j_GxiDc8}YadA;@f7Z3)&_%}rRzw>wh^6>~00QmgG z|NC9`A8&&|ri)c?rI~;}&M4&iBMv)df-_C=l;Q8N^+34g7&|_UJ<-@a#&29h2M*1a zmK?(U$`XwfAWWlCfzWl96emlYmYAa7Fylc&?eEtAzW1SE^ob_%etH*Sq{u95kl|_}UU$-%{i=9NH{uf0v{vk$Z@AG2zAf# z%VUiUA-}~h1T@@_q{P&}Fn2I(rTw-s*{680JvQLjITXuT-k4u1uQw*9cinl)&ch`) zN`AEfl(9Js^9Hj^;Vb8+o2>*v_kDGX!(P<{nrSF>MwF<#OP)L+%{LI#%_ZDWt-Ikw zyZDi_^M->a)a}lnnymILG`2$Q+cvR3I|JN3K5KBnIO*lqVTYO42_blEVZXs7OUW$a?4Jp{kand?BZJ4SCZX)~;fTZO5|Hz%d|+DTnI z5JQNbcvETy!`*Se<8knm0y!9qif}gpyjOr4IrhW2S6chvxFQbIL}y+XiM3tI@&wLAfqBPke=0<&Uz}MpFT1m2Ru6Z)lwRLcw zE^oINx%8jb;DY-dG(w`gqJif`s>%_0v!jO4Xy|`n@)l__>V?2ZB8W_1pM$31bD;?G z;r_#h+J8H$a--1gcOp1smg!Dw(vpGss(MOiefc{n2-U9uMnXvQc}Om_FUq8Yls*@J z&)StZR_&Yv2urL{QTWHR^EY6=%ik#nLal5<_xG;{Rq*2CJTu z3i{ODlD=!gyO&*3XYSk>XKH;$#|4`7o}8`Fv`?B2jAQfGesk<;jJVDRV!-)eOkvXZ zDH!xNW`<7;JxJiD@*Va(t#nK5BbZIv$t2bI z;Zh?Ww%XTSTq&H6R-B15N$-_oyFW^#U@!ad`;Z&E%kB*~A#1RGT`xXT-0rt3$~iN% zYd`esboR{gSCKnE<%4{0e#TPzul+BG`O3A;_KLL8i0@p?tZm1JKV3oXi+`RhLeKjU zJQmBiN@&;R{Ms+VoX_TWlWt@dRUK%QKUxjDQZ0aqzou1QNgl@|s+^c!`S~p2vtY2H z=CRSa6*0~|wSnoVpG`D5mxHvX#2cy0=Dw_t{bQG>Qo9rTz8BoL6lwG~h#iUed!`p| zkmYJkp!?vk>N@!gPoLK9rO7P+5O$~JxiO*g!6afovOOub?=&-HLA65UdmrumK9R-z zXuB$U^Z2ithFH8dbKn{7pbGD3-JAzfkIqH0Zwy|&38S&5(!bP1s2q{}>fDmpxSdh6 zawo?!lugFnmqp&y-6%)E{CW2j2M)*rE}dpNu^7od6M zm@(_Wcc>*nTxeUyU>lRlCk%Y-!+j34J2cks?fSI+Z zM_^W))k&OS@r+*63Vee=Bf!eAg5%FJ$1%hMAhnf`1KwwA&=YFND8tNvn-d19d&Wcn z?CnOkSa}e6qK^l1JH_euee$YK=Gr)~45lJ&GYM=uK@b+mkr?_ULS-VKOlo z5TA)EOfuVLr&6FJGAQrm70~di?o}$-rt-kS$?6o7EYt5TpaOQ%JdGJHN4H5S zTuJlL@H=~Dy5+NkVbiut0k0KAWjb*%<1xe<-1upa9vkQnoZK<`jPEEsd+_aDh_Rz7|$aAc5GT;WI3G=ey{p~ z2A`+O!2}2013|-GQ{U*(3T28|FX3f`sGV?L))F^-#BIi&xg%65rOk|2%tOy;F@sgQ zL&2_Im__0+jmX|%l^DDTNiJhBBGhA*%A(dhkB7h%#>H&vs9Zg!4_YWv*6?O0q$W3Q z#+(S$Z|u!?3B0*FXoi2as5v#Aglvq>cz3c?9w}?=54Z2LWz+aSQB^7y|e_>g^afqS0 z9?6TD!y>wc#LeshC`}kns7pXZo_tL($n?;~)i7xT#J^?xG`g-n_kywjGb(Q+QXn_^ zIllEjKf}CTXpp+5H)g{HNvB;LoXn&|jLv`YW-GIjuJa1K!iFV!W}xUuGq?Jbcj@i= zq$f*q11o2L>#uYGRVlnx;_N&wHAohke^-npjzk*XV2dVf6m>vICAoi_`pC)=777u# zSoEfdMaMx&F!+X@{A*8!J_3lhTLGmWyo`u!m3o}TWjT7Aomcy(tH=o~8PQ>SWVXMs zNQE$7y3qr*7|x;j#|x)Hh@ckkajZjlJme$|$4z9@sNV;nZI+e1s!auir6B(h(B6^X zwV?jQPczCRr=ND8<0%dP?>C)j#e1y#`7z>28tbt`y(XrEZe1-)1FkPwF~~AT3X4Kt z+D(VAU>A0zz-9%nY0LVwxUkFFP)oTYJlhPxNfnRpvHSyFdT6>cFJ|3-A?u(A%}>;trcz`+dGlvhA?Vtq^pKF39bL=a zT~gHbV0Vkq><(UMXL_rpp+G(43kj;De&h4=N0_XR4rM5OwZpLlvF?cUTCt=}LiD?wJ&Gv*BP z82#n*_mI|^RdxzFJ$qV4VZE}CmpZFGmr+FSwsPl!Zcubn%M=t!3v5-3_1Cdh_~umm zyUN#bts$Iz#Ek+p%qf4|^_hsA>KJC$-9;AfrcWZc3A991?~pAjJHU1LWa9Zr&vcJR zB~qO`bahTXh;h3E5qvTd?}kD+gGln@Y;IJJBsDIF`LDybuQ9cdP24U;+y3;HnpHOB zjuf!{M*9|f@P3RGa$r2(VJ;=#=;7lm+1%SMoPe0a74$~<{j@V<(r|0^DsRg$z({Nn zPQIej`QS#FyhcD4PH%9@h{9{^r&>1%JUS^9h{2WVFZT!!l+Y6oCfyY{3uChFws&at zr}z=biRXt5-6^Io!^V znxpmbWnM1Ql9c6;BoC!eO~`^34MLuS@9Oa<Ii=;&JXybz0s2 zl2kfnw5TgMzKQTpK)nVoE$5h?D}$uTExdOY3IERd7)j;(Nh~PWNh{_Xlt#B8Y2hDC z=_1t#!+_@F5rq%U^J8EqKDIi27R~{_<@@*(gWk8Jnav)a-d#IsfFTeagT{~~Q5^~r zM{*JQmaiTukn5RCP)3@GDGYcdN-t1)1gE1|*1N3=WCS2$e4~H>$vZ&X})AxNBB~K5H=RYDg$8m zD!FV|*S_<1xIcnt*h->r#IF#|+PHTw;WHVz<|~l-ni`B0hifAk_*pi$`w`N_=1U1W z+5&b*F~s=VbqaTep?m$=`?F$tM;%v}N%N<^C}|(!6in#TU`t7J8Nj7u{h#_AkSCR} z$f^##KGOVS&L5Y~_9{y%fKGFB#$sJqD39kRO!B@Tm$ivm`2s*x1|H|N=8|yjL;3)7 zDWg9CPtQnr&jwlAk#U0XdMDxHW6au7nAHFn9#kesGZPG3F*3TLwdFdUykQy`4GGnB z82AG~hMPr3=;E-GzTOcUhI#m*s+|+n(zk@`r?kek4OI?QBh-eaQv&@A=dkxc59M>^ zw!lpG%$#Q4sTv`qALvin};7UO^U5#?9n(O)-;EY zli_@-R?fo)Q-v*a@yW@DAU*qE_5A|a^hU#!+uLEUs@<o44@-JP$tD_V(VxcM7ip<@GIU$Og?D?EVH>*5Es21EwVGO>6KR(;zyxH zH|7ntsq;)eP9m{yNI+mmL(eF5LFrNBD$lK(#9)|)97l<{N^Tk!jBwVp;au6JyINkv zqBEP~^zvX2GdKyFJ=htyb@zRcpP)2J;&p3kD_s2A##@K&K->@LAOJqx@r-+Deg_(M zdYKaDd88kne8qu=3aDLa@C*0xSYV@2Ygo$a6qaMv_YAx#oH-o&Q*o?apx@3{)4<9` z0ExyrcUIJl$BBr();heNfQKkWBS4Y?!1Gpx7jo`QYKZ;I(Jq0X~m1`_@ zd5dNE&Ca#gS3=e zkR2xSw8jJo?fqN4pMNF+){9}98h*2YJ5fshl<~;4jK#8vA{dFM=uFt1)ww1)j;!+u zWw9=3`$1_$-nH;+O>V?<#?td+^bxKgnwYXI_4|~)6lb3gTl!(!)P449U2M;f=PeGd zkxnx@$`Dc!Y42W--rqk&kX8I6;5w=&qx8Q*V;u6~{KxpJqeTdV)_GkC9JSL84hVybJ24Z&_KnfWbh+a-g=y3bw%8bo>pV>$^^xNag{xzZ2;KaX`tH&) z9b)bTwD&o(WH%CS6v)F-#fR#dvJ7uQR3OQ1g)+CB>Y5GaX0V!Pug=SRG>7q=mmU{*h5yprEIOH%GgGDd_G^*n2iF3ZqsHsnLC5HGbyVh#BuP#fud6Yd=Cl4iBaKvH}*oTpC`R36cbuzb!`9lx5!^p8|y_3P$I@x z&r{%$^%UHd`xT&8xCl5=)sdPcx-+9i9fT>UF<9wJHPu~dUm z2A#hZKX;2vFFkLM(u);>og5qvP|*>w*{Z7JN>M0X-Hj0Cp5R=aq#3dC7J3<}8LW<% zzSKu4owu9wPk-AJ0o6F;A@&#&eX>~?bui8s&M$UE5k`uh4IneHcHJ?k-SgYzB_#v6|(`i_;D8K zzWkfaJMvYQ<-n9bWT2TSMUhk$VSP)#qjFRJ_k_->`-RclUf1{*fjmPrY!u0kiSy+^8 z>*;!O@a@(7ObiE+-i>~bUSsW2y3@mPmry24pz_!LKOa*KLhL}McR*ME`M4<$ETN{S zF;U*gFn0ptIX{FDPZcxh^=u5Ms|s_K4eX4wgp;RGnU-P!bIS!NU&_iDF5QvFq`wt8 z-3PZtygZ4UDU;wM_F)w41#50FJ0?pHRXC};w$2XYFZ4R>>W;!C*uk<37hSPP?moF^*@q;nz056Dyf4mxm#PTPw?C-xRjVA-JR!xjMt^^D zvF?dZffhK1=W|%8KIQ0Z0rtpu0C)%0Ca@Hrw-2AYb;3&%-^Tgrq*{#(K2qc!tSLK) zrxKLmH3-|ofA;+Xs@F*#9$Z$Dse~ivHqN*% z;GeC=?2Z((iDGST7SlwU*wJkIQpa;5s}$viRWcvV1ELBNipJpVp4lxJJL0&b&L5s$ zHd;~#GC}t;+Le<^5js!DA)ywpXcjaEO{FoTd?cz-tqD~Oo~nIj)2y@KV<^Lrmf5#( zuZ@v%ubBc#ljvCU-YwCJkXZn<8{`wc_|WYB&~*D&529=PLnF?& z@Df;f`gc#R;xFNH;KBl8$V6KuQKcvlY8-->jN`)^R@FYN(|Ao{fsu$Cb8$)(7V-Hk ziQFCR?diuYa3-AX?`)kLmv+E|`)k3No%A=7J!0~d3=qDZRD$jH02&zY+^^?Co7YSk=_s;t<}XXZyMA~OGwU8LVZzaSj@5FbU!k3PE9=AgrNHP5v> zjO+U8XihK=WUk=`Rq^eCyxrI&1^L@aPT~S?zrVO35zaD@@Q&`%rV*r!_Bs)U(y&~; zv}_mekr{vG=P%B3%ICSu)?9=WSam^g?c$8?N5?6dtvS|1!>S)=AIg;67wkQ+NJLiI z{?uma_S^a$M_i$kBQ~YH%=u}Q{Ayad*A+u)kHEK83WApUP^K^vZ~oI=rw4iv`%GzY zm?9%i1l_u1Cdg<;-kvjEGyyz@ob_So`>L}oQD@-v!V*5WA%s-|xfEWrkoukXV=v&|Jxn4}he_Vs;7gb(Sw$o9a0^Y)1#^7qx|h_)=#V3JV*3 z1=s7fm0+my+R9-^3ZuCIh?t9P5uOJwwZimIa@nBX)9b3RN-q0mZe}eNU~*Lv`I!Oh zcsb3TIR0n+Z)9V;N7Ob|RP5}HS6Bt>SP6U5A7U6CwF0?ovW4cnImNt#_+2W}~WUxn-n3HYj7u zYsF(3d;&UnkzBI?#aI?6mo4iG8dK|Ov=6XuCUyNSAjS3%1RXX=Z&v$HM!=rtQGJwN zlM1t)CVq+J0+-K@AE_K}>lb_M4>Q6AE`I1nlEKlF!uQTV9Rf>%pqc`Yi@&elEx511 z3mIk1>ESYp=Ew2u-;1PNsmCBmj`WgQb7E-rXCO6l2RPa)$CnY7*$p5t zK!kgeVfKd$4d+Z9GW31&rgZIZHI`Yx&H$&5U{@7S*+#{7Ji{`Z9?@wXcX-Smhg_z2 zznv$JQcp6b80XmkuAYtt#acA;D}`w-DOb#yg^93jS5w~2ZdSQpXP;IXDw5=XtrK+2 z`TS0+jx>*4GBCMj_zotq?Efq-S-=j=M88k~LFrD7)W+;?`?GH1_{3Kb2PCRIn{DLQ z*`n5|PmHIBxWILDs>5)N9`T;9jk=B-1dJG{fDnaj{O#2Di&&C;I~?abBP3 zTr0}vBw>KAu9JzMg6B`FPW7gPTH2k1B6BxPQ`yc7K1ioUOe+9O-JG)f1aa%3nI%s; zqN0MMWs&MJUDrqWf~J&lpaC%-4pgAUV%N(J%Z2;Mz*EQR)}dl)^K!}*SZ%*(biRXz zE4&(m6IqfI#6JZg)`?ADp7Zu17e0@qr3#qgS+qWnaH8OzC$Uwd9b^!;wWX?kY;SyL zzYK#_3Oh<*@(VTWvlZZ7mXhB$B^C*I@2_lwY+6;TmrQWIYkoyE+Y0J3Nxq6v|2+o^ z@;WNwIMq*JE=wY%1^RDaO&7!%%fUWo&ifwo?L9v z7aeYFxZ_~v-|G)co~Guubv4jfH(ZXnMDOpLotA4iB!py*qg^&WKFhyuxq)EXEoSSz zp*u$!YRjD-&PDq{On2x%p8w%W`*BL!9hc{w<_8KdsL$IVjCM{m5Z=rMS5CP{O6~y* z`xWkj4&TQ+i-gKOtdc^|8-x|4jSt=P@*7F{D`2p zAzFk=*^j+dY+>aw7tQ6}pwxAzz)To1NvXHRNd$rS#um1q*uV_k+vCugLF^HoH*-;& zr%th?AmH=97i{kXlFy5P>)_x*%;i-j&Hd^Q9VzX zCgq{Nsx{I=W8dU~?;>^s(8WZ6m1`($uWMy@F zXS+O8tGC?RbJ1cHZR}|zpn#=|5Y0_`tz&hok?`79nDzIw3+nTBaQyX^Hr6b0S)i`A zD9o7nXT%vP>54rXWjdcJid+_xcvDGq>@4{d!+tmYF~#p%?Daxj;bOJ{>}`r{aa=jM z*o~hpy@=4T+A3v-QAX?wS2lk72d}f@S(Y#s!>(Bzk@SZoh3@?$piumMtzRhN(UW6ONn-Pwq#DYr zkJm$`{yt5OxPpd64(?yY^*^V~q2-2Pn7Ou%wvCynH6>=D1+rSdi7tHlo>+0LuU)MQ z1+jJmgAZ$v>&z&pVZEd$?i*VodIq`V$S|A1-wibG4tZZJ)p(b|H%m0S&|F+81&6o* zqJ&8c1)7=nGS<;#-}L;=AdY^e3or%Y1b~QmEU*{GkjfnEIBt&XS7_u(nvX|>s0Ie} zwwhPoC6-QBOVLaY{d4i5o>@EHe?L2HNWViV_|G5y{ePPmitty7mBa z=QK21*rklce#6}lPQh4%McV)Q8iJTpS>Ict!Ga8O9jRh?=s-r*>(-Nl$u$1p`%$=V zqJFShUK!BfZh+Ul2=bRZ7$;~gs-!o(Uqqq{ zKK;udZZ-v4&{@NMCD+@Jo3Fce04C$2xFL=gn^c66&rGW$c?gUAHH{pV$3wn)qzbOnea#`w^>QL= zT(aA?2>dsWs1U4{am4TONIqDNi;+)in@j2?{#zqd7G`B*ob}w3LrLG-Dwa%A_19iB)3sA zO%*{eos+YzcJiT05?mt}Ilhu!?;d{{Od54>_Z-7A7J!0gj;`=jIjE-4u0oW_?fKEF zG7}pdaf{qTZu3qFKq+5)U<3K_t!19yw`^df@xfuu_TyU9FnB44u^M6zzG^%$t0d`n12{fZMv}?bbggI<)OPL1 zF3@pMlNe_N0&hv5X^GwmPuptaxv*H&mJO}&Nv$YEeVfMDURYSEC&#YYXHv9Y z3(Q(qZqV}yLr0HmC#urJ)MjTK*!#Y#q1a-HBh$61#KJZeD{j-57n5Zn@Y@trjDUl3 z#g*sV=6;g#H>^4?)=nk&@Da44eeACIB51{Zr*}Oqq#ws~3YFfLY>iMfY1yx?)D(_R zpl}&_<%sh`A1JU;-YCRI!V!871`uG+=}z;C%ZuCXqMbOXQJWT_8seV%#?Y;3`{+Ml zMrESI%_Aofoj4zl5qkT&8F3ka(Ii$5=)boLCK1cJIrGdpKZrJ+_?RJfsID#YWvue< zCoqOyQqS$T^q5yU_((VkxhMmx;xPw8^`hH89F#CA&<|ns WU<91mGN3@_?TgD1 z>bVVO#6a1~LfQxuuHW^gP*<^(NzI+p3D=|tr1ex#VqT_XBF-RuRlCszCOVeh*@8|^ z;g&3WVmcgUteu=w;XRP`Y!W6v9JsG368(8^RtrtYW6PO9TAIP=>_Q*Dsl5*m=Pi(T zpe>n9m8d}BU4rs@LvN4*s_1$oCXzt9A!lez;OlM?XxHVERP~@5=hc*>u6}fZioie6 ze~i93Aa=LTirNj8%J|rCfX32JzGYTtmF1jX5-i^!jCbc{OW(#8FS@HQNSw+)9Zn6X zi8rF6FZ(^wkxt6g#B0f%h?0jA&t*cb2F~t8PEzK(2%%EN#eGs18i4Ds5@ACMRaIq#|Q@{Y=U>6 z{#&sE%G~<5uQtw39~A3HGTK92^}Pr_0bQHF`}cY=k*yPou+%?j)GeCocgBfo-2|41 z!XWI8FCpKRGeg>rh-T=J>=d27^aFJkZ-H@z9cB}_xMP)0deDn%+6za7emEzr%8Z3{ zD+uUMLRo#IdrPPNXuY?DL^nQ`yUx4Dz>)g(SsI3xGVO_XlN4~q9{Pf&w=Mtlfk|o_ zPb*8w>p-5*HZ6$}f6;6kqRLTkyF;Fh&IPu2IgZera(kyesH1~IU#jB02kz|kEs6(P*JojZ~}Q_8D*56_SL9| z)d&}Z#3`TcK0vo?-sQZMRAG_-_%gq%G{3(KNdbW5Qc)-l-6-8bq0=+_3KA`~B3l7iF9dhXBdWrBa|}5uVN$HfBNRF&cDorFrSs z4=dxuZwt;BFOVvdrb52BMJZm@{-lVCKsqBeb^nzRVs6RtypwL(kK&$tF0Tz37Q1%; z&ecJSy8JD@p5$Vxnken20C82V^Nf;(J`3=*#RK`YKIH8vmHzQwWy*y$b&W=e`>@NX z>g-)*jC3)9OKfO-y+Lca*+2d$hwi*{HcpP1U?Z0w@-$a9s2YE?1}?b}13!0|!D93C zvpBg+x|$~L$fn}ssP-A$WIJcqOZg{HG@1Ocv^PeQm>l^+M)Rb7}V06fyu?Zg;wm7of3sGOP-@28YX zpPET_o;~RajL>9@Y=$My&ge`D2S^M~Abg#BHmYVmOhE~f&Bkm*ho_pgvft_um0QH? z`R2SjO_HxC`r06_6@TMpz}tdE=C8*)2gu{G?Rue`sJYD{X$Gq!7}QnDiTd$^S_se> z;F1KX$C|;jNKe^ph7Q~EpQQ;vx44S$ z^!N0qNx$Cc95NM-!Sa!crim#l*uiI#{Jsl8wAG#@st~VMX$LcBD^_^0KwY|Fn`!q0 zJKW!N+~Gua)RnvvW`m&JXsrsVN9*)Y;-7wZ4%7P{Qz+2|69<7HeK08Y!ymy_cMzHRm?) zo-(wj(3@V*G_e=;EpZ`1h?VfQ(cwlku@+G$?BeSjka(DAE{bl49(n93bhZ~jk%k+S z5lXeFOcB(s-Pj^FUVUoEAIki?r5hPz8kLR1elgCa5D~~zxn65LyCqn;66=R8B-{1M z2yIcR@UR~?r6f{*G+U%V@FxohVQgyGl$w*QzjRs3g0E8GqU#-6H;D$2AeR?B({dB7kL^=y@y^723t8k{>3mfyraZzowNxelwr zaE99>^hb?Fx=1{DX%MHM8U-?@ADeVCEx#;QiUBx3=`KV<)Jx^7)6!f+lxMXx5cw01 z=30=lVppSdtE#6wQg_7MFOXn3Y!poke)Sx?c!%FIHHFL%Kf%&X@=rg#EPjbGNvdZf zLL>$nYbklJE5l=947IL1;58h=ZbVx3HP1>&iBNG>ax0zf2{Mi|cF$EhLT+X6H%>@h zvEN=voSHfq4v1OAONYPy@@|TjBB1jJg|2t;T%L_B^Q3V5o$1X&N{#n4>KldF&UT^* zO^MMCA)S?8@kT3?;oDEnNshvSC#-Wj7C5)L3Zj5=P?TjB-gnmlAzo20&3F<{MQBY7 zVlcI9y$tw{G;k9*Sv}sJG-mGC)BWfhJEfE~R;fSURR28^-t?Lw4TQvvbmHd(1r4%o6V)g~ z89B*~h|z$cL_x*f{qAUjnCj!fn}hz1P!1?lYVS>X3#XDRXIy*@(!R-v`h6iVTNH zaChEpDmbbu9j_vu(Ju@tP*Q&48%UO4GP$KyrU;|o4Z#Ym_~KfbHh>h!sb=X;H0WnS zQ_p_M9SoVeqzgvy=LRj3G-v=zk~H%+z|-CCV14Ht*R2Tk*BL|3N+!c?SEt}}mGcPF z#lKuE7S)47T7N-@__i(w58}goG`(Oq64G|}VVIIuf1}II!Wkg?Dkg$thnY?eDjYpD zXoi7U1$lK%CB~V>>kwTnSbOciKt;O*sq*44Lf15Dq?CoFki;Y^4J)VR^`nula1VmI zBK@uBNJP&#N}hernZDTa^8I{~r(IOUOP_V69kSvSe#2*t*z?E{&BQGNpF}D5|O|)u<4DYK( zITX-x0)YdDhzjDE5il8MsC%*W^fTn7_;BAAnmX{Zv_exLrfPE>B$%&NnU+#ojg)=K zq_OKw)G-y_BT7Vf=Nl$OtxYc@E2G2%Rw?daAROpvlu8-e5ByEhL}M8X#Nh{K{pOq~ z5ngM(aL8~ce-YFpHmiqUG)}}kwCItW;)u-sZ`AO2_I5dxziBB@Q6Rlq5K&zATfE5@ z+;1(bne*92qCGy*dM|lq%hc;lF+4XZBMPc(c+=n|#E&SB1UOEtYQdM~(fJ@rvSN@f zbGCNCIBj;6$QACQVU&EhUzHBw>bFzy#Tm*B-k$`M7Y4Ak=pRqx$aZD5i!2-Sw_>K7 z-PISr+yj>I@@s@=-$|sTmNn4ohsN2!+@FgttZFki7za*zh+h*GMiw#g0wLq*U$9hS z?@{x@ut+&5)W7vNAAFb1miWjekCyl;bV4IY%7VKbn!{17k!w6h5*CA~U{>d5YAD-`w z#ht$*v6^9=!^dVYlf62ga;L#J3KEEmaA}zH8_tZrHRxLtxbi<}Kw=lXqq#Bh+{=DEJw3~?|cm=^Q zqXe)N;aL9;nUGv2k(tQiO*$v#a~BxjZR8s+rC(d#i|z^6*f9)=R*Z+r+jdw@FL;iX5RwrJi-`BPQ4-j${`qlX z^IB9gyjS*EX$(Bt2Z7-(rs-0Lj?En;^O$+XG~5JJaZJ4xR+*a&8!Syxn?ZBj+7VF? z&nbRKGNM_+RHrDNzG`s(rkYTefR&h;%V5Hqtl0aSAo<2WsTK zX`G&-2+~`tp=Vt^#2?gPdp%&Y`!hZRY=_em9)+-du9|^l)=lS^xb*rvq{%`i;R{YP zhY(iOLH(N9-pf8h7`GRsxtctKk2!P&_G3!~@kXM{M?A@r6bgYU$FV zG_WWWq2iZT)u?6CJ*7HqoK(3bihZYI_vRzz2rI_fAml};Ba?}+-|`?YQBP1wF4^?2 z!pt~ixJ19SqwS@59H?8Qoeh8gj6u;PwJd+1z}LuD=zc7Gw1eFmFU~5jDX~bG%%zC~ zpuf_)jv@Y(6o~tYQ=56yl~TBb$wr8_do?Yn+V!At?m_mErVAE7Oe1uRv6!>V;N47{ zt*x)s@yTC(#p+$dZa+`mhN!CW9+NZpQ$7dOAjt7Ban9zG0{wpgM?kp07Y$e(5!dCS zH5xbL(7R~^27QM{AHRZfn$-Mx z^QS9=uR;gKht1Ew3O-oEE_N%D4`NUdFobT9;3*P7E4^}NJhFUKr29nqc=C|S+?t%D z*T(p=LNPLj2#ExDD1dCHkvylNHeadsSQy2L3i4Kh(f(S8WW6t>zGa za|%Pj`tkUGs0{o*qw@haf~(3K#(H3Rjy~ZaM%4w>ymHUTw+(3tsbHbaph9um4GiM;QxCS zrv;KI04!YOu_2P3!TteATTEN6zy?v2ZVD{Mi&DjhK`n_|vW>tEOuNyIU#0DkKIXQ2gKOGWG4=MI9wd6%HnFUBoN$jb8jc0I-a(|t)JBzNsu zjkM+PHxW1>{glr0SUo($v0o_3X77v~=52a7*tScyXLQ4XwFdq$<4UN9=rbQyEzYl# z3@)=T>euyk>La1=?*hDn6$9sIlWeaI>Cd=Ud&xLlM=U-A2)+D{kT;~8Aq%bgV-r~M zYMRPHO|ly(%CrHfZ+dBm#|1aedUL}mLLpjPs2^=u1g6P4=r_YRpC7kTezMR8og=$5 zmEUv&-gmYem#qz;U1KX5ME4ycX2Y`Eu6YVN;b@=IUAZ0Xxt1NEtz8W<3|N2?p5s3; z7{IEqn?UVn4bjTTV#l#g$0B}(U%(6WaUK=iaA6!u*)c3zLd@$#U=81NOFV$CMf4c} zpmYv-WISP!OT}*wv9LHxQI$Dsjr28z}6Xzh=qz2wEItz({3E!3fZ z&wnN0>pxx5kf3{nF!w!)7f2R`dRaJL>_e=j>ly>oxONZ=FrIpY3u0>uj=jz4$9UrX z8f8ViM9On)=_b`>>98d|R$+Z`_RZg_%5uti=DcIz44{#~q+!pbNw%!qjf@e*sU_{< zxi^$<4h~l)U-CXFd=@?x&f?#zdO*&~^2nty38IDAY_94jS>VsQ4`6OzXfC0IF$o`r zAZQ7Ni~Avki}nJU%gCCcPs3z9)$ApZS=rS6rzL7 z(UZ{I>*?vE+i%Y#78CxUEe;q@%yINkT&t1gC&0__f(}1328C#UNJGWSnEPSpVCJoP zqK#6fma$b7z|QJa>Cwnlfr`}Z5oTi!@A|R>(@e+s=-he?ItTnHqW$O_7z@fAn;Zro zh&>+ebO*(GrcPYFcSXRin#95M6~;}AS<>v&KXG z$gTt1PoumJK<5nm^IXDFi*D86IHew)i9!}YjWwT1a?@sVe%-PFSbr)Q4w=3t6aN+^ ztW&;#eYvK`uOQx;6G7tGoQftk7q0w%BiEv#$EV zH)r}EpSw7q|C|zhG6bZlhYvdgU4_hkHa$v7<^Ld8>`ZnpOouSJ#gvz;HM-z~K+&z? zu!~{GM4Su}+*q+FO}b`2T3QFr>eKXm_1Rc7`a4HEW{}gAi%-D#^GYLfH*v2*U_lYW z;^i$84FzAe0%0J4X9{diO;RKi$*NGxm!+lNuG#zV2XNwwW}v@#8_H?nnCDo60PhIK zJ`e@*d7ddxi5yEBlR#tcmT0#EZ9n&sZ7I#)qJJ0uu-zjt;P>ectdLjZ)8k$4Hpqu! z3VIj%;MI^5E201H@)5=?44VSA`9y~3dNNIiMjYiQ5AIVQL(dyk>AzVJy~wUTHN1_ z9y?;NG1c;TYJSD5^sU9wS;+`y&#?(dmAzkQsaos&U}1(#Z5Bx&nBe~eDWp?B2p;K) zO3V4c0oz3mB7{R%8f3Y^<6@PMhx4WKLm45_`l#DHSS;d*2P-u61VgI_AP<9R_+1CS zzfM(Je`!$h%il{2XiGI2SPo@0^i1PRF{GB&Hh&WjYdXL-?X+W}5rQs5F|XvfCW z(b#=@oo@{*4@^i}amJMEvS9A2&lxcM`8k=b++C(N5hT=;JP-7qrDJ^IRrX79odGVe)V!0n(l9n z6=We6M-WEV0M?hx1S_M7gpGoBYuS5_^!mhu;g$7QeN=qnKv$Z+Q6T@onQI(Gx}?GA z(`*}<)h*`pV*4GUb5R_fd$j4Vfox^07AsEwcj5j#5McftVL0805cIVL{NtR*YwI3o z>MU0Rz+r)p!4XxPfBMIf0U(5T3qt;`7oT@ZUYv)k{=L}5oB)`6 z0ikpCIeQ=U!}NUnfM@Gx{~nW_k}>A>WD*sA_65h+*@6QPU^7uk6FU}m^L{yft=Qj~ z!K~N;Og^P|>*MYy*{wJJzfgcf7?V6?{ce@vg=&WB>zopxiro?qf0+#jMn9I>VV^w1gO80D_wIhBsB&nmS;z?saTFHK$%UGXf{`tEPQ8d6T zssBrChxka?a=6f)ebLN@wvBxz*X?`d2eAN+JKUxog~e3>AxHzI$e=n)x6?1$KM3uS(tB{L1xR0EP;BL0$XR6zBn*S8tFZew(L$6(zR|?`FWl=cm3P1B z;_B=;+Q2J{*&DH-Z6cWP3E;iw8z1g{Tw6!{tS8y3JFU(<`Gm=fmn>vVQ-E%R38Ivs zO`j>wYA*R)+uFqoN%qHb<&53NGcxaI1o2=!(Xf*&fU!aBHHUc77=M&Bkb`2mrjP=J zmkh1erTacxIJnsHLMVL%&Bx}um&SuEWP`kt5|_=(6l`P zsO^0Sm@C9>((DWGeakwFmRS5|?h#^tumEvqdkXS^Kk(h+1xN2+%RTjEOCcuhVZ;NM z=k7;|EqlM+ivfAiK5c}sGe8}c(w}EuSK;i%KzS$h9^Jf65o1Gf^zTi||ER$K!E}l5 zIb|7Usih8x`paJJ>aS6Q5m|oL04=7G4%i{a5x-(za!Haw%)Veoio>7F)uV25qD?U~PY3(1P{|q3v{J&;*3fazQ>rP(-@ca?SE>Wkc|0@g@ zQ1+{d4Ziw69k0EtVt!YUE#NhV>;aPf8;4oHIiV*!SW4mP@#x)tkK?0^rG2XfFg$Q|si&e-X>suX{dNYEp2zXgI^T9R&CoFh@)M z0e13MW*EUsu&3q$KQ+8g^i$n@=X=dis$)}J zB&g$~HGsGANoDjOKc2tg_}*c{(S+a$4UJ*^;NxTqFA487m%of(u!2#dwqqZJlz@Wg zoRqq0NiZ)K@jXCL2mo}VMQk~X>Q{4sB%ffz2{R_L)8Tia^~xzR9etwATy!U9J_{~! z@7-CWj{wo(*WVd{x|{vquLak~nX*Nefmy9V$bm>S)*n zQl%a3)tsGb0k_5}-uRv&>4{8p=lu;fXEdjz=XFhWzPc+C!OBW7?hFiqB}5&zcF3bc zU||%L_2`l30Z|vjl~*!L0s_=L_C#pejfMmz%Yijv*>+#kW6o|OedERPy>wb9NG zj&1lFcGDA*T(?f>{^BWP3&0!9rDdj00x=U4xTL*rQ;t+-BZjSAlWT{-bg5epf#paoei0Jyhnuu&~$J;DWc_Shm{ooY)u;UawXWeh7jO%sx}Oxx#ad6xx$Xq3UiZnejggsEgwWgKU4YQY|ZH5k=w5gAx# zkzn7go$l6PR`%kr6cRbIfO{Q)V5Y_wt-_foH!$mEH$yZfsZMr2wbk&ILA<~RK-8d7 z%*H%_@pH7l=C#*Z4jEnjF`2H)N;6UOROae+Awh$OZ)U;^D9+5Qai(1fmeMow_8A|b zm8j{Kfu(O~QMAhsLIla?N6C;dTZOC&-RtgQ{Cza~fv*}~?HGykxZG@r-_NA|&$R@1wzG@MZh9?{&FE#z3rbD-Zg}57I+nQnviNWxQXDg5q)Wj#-6)G6Iqa28lWLsrekZ!R<=P>T{j&HS^y<-=&Q3Lj?yflYrSg@y2_4pjFr z!+otoS}P-Q)B&F#0&@p1a3B3P2$X|0>pQ(KUi5x;LR{rxgIj$_xu;JYf!qDLh}dhn z&iX-N2WyU!!Ilc~?VHJ4aH?&6OI!pG>c@dbB(D?h^t{)d^-m?x74g z)qgL^PursZ_iJ1II!CKvcNIKc=L~XpBh51Q9c4GSygCmjz@joHi9;$))!PMP6`s=k zfutIaTXfxKax(+Kq5hebF&r2N6I`HzY((^V#drHDTQGCNqVudAx%_4Se&Pa;0hB(% z9UgPc(|`rCP?r5|tS~=dC3}gp;PN?dW@^xPO&Fs1p7SpwE0y|Lr!GQ^RI+RGc5ndm zy-ty`V($$nFcuIhBLGC3q8Qs;88WSqQ()1>%iovrp9tdT{Xg>L1)x@N&64K4k(RN6(H)NWV_ zrfDO;935(>SBV`k>xqU{Y=si==Q*g`)HiUw%)Gjb3IM&4?e0gPwncI3YK@`1t57fI z*DNc8&rPcpnT?=SC^?BWhQ#a=6B1DR>+rk#QLG@i9@LmYR`0>EjUoV}(2cjwB6|>c zaE)!~(EiAC!tgGuq9FYwX* zI&NI5+b;XA#|;(@%OxYO@PRe71iHOm5{3_1MAEiDSEM(D*}Ob6P>Q+QJ?M=IkEvBN zMGy&j-J$)9P3u~DWhxL{Ct#1$fkUhTYKMpOOwd2bsA4Y*OVd2|d_sO8upJij4Ls-1 z7{yF%Su+}>xJ+3ZG5PXa2va$D*`^78pkQw*C{(Fkq*H5?-_rOe@vB0q~rmYtNue8Op%!p7Kd0~N5}YXT^baY+Rmsa zNKn8%FUtZqAU&!~Aj2M0YGWYp;2XJtku9 z+7pf(?@!(Bi@MF0P*m~2msx>VXkqEQIgM)_$L zid`)CI)MG6_?I$bniFkGtCo3(<37tYk~`>ldhz3g*W zxb$3c0B{9zs~$l#Huq2dQ9lPOA*4Ur2sHbANSz$`pqa>hgmr2Q*2*I^^ezUmEX%p} z%m_G4Jw+)ic^8mOfCQj?ZM(hIBs6B8`~M#)PD26wN*`hX1!lkRnTz?}Z;RygGuqr{ zzp(a(O+)*1M{7y?*OW_c`!MYoXtR>Wu+L)3zJL(wywy)@Q7Mk@#Cfg{>O;0BGgPBI z(}NBo3bIyTv6PFH%7bo6r0bNM_#xo5=H_UOpVVx=zKg7xZT>)n|2yNO=_cxU^+1&F zewGC0At)S_D=qX( z3rs`QyVZKXj{>uuN6HE!#ENY=unx+$%6O@gLr^%v$)a6&@0Rr=aD9G+ znfT&`a%9@4Xrpo$3uj?5+-l1$che$q>!juCW#Phm!5l#0(aPS!9V_{w{EnSXC;*T8 z?}pKS22b|0Ss|r?z9~1?;gQ?nX-Pm1Fg~;bhHg^O@$Rl)ik(mz&z{b!xNaOa?bvp~ zgv6tB_CbQ+0lU#%&^Jg>^n450_aw6p{h>s@P0}L#tLl!Y&492_FiEKAc~F4_B{_1n zd&x9?sB^X>IPBrF%_V{zpqX_CKLPP|>hHLyUPdbTP{P=x5R8ixxo!b`evYe5gT++? z0OyX2^cs35%9(}1*l)fKftWH+Sw35n*X+M5X4RFTZspAxXDp4lXhJOW6!VJe(c#27 zbuuGSz-&t5UVqj|S6NPmxzj*dKUX6Vmp<~dH+}ciXBz}fC4oAe!K=P|2jM})Oz?zv zXdEl=)*Q7!kKnZIc zxUWy}Q@T>-X-ad&HuoSfHwtg6QTjbppbp=9aiLJsRx805I!1ng52s^iM95$fz^kyb z)%4%;YsQTpllXF(*)}9tLG7LJ>_^g)yYCbB?Nd7r-?vyb-v;*5$Z&VQuyr?Ic z?>@hUd*U~Lmn2xW%ZN8gkB|a(hTi6NJ06m09DqoP$yLdb=zGUf%~c?c4$KpRes8kn z3e3X)mVqAbmv69nW2-o$^HrZI@fGJTx3qUsJ4v^;lJ&`Oc;wUjzCWO2;%S*hsNOe^ zS_ma3)*wsS?zrAC$U+cw^xBOyYJpDZXg``)(Y+l=De`&?4XIHNx4@|ZMN)dDDa&^* z8ZNAHrraqPl>0AgRv7dIFkJ&II;%Fa@q(7P00oT}A5Pl-7K}@<-M}GWS$Wc=*Py}! z7tGs;SbN^4N04xL``SmU8$^0lw!c++VPr7vPBt!(>cCEvm}@WPKzp4&g1oputgA{1xI~BM_|g*+ z3nSpfLl65ZtCJacEGL$zuR}(*;Jv6@nDUV1{<0{;F|mv_N7m>OH!dnxWFKF>v?2*)z_NRTI5Nd) zi(tVtb+uz3WRewT**F&@s=?CK|x218P&!*>$T44%sKX0GT4FDHnZ`KS|1+ z_;H+oP;-%A(hPxr8Mrf46#{%wN~YO;`=h4rLT|z8!R6_#CbYKNA8qWIzc=QOLK~I| zfz*pZ{=xLXP_~Ut=&xBmS!;v7%XO>`aI#by#5;z-}@W&l#@aD$q-!t+Qyj7XmTK4Fv-H@e)2|a3qLX zc5rp8DIuWPY`t10x`csF3KE36^j^E8A6uB5p+RAujdQZx=Ap@M@t zgpJm2vaV%lIe)6<5L+QqnWJm~aswb2Wcrk2FdrQ$!=`QC(ojR`#@m|RnpS8M;$dB$ zI(DEVhCWzh*)n=99Lw?QdyHdY8Nj85_p@Yg$dPkcM&8PCHb<8M;@GpQcsID}iH*7% zUI!P=Pk;c(P#xQb=RnUaQMi4KW_&m-ae4d=D4BY*#SaH;Tjyl;hoiQm$YnE{Ln}|V zg)E|z6I2!k{qi$=iqk&Rh0x3L8XJZYW+-SehEnICS>?X=x&Rjm*R+R$AtWQoIN7X# z8GhQK*y`Irja|n~@B~XW8x{!I_U{P)6?p{TfWD1g)IYbanVSD1?DZ^>6HI~dXw4Cy z&hwW`JJ;GnlTX=|?z`eyk+R6cM9xQ$xerT%Nu}LQi3HXu$S<1G4&b>@v7SsIGq30* zMCT8>T5HTKGKuc)R<*MM6G_3Q)u0a0xwT6L7@Nw5|F$XW$CZbOO zGHezwOf7X4yQ%}$FE{ixwB4M5!ho$O?vGE}8$lB+sqxS+EskmJzGrTU50a@kO8 zbzAmZJpy$+wsjR9olY{|;*qc*Mcr?*b?(Jscezytg$BbDe>veF49@Jw^`m;=6_=7AlF z-t!_TvQhNZ5WsBo6goBq$D?~wd6^3<4bY#8m?xaM56*c2h(Iul-cqh_p>p>}1bswL zoT8L?9L4{%B#N0!6cltqOrSzsn{(TN&`bVBBU;rzHZI&suX2j}`U(qxa5F7ib_6PQ zt?ak*7DD>@?NrgK9LFLzNA`Fv!fc={alKvb!YldTc-vpA5__GpHNP_({{zWC1EEj< zlbRwTdQkWdX{T>cW=V4(mN`QZU@?D1AGk@;!@1gH;~CwjoX(KsXj~%%cEo@m;hoO# z66l-d(MFQB5=~r3hwgrk->73qq0^{T);3l%zCciVvX9lk5%{LS$9mT*7R$oC9EG6!WIZo2R>?M)?kLHH);M%N(+ zY}0M$W^P5|j@Zi7-;sVS$F0DLC>km0M=KkdlIO^n!13kqb@Mz}`Z`&oy2gmsQk}$P z{(3^sU{J1FnzyqzV`)orkDabLF-j=d>(^~d^&N4mQqMaIv_RWJw(tH4M?Dt(4MpG% zTJx2w#nxWcpm$$%0b&@bUxIRc_gL#Xnka`MoB#!JDoD&=7+e?r(ICAaQytlMVi9&r_Kfwej z`J6l=a8-bfuPhX6kht;%#y+EqhaL zj0QsyHg#c^@p-7u(O)<>@uYU;tfVKWL1l zY)#_i42CZM;xtikdkIymjm|2d!`2z=;1A1JILYIyUR&tXiPpPYNZ;{J1pgH&F~cVT zHNUT4SmNPn%Nay$XfO#vqtLBG&BL7y77$I9%OPJQ*!xvl#&R}dg+u$}5c}fP2QU<) zi@v4)Vz%Wk$~pYca*Qv2pv^8*sD}pFB3?{yMpG7yJ_WkmKAgMW8T#1|?1hD=30M?6 z8Aq?TRJ!C%fy2&2cg;1sN6B0=tjIvWy&{^mMRd44gebh~TG?h<)Lp>3yy}!60 zmcCE*3v$sV*>W{Q>02|&qTk{y&0S4v96)RD$9;4}eU<={5O6C(G_6#29arA@t$0$_ z)?uFT&gy;phbZ~kpVFvd>5X9{+`oY}!Ovj8ALM+luY!e85%gVo5iG#Bn1ax|_MQ%|;{Q~hCD7Yl z@QMSq?y-gbhbB~TR0};H9YCc@aE_g(OT-bii?BY_sF0S?LE}`s?vfm|^E?E|=_PLH ztO`gAOMvh?&?|aSyYit+b=(ux+#wuxr4l)>R-vX|x$3kSy}AWt;V^ER5~r!@uToYK zHodtliqh^49Q#TPzn`Z@BbU^W!TLmOEFXpa!hLjoKzdzFCrbbdOzMFxG@@xA3<#hw zrT5~RaZ+Qx>G!EgYwXAhRYuEY-h>Q{3im>eF{AKDR5V$BlfXCdOtzTGi=cP<7TU9R zq5{Q{D9l?gIUs$n;&8kDp=uS=|H6OndqF>fNB%oP;Sk@AAfJd2{mtZ{m4^YxX=z%E z{H0Z_O4Ekig*(Mko}pG#)-8i$ms^&zTBj~7^p#y1F z{z8Y~o*f>BkwEA^03y3=74M=y&I{3!`}Ikz&P0O<8f#DFTWNlV%J$F+G{dKOxDnbn z9Gx_q{4jOrV^n;>;c<-mpSLUOZfP_S5IPTbp4N2}7u5BO*z0h-xWB8Tdh1zS*ylR- z=Y;!)A%V*@5l0}6Rct1X0|6Mqp7dnRUYosJ1s)R5)xLL$r2;LMQg_Q^x->W{I>?GB z`d3sFn)M-m=*Z=W)19TKQ=ylAmktuYE6iOf0C}@e!;VKqrasJd(Ao z7L&_2cbo0!VQc}`@p;57*Ix^b;6Rt{7la%6XeI}AHP`@YHzDb&3<0?N0OyPe1HHwc znwRuKL!PGZ_H^B&q`X44e0HhOml5&x4HAd(Tlpvl?&n^yGi>qMII>j`3F&oAu&oz4 zuyId7@C7V21PuVvqMNJV5vwBHMOWivU7>cvaG*=CAmi01Or3Oh)LWs*mF!z!Q`7&F zV_zEtZH5YYDME=HR4yL`P<#G5zlODYsc~0~C62#pxsz$<3-`zpT;-^A4TgNN6b$4gT+-j9b5;opOQzD`iyieZ@-DFQV9T<&LOT#|1UPm z6{kYk1_8Jn4E7A2*{UM5u|(p-IeDX{Fny}r|}Do?*%zd zOiN6&uf<{k>P0z|YKpY?NoRv=VB#JlB=4r_-k?BLqHO<3{4osD_T^JXRp)j^N5wuT7BKW^ zN0_{-PY%*t2n`_h`c5S7(QY>kX?0WMo1dBj3Ty+M{pS4Boi6T2MW*VQ3Y4- z`2N1(vfC`Y3lQSJOtPjIFjXtGGcZsrr)^nZuCrvLn05#|Ua*G-=%(zid()-&(fe6k z8*I7>RPm@VfIRlcbb_sjGt`5z%7IthLwD)Vz<@3{SU9FFvQ}<_;h2>)aAM$Pw=an#e2#yG5p#sJ4Cj!Lx z*Kh6$n>6OV>lCTCOJsH4e@67o7TcXtLXUJZ`+A1%*h@1+8gH0q@r{K04^~e_Ij%e)jahB-8*k{BS>X zwbzv{CjegeG zvW(h<_n4w3?|L37;nf4jMYhC&3Je^rS3vI8d|2aE0XDzAgKjcvBlK@IEij)#SjfkJ zVIgNKWd2+%zeb)_uNfmc?R4-Ef&!Cq0%U**b)0#7RB^ zxJxWwJuks=Br!oDr~RD2SoUgb@;X?Z@=G!=VSt$QLq4oyH_Ks`3gCQdatS}f8(PVY zz!yb%S?)wwLgAcC4d;N=hFOEWJ8zKodjQi0`h0sKYJ2y5H??h-ODl7YQ^W`~yOd5T z08(1J%(Qd6UOHm1z_*R4W**U%1j@-2NSvuSB=o?2ekv<76;`uKrPkO+vU^f)pvFSm{=I@L~3{gV?j-y;k z*<-Abu}Rn3z_GjaP|31^aLH2Gw^3o|Lm=T$OEK2q{bk9Gn8y!l#qu|hD(f?OR_|2$ zS%^e$G550yl8+8BFueNW^p+JYTXpf$EBCqID&vU>N+rPotujQX4JMv7_JMmx%aq4f=mcUuU?%HWE5|9a3(1T9}%K1q)( z<>E?Yb=X>om$(&M`acq@$YSTk-K?04CcX@%DZI$sl?jyM{_Q)eU$ur9Bc~ghM(Ov! ze>n2p#S8F`KKCdbtC#-&5YR4mk66}HW<57_#`yS#b(d~OIGL~*;Cg6h_qK1jBxQJ*-*quA8}RK%>dC-p*Rno~FPFDyo$u^iIPUK+kp9DmO9 z#19{>*DRXS3xVW`&1#_j*G)$@Y&sxPv0$HoL`2=jP~^N{HA2-N-DBXOq39% zNly+aWpL?ckH|zcfeWSu^CQG2(%>=*2P1%_zSv*6PgA#qMzPiKl2rB|dm$nI;e6V& z9}&Y$OCu^n?pzUh#W%@(8ffb3=Gmiwa1Thqov>nQ3eBd~VylF$A>K76_5EZH=7fDg5;M)E#j#I<86uTXS$iRMrMN4b9tv?)~?5b}JU- zP>p8^X`|?jB2{_Hc+zCjW|{*{)fa1V_fI-bRg~sf0%w*f0Ca_s##))LA&N&khaSAE zrIMFKOqyx^spcN#DE*~#{uv)7D=6E+(-&y{e{7_J-&sb6TP551D1xF|Dix2hg-xM0 z;SoL+AX$=5LVN&l5n2lBH%?w2P7LWKO8Y;9@R32=!34@!3|1`*-~a>%+CE|BsC$`Eh5x(n!V>B0GeY3bbECZP|%)HvG7u1w4Wd_vX* z0rp5{`ixf=zT6wrOkON%?8oRWxlS$c{wuJDk_YVt^#@pU9q{*(-bpl-ZjB8Cy zL>Srlkd3{<`f?2$o0MEy2o|y^?@oe&eG!2PL#v)(Y5)KL00007TuL^k&zpRAu6NS) zy1mN058OoWD_pEwn9bFeoTX3lmy-gxmWE*_i3*n@2xpEQ+Di)8`-lJl1IEp|`4|8I z01grA_Y{j9i6diw{W+1Bgg^iQ2MMFV$N@Z&HTI;l5fODC(l;EPcY7SO!&_D)2C9?ef1>YS-J!|GhJ*SKR|v>$dS8)*9fRD>(PU^Ag2i%E~!5YkN} zkByQDUG76JF@&$iP>$=HeEcKGWS1%!t*w+eax=79UDcVff@FCj?lu{;xMW{yT9PZj znlBpPuH)l{Y9CZB{Ey2|C8S zi`GaV`C_TBk#|!XFtmxSf2{u}ar2JF&%*k9p9UdLVp!T)m+sO^Yj{*C%)(SfwQL3F zfs9-pw`$r^)#FE?vqTmSxA&2zA)mcidFEH&n{o61 zQ8dx!^%|5VryLaAX?I{qKqD}k+2MRT693-SD3>tFNqk5*WRsicBL;{~ERZPTlJH5A z?Hw@e`rq1*0|n|Ct0yJ__RibF4!^ul#k@guV6jucougl1wai_bV^4iwMB5{~V=AA_ z1D!W3+{vlRD54})uE@RT4`xaze=uC_FLf!yCQxWZDT8=btlV>|LjaiQZr7Xzw?a&r z*2A0upSnfWV0%Ai1b2M~M*4c}t$-gL`B8W={nIbKeg_FGtL5s5e5BQTTKwII=F}*z z7IyN%9ow~4(`);-oMMW>mQMJ6mueDB4Dh`7%j`ZhyingZz7E%oO?hVe`*DUxKbiyg zyO2sT9g4X&O`ie||n1Om<5kNz0s84G8_KLGqQ# zaebY^yZ8`4EW;7N_U>zr1ivEH4@)JX)zQwSCFnN|oA^Bk zaVj#pdoYAyTejl<-6?8Bd28Gn*&?3?d)GyZ?+=hccOt?_q98P1m9r-` zT2nU{j&My*s@|b-M$Ly5o*s?v@L-_}CWXegX7=Z50up(gjlzE22Ts+vTF_H9H-_PH zMo5g`;CX2-UYz^Ahtd3^r!IDk}RMHFbOTh;G0 zKmG#nxpEO1R0vuIB2lMJPr@~&yVCuQ{?ts3M%V}IbDp}syf)4$_@sd#gIW~k{!kH& zjI1T6d@Xuv&}rF+_Oh|hz#o25VTM0!4fxZw@JtG!f~u9B(#vO{==f5u5xEUmPCRMy zHH4^zxHAA6`&>^8=!FQ{t1VrD?1nKH(eA#nb9m|!?x^lxE}s#1od5C}$1_OlNZ}gG z7W$RKl`Sfh#|W9kHk_d1D+1e5qeisXIOS+E5@){}w0r3;S7^sTaX(*GH#aXjbN3Hv zWS1^F*=v>~5oyaO0PcqnPynVG5@(kJhI**Rd};OD7L-hF*dGp*q54%b0qx4&un|lx z4z!ls@kxRu4qhVM&D=z#mbAZA5w@b{(W7<=e$3ML;b2QGKPaJx;zOJtGc7b&4lMAK z((bu;!$Ui-MJU9r$Zn4|`^VxV6f!UVMVSK|C*1@?6vACYVBOMAsj3!8yG7UqPb47X z@G=s}XRVWf93Fb7A6B5{@2lY7GLsk&;dFU9Y9c&xCyd8b*m|fqg~_32p)oC~xAo$O+Y=HmoXIQ{^Tv z0xQ|F)}S?GLzNz_!(Z!im9@$|5i&on8N-^c-U6@mE`9Asj-#j%#k-l!HeZdoHP0$N z5F8bbLkdYVdH?_n&=E^^0*YWIX?`BK>q~LOD@Z1DS}WT029TV|r~ridPtLw^sLJ0z zl_tw3sOST=E+=g!(ZfITC4%CCyQU~;lq&gUJZn8WFq=vTAFTGY)QzA4jH&# zqL(`sVxxk&^&`IbQYqV`$nN&eCgGOq!qRVfa~HMgJC*X?#OOfus9`(qN0yP%o9{L{ zbv>i7IR@I1`0&>+-`y#FuhSbJPMolhofp7011{e~ldBgaB>*|gkoJL5lU%7wJ{5W) zc4e3I~PLX=B|u zNf{V}Zb}Jg%bB=+)(p#8rxYD70geR`;RStL?((+tFeCTF9r)kR(aJ?3Vx2nqTAG7D zzheA5+zJ|_qDIW?!4<7hnkv>fi}dD)f$w-Qrm^OZ z9#yjoa5C=crT5NY^DAD5WE%2eC4?AxD1Xy|3kgKip4IDd1ardJ7$y})Q!S)rjX~0| z&4J=THZ03Di~T=ptoxUN;1E+r95WJ;pvXK7^_#=xjm5!p8=nrymvq3wy%|(Q<;Z|J zSm;jmM2rcF2UX;Q;$$i__Mf~|6V(DB(CFVh?n)Q})npe)fG+pr@Y-Yp8W-e$WpodX zf#)S(dl)Gjs?=tH00CW)qNtI8)jk<5L7i+hG~q>BPCOIyAM7>yF?=|i6xYSAhrlj( zi3Uouex$8OazBSb(3gqC1UEy%$;$IBlsv&nwEm{fOP>GktRx&2$Dlc-St1w#|7Hk7 zBJ(EqU6O}>8OQY&Lw`Yz#Z4J? zo^@!g+pi*)%`j91!Ag2m|5(6hQ1$TAwJ54$@(~#-lpivt#!VC)WiC1 z(Up!28gwO-DeG{WEyp2v-B^4=?b#PO3W#PsfT85dpxI^KHu(sZ*o@aiAlmGT=He3x z2tI~S$ICj{aAsA|Uw1?w%`(RxJhLmf$%gJshm@9t+l zrytOX=lr2NgA=_X*$K!8_(2v-YO)3O6gNG-AU zhUl}hWSe5|>GM4o6=iW@4CRU93h;Hc$~V2)#x;~KR9f6}=r=PGBNpW%Vpv*)9r&nT zWt%@y?x-1{8=t%ulFUo=K8jXRh`iKmvt)-#9XApSd5qHDL6W~Lj7Vh?p6yu#2l{3` zSf_V(QiL8^UQHP8{C&@Mm|Aj5O^|nf7L{j&hL+QGezcsV0&nkG|sPzEvgKJ8Hd7MXdcZ1X3=tx4S@Q| zhp(`ei_|xIqhKGTJ3|knr(D9!p67=s4}hfRRpE5`j!+q43TPLnFD!~47H>|AzI-`$ z0T7E-7331}>!r8Ohl*06Q&@e2m>E^KUZ&+UN46`F2`wP@B#Ql6nm_X~L_Jj3INMuS ziSNIHU{`FWu!uMKdzv^_X-`Yyi3_O+7p5!IK<8+f8%vzWJW(>jDQH&d-i{hJT-G@$ zRC^ik;>xGfsnFAW-HWf{9YOGm+0gD)#KBP;+kJp!Ud6lYy^V%g$a3p5TG{xk`$=G( zfAfc38u?7Mthg7xl&Mzt-1wj#b3+<$y^K93-Z7{00NdG4PMWOze*4DR%%16bHLv4L(lYOSll<sMm4I-8w%_PtCDqdJA|vJ(!w7;s8%iy18=7e)4@cdC#Qd!XzM&W8Ugr zGt&uEmJ*5jVyh@BRtj0;q~PNz^7`lbB&YQt-%kv4Fo|v?_nY(5U)DzY$8J4TZ{I7z(7a zK+y)zxul4g7N`CsnC_V-+A2wvB_Tv3`AT<(azafM7*5ScQOMxU_Y!|TgJn$B1R;%? z>l$=`E*BU}K!!a={MU_fBJ)-%$F@In*+@xXLP+B7Nx#Hx&M^0#KcZ-DV^Y951x%*<$!U9@zlFPu&I-qbf-phapTB4{Kpr zxKGfTCKHj*n>YiyVvVn{MGZ*J&YuGbd`j#sUj_bGW$J4)oXc z!j+moXL+vCZ{63z(p5CL8zrsp5&;Ec?LtnA zOdN!^06TyH9nLjLOJ^;2sXDcG4=C~Z}X6__k(~Cho5&07J_~S+U*LX)NzzQkO|&yCSVs@ zSAGK2O{jLZ86af#=HKw_6JEFkzj(=q<$LfDBG#ZRcsBH4)S87*RD}Qw6OE?0V2PfC z|52~w>2AX|=j9?taUR`-|G>ls66k|AG4oBZ0000TE%wzhJ~Cq)VDaOG&Y@6Z601Lb zZfr|XY$gaM*(tUg)oo;?J*@!iwert3i&TZGnVhjF`0-NH zA*!3x!ONRw!A7ffCIJdT$iGh=?tfn$4?2e!`R`ST>bVZwQsh~J8Y~xl6g?*zoV3hW z2LjBBDY(UTuSSo4A*<4h?chkyV804JP)k<5l$nrfu0ZFx4~ z(8CO#Y?%B>Z~v^|2im422n(Jnm~X-#({7AfKRv$_MrqVmQC#KG$biPSW5~bSQZZ;Tx?|xV|JQ{ zsUe{^)&AoSorP7S|KiiBbL?PF_8#t&!<5&!oRABw>pR|#k_=b1_G~ML2;rjXvSpl2 zqgTVP(b>-vkrBQS0001-`<90n*1Gg#Iw)BJ?$V%y1%4Qj>ZjNyOsKSNYu;=aaei<~ zVmBIT*tN*ufurNEV<*jJU(Y~P($|sK21$zXnLhbDa7sE~~ZS z{0joLA9wV1#pWHCc*C;q7{VBW7JDtWaX9}A+`*MWVvHxgijsJuc*8r zuugIcEt8D1v{|J900000000yRQMUj91$ZLPqIgCRVYuUu6?{w{>S4NoC#_nJo3t9ewD3Oz*NGCvF1r#dRms}TEws|2+FjF>GqXTYA!{3A z?)AQ_3DNe%R-e^NFP%A&Y&p0#!HH88UB?(PoU zu)-6J7m{D6RAtVNZfzit&`fbn&0eZ~Ay`cHk=M~tFHIxX8`0JnaZdhxwrtn1GNK8{ zg8U3t#NsPm53|Uodm^s`z0R~Nd43Apr9EOeR_{j>yM+?AcQhaeoRtaHApEI%H%a|b zwC$r)*4LsD@M;MAkV{?4lh7I)-d@3vIujE)OzrmkR`|g&#bEpF8$e4zZYUKbx0exJ z!)8W70TiIwJWcU&R9x?3gQiKzkPs%4B({k~$DHE{kG`w1(s^@RS&RQZcoHZ@tCCj5 z9Vf63LIQ;U@(4Eu5N@bwz(cgR8I{+E#&hr6i;NaPbuOofR#lM`GoIk z;sxp*hagzR%rp+f4Rk!^QFJq#-e#r1Q}2etkUnE&8dYeyV{;-83w6Ja2hrdJN=>O~ z^6@3j*6&W0q`7Qy=q9V@#M!1gC|FDnldHD&nAsuJvpNmevc@20Na>^69eC!o>$j{6u2WTDq{BH1;$RHUMKGnT4iOT$vy^$cq4vVbIB z_3qDI(i0ZZ8caY@i?0=E4%?RTTmedP3%~`hT+4g-{l5yQ&^;@|O)?nzC62&ZS)GiG zEYDk%n8UmCBTjRRENZHXPFf>uU8upe*14MeM!{K6tORC zZ{l&#UoXc3(bHfz6&wDd*O+!sG zn<}nlF?kq`1Bp-yML41GnGxMIrkH1kZ9r*bWan)+a%L$?q-0-ER-q+N1QfbiE>mEZ z_=c6biZuZO-@(si7gBVmr{JsTn^0flDf+_H0h-a`)Gj$7*o<(bT2J&!XKIyNs>F}< zu+FuvtfyL2nCHqKq;J33BQQF*71+A{-(&=BSVwsI?P&P z2mWrC7c62z33wPpd>IL*4YtX7W%ML_07`M#F9#INTDR4SZPa%O=sso2WSwzdFs`-y z?d02`=S7d|kQZuZo?r(O@2(XTf(YVskcDFqT&o;z`Nc6@wT3?<=yS8|YS3@PROSBG zJmb4p&N{@&{t=hYl4$f8`XjeV`l;zX(?>SbccU{0Jo`OM%~A~RECd{bErKkrVT{Zt znu=tZw?rtNUHn&e@KCe`nSznVKWy4Be7VqTdYFF!5Kh1hFS@TRGK|Su9c@@UE1Q1s zr&oE@ZeSO<`qq^I>m^>w6`^@@%HOU*CyHi zn+GjP8Tgen(h<4_%PGmvK8=-%(&1wc8K0K+ad3EcrFc{x@5kI!(x#va+GMh_o}KzYsL%Ilty#(S%0-vo@6w^y1E`2cWPG<*sB&_rLk6U9wrH- zbNW5Q-Oudy+!s{4rbTamZm`#AcJeCoF6tfY#xMxH2oMd_1KDS<4cGKWb!pcJNy zqYA&B*dV{0Mo>DIQp_N9=7t7mJBmBx@6rde-I1y7G0|>ura1zGnRW7MYCT9P8>Bge z99Jp`2CuTJb;C-EQB}!xEyN{r3z2rAaIOo#9bBG$jwqQqL^6Ht@i^0pW|Zg|ktbc0KiD?LG^xxV@;=t`|eeyNS{Z-8J4s0jUW)=jF>gD)+U0tG8%HBcG4p zDiji>@trdRR(vfb$U$ElzC(3x!|UWzWoe8$r9Z)sq-5rj2Bgbz664ZcCPXz9r{5{1 zRYwduTNxFkB2LX)@$M9{As@Q;0Dga-#+5CEWVkQC%}Jtbbi{S%X6+lxU?vt^k|68m zz8u*y6BhiiT{fOpAe(sCKc6_eqEd%+LK|JQOoib8+Rd$Hn7YY&7b`8i}u9&{vH~HGmKkee83r4}Z zlM(p-Y$)N;WQ3kqaKi9X`ET-xikT1IagB`=ea5^`svnK}MGKK?`chSZ${61;FAtanS>V&h5??^PKJ#{DHbgKV|Lx!io#&+j4>9){ zy7AHVHdfHKQQzjDnUAj8Zaby(Q>T4{U5y8AVR^mcxijW66)+1tF%buE~CU9<1 za*f9VM`Lxwu2+?$Cq7mi27$M~bPKx<*sEyOxoIu=)TBIy+j6`<&chtl%)OIo#BK*S2w52Zgn%iG!DMc41fJc68+yjy3}7~0Z16z{Sj4yDt5k`zk?HAc&(fX&Z7h3yn`3`4n0 z6hq|f*P4=a10+9jG^w1g61Zw7Ji``3*D*-DFrurna1^LDkI=+@24pOM!-Ghq^y1Ru z0xqdDB6i29iI@gKWq<&aa0>#=8GbSX=I}X@Yo+wNfm;!c=y=U1*x{QP%nffzZy~o@ zW<`+0KBj*D00Ky%+ds!W9s!Mi4E+{y`%L!PuD9GUCE5G}e}!vfdY_u!epA*vxrY8j zO{Ahs4(iHNeXDjjE<@&cUIfkVfDd+>QesZ;ruJyY|A35+hVp$$f#oJ(Q@v}7CMXcz_ho@ zGH6$SM4YNk8Fu?LS+~@jvF%iVFN%Lj8Z`QK{KIe=oOw=|&$Wgx_EDa61DJjR;aY+R}xyi99*kC>Gfv~ zs^5HXvl*=v-nA&o2#XI&<&5C@Caf0C0M`nbt7V-7UqtlHYL3bw6XdYB*$0iQ(CNNo6S+QUs_=W?$4bHFYsu5cyBPRP^9 z{Z9~xczYRzUg!k~?PH@V&tbfUYef_(D!(Z2eU*M%aQtS?A7f^Z^+C^O9#0M|sx&Kv z`9*iK49D-6B(!komzQJpwSozALpwG=mYQTT4$(8bt?~9D*rHR>+ZhCeDx2`@9p+d( zMV6erL)NHl`;WvW5 zF7%pmk=1GP6O7#NUGzJ~L}!+Ose%W9KN)#jalFD6^1T%n;I#O9KzuWxNYR<_18XQL zRoR)%s>C1HEY*aL`VE%Kys7X20KLcXT0dV54Ku~c@~@^8(wm^k$gWNQ74X?*U(H~P zE~9uQ4gv8xRzN+?-JJK5szjH~Z5^$ulL^pZv#0_4&y3w-w%GwO6yvEP7B*Wc#@bzb z@%=VAA&vu{CnMt^7spJNM#+nB&}h5_`wETs3mw)FeN}HB*c`uSzvIA8A^u+I0?*R2TLSIjBVk9&!qOh9k8XmY=`#E zEI}{Yd}%}o?e}JbJ0f(BF^Qw0q51rsKLRk_Oaz;E%VJWqC0S*O*4#zvgE5M2i6|rj zB4?naJ`Ol>!=LhsD9Wo<-;TP)AH1@@LNO8QY~W*$iEGm2m`20X)>mxilsEyAgat=a z(V`=|f8j9UPtPh@U0Qi1tM5b#%GPhIywm%gitZrliw9!$i!#SosV(V%$mx# ze94?S#}Jh5K^d#&d*l`0KXf+?%UT$snsd1v<}!J19cUe8${MJJT-XG-5Zg~`kW%cb zdioTvz?>^5*wo5)jp9#PiqqEu5L1v}nNhl#RUnotP=%~OS}cd;&cTL1Oy1`Pq|?7` z+a%9V`Ys=4|1Ou{fWoy+l@u^Bg%X^SUuXH512&yucGYBK%79$yJ5IZeQ}>k$m$CpC zlC3BA2N@vl`^M1o5M%*ieimO$r`CKYB^wYJ;ZCrODq}DwDp~9L%QucWts|`%wvg*9 z*N!3|54&UB_IuE>rn>^@%h04VVgDbz-|S~i^2S6sH-?9RbZ3-3V7h7`Q~UPy!wZiN zTej-hXRmLe!<;B4{ z!2`>t^Vg8)rJ>4kbnsv^&;`E$8map3%lQW{Uz|k{$wnb>4sr$w^Z3eBAM;9ZGuN$y zDcNVk0f`u-LNyGP>NAi6pInMt*1+J7kx;i+4%2mYgol8lP<&HXjD@0)OMC~&;Eb#c zWm>)09LQ8amqhl+srvP=5U|qzyROTSzK~#XcUPX1aA(=V7y}r}0P^PZRYeC&>;ln{ z0_^#AA4)m^0E7=y43_WU-fTgj2+_E%-e^_wgFH46%mxj?3&fZ%y7gw0c*mzJZX-K} bZw=IP-#y8-Snf?bGynhq0000000000oQbk| literal 0 HcmV?d00001 diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 374a88eca..2b9d9b811 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -18,6 +18,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). ## tip +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): support [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) as a datasource. See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/) for details. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): `-rule` cmd-line flag now supports multi-document YAML files. This could be useful when rules are retrieved via HTTP URL where multiple rule files were merged together in one response. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6753). Thanks to @Irene-123 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6995). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): support scraping from Kubernetes Native Sidecars. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7287). * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add a separate cache type for storing sparse entries when performing large index scans. This significantly reduces memory usage when applying [downsampling filters](https://docs.victoriametrics.com/#downsampling) and [retention filters](https://docs.victoriametrics.com/#retention-filters) during background merge. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7182) for the details. diff --git a/docs/vmalert.md b/docs/vmalert.md index cce31e75a..94e03a03b 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -10,7 +10,7 @@ aliases: --- `vmalert` executes a list of the given [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) or [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) -rules against configured `-datasource.url` compatible with Prometheus HTTP API. For sending alerting notifications +rules against configured `-datasource.url`. For sending alerting notifications `vmalert` relies on [Alertmanager](https://github.com/prometheus/alertmanager) configured via `-notifier.url` flag. Recording rules results are persisted via [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations) protocol and require `-remoteWrite.url` to be configured. @@ -31,9 +31,8 @@ please refer to the [VictoriaMetrics Cloud documentation](https://docs.victoriam ## Features -* Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) TSDB; -* VictoriaMetrics [MetricsQL](https://docs.victoriametrics.com/metricsql/) - support and expressions validation; +* Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) and [MetricsQL](https://docs.victoriametrics.com/metricsql/); +* Integration with [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) and [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/); * Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules) support; * Integration with [Alertmanager](https://github.com/prometheus/alertmanager) starting from [Alertmanager v0.16.0-alpha](https://github.com/prometheus/alertmanager/releases/tag/v0.16.0-alpha.0); @@ -458,7 +457,7 @@ In this example, `-external.alert.source` will lead to Grafana's Explore page wi and time range will be selected starting from `"from":"{{ .ActiveAt.UnixMilli }}"` when alert became active. In addition to `source` link, some extra links could be added to alert's [annotations](https://docs.victoriametrics.com/vmalert/#alerting-rules) -field. See [how we use them](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/839596c00df123c639d1244b28ee8137dfc9609c/deployment/docker/alerts-cluster.yml#L43) +field. See [how we use them](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/839596c00df123c639d1244b28ee8137dfc9609c/deployment/docker/rules/alerts-cluster.yml#L43) to link alerting rule and the corresponding panel on Grafana dashboard. ### Multitenancy @@ -728,6 +727,10 @@ implements [Graphite Render API](https://graphite.readthedocs.io/en/stable/rende When using vmalert with both `graphite` and `prometheus` rules configured against cluster version of VM do not forget to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on the query type. +## VictoriaLogs + +vmalert supports [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) as a datasource for writing alerting and recording rules using [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/) for details. + ## Rules backfilling vmalert supports alerting and recording rules backfilling (aka `replay`). In replay mode vmalert @@ -1323,7 +1326,7 @@ The shortlist of configuration flags is the following: -remoteRead.bearerTokenFile string Optional path to bearer token file to use for -remoteRead.url. -remoteRead.disablePathAppend - Whether to disable automatic appending of '/api/v1/query' path to the configured -datasource.url and -remoteRead.url + Whether to disable automatic appending of '/api/v1/query' or '/select/logsql/stats_query' path to the configured -datasource.url and -remoteRead.url -remoteRead.headers string Optional HTTP headers to send with each request to the corresponding -remoteRead.url. For example, -remoteRead.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteRead.url. Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2' -remoteRead.idleConnTimeout duration @@ -1356,7 +1359,7 @@ The shortlist of configuration flags is the following: Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url -remoteRead.tlsServerName string Optional TLS server name to use for connections to -remoteRead.url. By default, the server name from -remoteRead.url is used - -remoteRead.url vmalert + -remoteRead.url string Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'. -remoteWrite.basicAuth.password string Optional basic auth password for -remoteWrite.url @@ -1441,6 +1444,8 @@ The shortlist of configuration flags is the following: all files with prefix rule_ in folder dir. Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -rule.defaultRuleType string + Default type for rule expressions, can be overridden by type parameter inside the rule group. Supported values: "graphite", "prometheus" and "vlogs". (default: "prometheus") -rule.evalDelay time Adjustment of the time parameter for rule evaluation requests to compensate intentional data delay from the datasource.Normally, should be equal to `-search.latencyOffset` (cmd-line flag configured for VictoriaMetrics single-node or vmselect). (default 30s) -rule.maxResolveDuration duration diff --git a/docs/vmauth.md b/docs/vmauth.md index f9e2c5614..9a478d3db 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -1059,7 +1059,7 @@ See also [security recommendations](#security). `vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page either via [vmagent](https://docs.victoriametrics.com/vmagent/) or via Prometheus-compatible scraper, so the exported metrics could be analyzed later. -Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/21394) and [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmauth.yml) +Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/21394) and [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml) for `vmauth` monitoring. If you use Google Cloud Managed Prometheus for scraping metrics from VictoriaMetrics components, then pass `-metrics.exposeMetadata` diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 46ed4cc4f..2d98b90cf 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -786,6 +786,38 @@ func ParseStatsQuery(s string) (*Query, error) { return q, nil } +// ContainAnyTimeFilter returns true when query contains a global time filter. +func (q *Query) ContainAnyTimeFilter() bool { + if hasTimeFilter(q.f) { + return true + } + for _, p := range q.pipes { + if pf, ok := p.(*pipeFilter); ok { + if hasTimeFilter(pf.f) { + return true + } + } + } + return false +} + +func hasTimeFilter(f filter) bool { + if f == nil { + return false + } + switch t := f.(type) { + case *filterAnd: + for _, subF := range t.filters { + if hasTimeFilter(subF) { + return true + } + } + case *filterTime: + return true + } + return false +} + // ParseQueryAtTimestamp parses s in the context of the given timestamp. // // E.g. _time:duration filters are adjusted according to the provided timestamp as _time:[timestamp-duration, duration]. diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index 642b88364..9cbcce2b8 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -2384,3 +2384,28 @@ func TestQueryGetStatsByFields_Failure(t *testing.T) { // format to the remaining metric field f(`* | by (x) count() y | format 'foo' as y`) } + +func TestHasTimeFilter(t *testing.T) { + f := func(qStr string, expected bool) { + t.Helper() + + q, err := ParseStatsQuery(qStr) + if err != nil { + t.Fatalf("cannot parse [%s]: %s", qStr, err) + } + if q.ContainAnyTimeFilter() != expected { + t.Fatalf("unexpected result for hasTimeFilter(%q); want %v", qStr, expected) + } + } + + f(`* | count()`, false) + f(`error OR _time:5m | count()`, false) + f(`(_time: 5m AND error) OR (_time: 5m AND warn) | count()`, false) + f(`* | error OR _time:5m | count()`, false) + + f(`_time:5m | count()`, true) + f(`_time:2023-04-25T22:45:59Z | count()`, true) + f(`error AND _time:5m | count()`, true) + f(`error AND (_time: 5m AND warn) | count()`, true) + f(`* | error AND _time:5m | count()`, true) +}