2020-04-27 21:19:27 +00:00
package main
import (
"context"
2020-11-09 22:27:32 +00:00
"errors"
2022-09-14 12:04:24 +00:00
"sync"
2021-06-09 09:20:38 +00:00
"time"
2022-03-29 13:09:07 +00:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
2020-04-27 21:19:27 +00:00
)
2020-06-01 10:46:37 +00:00
// Rule represents alerting or recording rule
// that has unique ID, can be Executed and
// updated with other Rule.
type Rule interface {
2021-06-09 09:20:38 +00:00
// ID returns unique ID that may be used for
2020-06-01 10:46:37 +00:00
// identifying this Rule among others.
ID ( ) uint64
2022-06-09 06:21:30 +00:00
// Exec executes the rule with given context at the given timestamp and limit.
// returns an err if number of resulting time series exceeds the limit.
Exec ( ctx context . Context , ts time . Time , limit int ) ( [ ] prompbmarshal . TimeSeries , error )
2022-06-09 06:58:25 +00:00
// ExecRange executes the rule on the given time range.
ExecRange ( ctx context . Context , start , end time . Time ) ( [ ] prompbmarshal . TimeSeries , error )
2020-06-01 10:46:37 +00:00
// UpdateWith performs modification of current Rule
// with fields of the given Rule.
UpdateWith ( Rule ) error
2022-03-15 11:54:53 +00:00
// ToAPI converts Rule into APIRule
ToAPI ( ) APIRule
app/vmalert: extend metrics set exported by `vmalert` #573 (#654)
* app/vmalert: extend metrics set exported by `vmalert` #573
New metrics were added to improve observability:
+ vmalert_alerts_pending{alertname, group} - number of pending alerts per group
per alert;
+ vmalert_alerts_acitve{alertname, group} - number of active alerts per group
per alert;
+ vmalert_alerts_error{alertname, group} - is 1 if alertname ended up with error
during prev execution, is 0 if no errors happened;
+ vmalert_recording_rules_error{recording, group} - is 1 if recording rule
ended up with error during prev execution, is 0 if no errors happened;
* vmalert_iteration_total{group, file} - now contains group and file name labels.
This should improve control over specific groups;
* vmalert_iteration_duration_seconds{group, file} - now contains group and file name labels. This should improve control over specific groups;
Some collisions for alerts and recording rules are possible, because neither
group name nor alert/recording rule name are unique for compatibility reasons.
Commit contains list of TODOs for Unregistering metrics since groups and rules
are ephemeral and could be removed without application restart. In order to
unlock Unregistering feature corresponding PR was filed - https://github.com/VictoriaMetrics/metrics/pull/13
* app/vmalert: extend metrics set exported by `vmalert` #573
The changes are following:
* add an ID label to rules metrics, since `name` collisions within one group is
a common case - see the k8s example alerts;
* supports metrics unregistering on rule updates. Consider the case when one rule
was added or removed from the group, or the whole group was added or removed.
The change depends on https://github.com/VictoriaMetrics/metrics/pull/16
where race condition for Unregister method was fixed.
2020-08-09 06:41:29 +00:00
// Close performs the shutdown procedures for rule
// such as metrics unregister
Close ( )
2020-05-04 21:51:22 +00:00
}
2020-11-09 22:27:32 +00:00
2023-06-13 15:03:53 +00:00
var errDuplicate = errors . New ( "result contains metrics with the same labelset after applying rule labels. See https://docs.victoriametrics.com/vmalert.html#series-with-the-same-labelset for details" )
2022-09-14 12:04:24 +00:00
type ruleState struct {
sync . RWMutex
entries [ ] ruleStateEntry
cur int
}
type ruleStateEntry struct {
// stores last moment of time rule.Exec was called
time time . Time
// stores the timesteamp with which rule.Exec was called
at time . Time
// stores the duration of the last rule.Exec call
duration time . Duration
// stores last error that happened in Exec func
// resets on every successful Exec
// may be used as Health ruleState
err error
// stores the number of samples returned during
// the last evaluation
samples int
2023-05-08 07:36:39 +00:00
// stores the number of time series fetched during
// the last evaluation.
// Is supported by VictoriaMetrics only, starting from v1.90.0
// If seriesFetched == nil, then this attribute was missing in
// datasource response (unsupported).
seriesFetched * int
2022-12-09 15:13:29 +00:00
// stores the curl command reflecting the HTTP request used during rule.Exec
curl string
2022-09-14 12:04:24 +00:00
}
2022-12-29 11:36:44 +00:00
func newRuleState ( size int ) * ruleState {
if size < 1 {
2023-02-08 13:34:03 +00:00
size = 1
2022-12-29 11:36:44 +00:00
}
2022-09-14 12:04:24 +00:00
return & ruleState {
2022-12-29 11:36:44 +00:00
entries : make ( [ ] ruleStateEntry , size ) ,
2022-09-14 12:04:24 +00:00
}
}
func ( s * ruleState ) getLast ( ) ruleStateEntry {
s . RLock ( )
defer s . RUnlock ( )
return s . entries [ s . cur ]
}
2022-12-29 11:36:44 +00:00
func ( s * ruleState ) size ( ) int {
s . RLock ( )
defer s . RUnlock ( )
return len ( s . entries )
}
2022-09-14 12:04:24 +00:00
func ( s * ruleState ) getAll ( ) [ ] ruleStateEntry {
entries := make ( [ ] ruleStateEntry , 0 )
s . RLock ( )
defer s . RUnlock ( )
cur := s . cur
for {
e := s . entries [ cur ]
if ! e . time . IsZero ( ) || ! e . at . IsZero ( ) {
entries = append ( entries , e )
}
cur --
if cur < 0 {
cur = cap ( s . entries ) - 1
}
if cur == s . cur {
return entries
}
}
}
func ( s * ruleState ) add ( e ruleStateEntry ) {
s . Lock ( )
defer s . Unlock ( )
s . cur ++
if s . cur > cap ( s . entries ) - 1 {
s . cur = 0
}
s . entries [ s . cur ] = e
}