github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vmalert: retry on empty data

Browse source
This commit is contained in:
AndrewChubatiuk 2024-07-25 17:13:56 +03:00 committed by Andrii Chubatiuk
parent d53dd6ecf3
commit 3bf05ba979
1 changed files with 57 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
app/vmalert/datasource/vm.go
View file

@ -20,6 +20,8 @@ type datasourceType string
const (
datasourcePrometheus datasourceType = "prometheus"
datasourceGraphite datasourceType = "graphite"
retries = 2
)
func toDatasourceType(s string) datasourceType {
@ -135,28 +137,23 @@ func NewVMStorage(baseURL string, authCfg *promauth.Config, queryStep time.Durat
// Query executes the given query and returns parsed response
func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Result, *http.Request, error) {
var e error
for retry := 1; retry < retries; retry++ {
req, err := s.newQueryRequest(ctx, query, ts)
if err != nil {
return Result{}, nil, err
}
resp, err := s.do(req)
if err != nil {
// Something in the middle between client and datasource might be closing
// the connection. So we do a one more attempt in hope request will succeed.
if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) {
// Return unexpected error to the caller.
return Result{}, nil, err
}
// Something in the middle between client and datasource might be closing
// the connection. So we do a one more attempt in hope request will succeed.
req, err = s.newQueryRequest(ctx, query, ts)
if err != nil {
return Result{}, nil, fmt.Errorf("second attempt: %w", err)
e = err
continue
}
resp, err = s.do(req)
if err != nil {
return Result{}, nil, fmt.Errorf("second attempt: %w", err)
}
}
// Process the received response.
parseFn := parsePrometheusResponse
if s.dataSourceType != datasourcePrometheus {
@ -164,8 +161,18 @@ func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Resu
}
result, err := parseFn(req, resp)
_ = resp.Body.Close()
if err != nil {
// Covering possible empty response
if !errors.Is(err, io.EOF) {
return Result{}, nil, err
}
e = err
continue
}
return result, req, err
}
return Result{}, nil, fmt.Errorf("failed after %d retries: %w", retries, e)
}
// QueryRange executes the given query on the given time range.
// For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
@ -180,33 +187,39 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
if end.IsZero() {
return res, fmt.Errorf("end param is missing")
}
var e error
for retry := 1; retry < retries; retry++ {
req, err := s.newQueryRangeRequest(ctx, query, start, end)
if err != nil {
return res, err
}
resp, err := s.do(req)
if err != nil {
// Something in the middle between client and datasource might be closing
// the connection. So we do a one more attempt in hope request will succeed.
if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) {
// Return unexpected error to the caller.
return res, err
}
// Something in the middle between client and datasource might be closing
// the connection. So we do a one more attempt in hope request will succeed.
req, err = s.newQueryRangeRequest(ctx, query, start, end)
if err != nil {
return res, fmt.Errorf("second attempt: %w", err)
}
resp, err = s.do(req)
if err != nil {
return res, fmt.Errorf("second attempt: %w", err)
}
e = err
continue
}
// Process the received response.
res, err = parsePrometheusResponse(req, resp)
_ = resp.Body.Close()
if err != nil {
// Covering possible empty response
if !errors.Is(err, io.EOF) {
return Result{}, err
}
e = err
continue
}
return res, err
}
return Result{}, fmt.Errorf("failed after %d retries: %w", retries, e)
}
func (s *VMStorage) do(req *http.Request) (*http.Response, error) {
ru := req.URL.Redacted()