mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
090cb2c9de
* app/vmagent: allow to disabled on-disk queue Previously, it wasn't possible to build data processing pipeline with a chain of vmagents. In case when remoteWrite for the last vmagent in the chain wasn't accessible, it persisted data only when it has enough disk capacity. If disk queue is full, it started to silently drop ingested metrics. New flags allows to disable on-disk persistent and immediatly return an error if remoteWrite is not accessible anymore. It blocks any writes and notify client, that data ingestion isn't possible. Main use case for this feature - use external queue such as kafka for data persistence. https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2110 * adds test, updates readme * apply review suggestions * update docs for vmagent * makes linter happy --------- Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
91 lines
3 KiB
Go
91 lines
3 KiB
Go
package native
|
|
|
|
import (
|
|
"net/http"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
|
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
|
"github.com/VictoriaMetrics/metrics"
|
|
)
|
|
|
|
var (
|
|
rowsInserted = metrics.NewCounter(`vmagent_rows_inserted_total{type="native"}`)
|
|
rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="native"}`)
|
|
rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="native"}`)
|
|
)
|
|
|
|
// InsertHandler processes `/api/v1/import` request.
|
|
//
|
|
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
|
|
func InsertHandler(at *auth.Token, req *http.Request) error {
|
|
extraLabels, err := parserCommon.GetExtraLabels(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
isGzip := req.Header.Get("Content-Encoding") == "gzip"
|
|
return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
|
|
return insertRows(at, block, extraLabels)
|
|
})
|
|
}
|
|
|
|
func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error {
|
|
ctx := common.GetPushCtx()
|
|
defer common.PutPushCtx(ctx)
|
|
|
|
// Update rowsInserted and rowsPerInsert before actual inserting,
|
|
// since relabeling can prevent from inserting the rows.
|
|
rowsLen := len(block.Values)
|
|
rowsInserted.Add(rowsLen)
|
|
if at != nil {
|
|
rowsTenantInserted.Get(at).Add(rowsLen)
|
|
}
|
|
rowsPerInsert.Update(float64(rowsLen))
|
|
|
|
tssDst := ctx.WriteRequest.Timeseries[:0]
|
|
labels := ctx.Labels[:0]
|
|
samples := ctx.Samples[:0]
|
|
mn := &block.MetricName
|
|
labelsLen := len(labels)
|
|
labels = append(labels, prompbmarshal.Label{
|
|
Name: "__name__",
|
|
Value: bytesutil.ToUnsafeString(mn.MetricGroup),
|
|
})
|
|
for j := range mn.Tags {
|
|
tag := &mn.Tags[j]
|
|
labels = append(labels, prompbmarshal.Label{
|
|
Name: bytesutil.ToUnsafeString(tag.Key),
|
|
Value: bytesutil.ToUnsafeString(tag.Value),
|
|
})
|
|
}
|
|
labels = append(labels, extraLabels...)
|
|
values := block.Values
|
|
timestamps := block.Timestamps
|
|
if len(timestamps) != len(values) {
|
|
logger.Panicf("BUG: len(timestamps)=%d must match len(values)=%d", len(timestamps), len(values))
|
|
}
|
|
samplesLen := len(samples)
|
|
for j, value := range values {
|
|
samples = append(samples, prompbmarshal.Sample{
|
|
Value: value,
|
|
Timestamp: timestamps[j],
|
|
})
|
|
}
|
|
tssDst = append(tssDst, prompbmarshal.TimeSeries{
|
|
Labels: labels[labelsLen:],
|
|
Samples: samples[samplesLen:],
|
|
})
|
|
ctx.WriteRequest.Timeseries = tssDst
|
|
ctx.Labels = labels
|
|
ctx.Samples = samples
|
|
if !remotewrite.Push(at, &ctx.WriteRequest) {
|
|
return remotewrite.ErrQueueFullHTTPRetry
|
|
}
|
|
return nil
|
|
}
|