mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2025-01-30 15:22:07 +00:00
lib/protoparser/vmimport: extract stream parsing code into a separate stream package
This is a follow-up for 057698f7fb
This commit is contained in:
parent
66f0a78810
commit
41feed813d
3 changed files with 11 additions and 8 deletions
|
@ -11,6 +11,7 @@ import (
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||||
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
|
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
|
||||||
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||||
"github.com/VictoriaMetrics/metrics"
|
"github.com/VictoriaMetrics/metrics"
|
||||||
)
|
)
|
||||||
|
@ -30,7 +31,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
|
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
|
||||||
return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
|
return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
|
||||||
return insertRows(at, rows, extraLabels)
|
return insertRows(at, rows, extraLabels)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||||
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
|
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
|
||||||
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||||
"github.com/VictoriaMetrics/metrics"
|
"github.com/VictoriaMetrics/metrics"
|
||||||
|
@ -30,7 +31,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
|
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
|
||||||
return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
|
return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
|
||||||
return insertRows(at, rows, extraLabels)
|
return insertRows(at, rows, extraLabels)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
package vmimport
|
package stream
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
@ -10,6 +10,7 @@ import (
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||||
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||||
"github.com/VictoriaMetrics/metrics"
|
"github.com/VictoriaMetrics/metrics"
|
||||||
)
|
)
|
||||||
|
@ -17,12 +18,12 @@ import (
|
||||||
var maxLineLen = flagutil.NewBytes("import.maxLineLen", 100*1024*1024, "The maximum length in bytes of a single line accepted by /api/v1/import; "+
|
var maxLineLen = flagutil.NewBytes("import.maxLineLen", 100*1024*1024, "The maximum length in bytes of a single line accepted by /api/v1/import; "+
|
||||||
"the line length can be limited with 'max_rows_per_line' query arg passed to /api/v1/export")
|
"the line length can be limited with 'max_rows_per_line' query arg passed to /api/v1/export")
|
||||||
|
|
||||||
// ParseStream parses /api/v1/import lines from req and calls callback for the parsed rows.
|
// Parse parses /api/v1/import lines from req and calls callback for the parsed rows.
|
||||||
//
|
//
|
||||||
// The callback can be called concurrently multiple times for streamed data from reader.
|
// The callback can be called concurrently multiple times for streamed data from reader.
|
||||||
//
|
//
|
||||||
// callback shouldn't hold rows after returning.
|
// callback shouldn't hold rows after returning.
|
||||||
func ParseStream(r io.Reader, isGzipped bool, callback func(rows []Row) error) error {
|
func Parse(r io.Reader, isGzipped bool, callback func(rows []vmimport.Row) error) error {
|
||||||
wcr := writeconcurrencylimiter.GetReader(r)
|
wcr := writeconcurrencylimiter.GetReader(r)
|
||||||
defer writeconcurrencylimiter.PutReader(wcr)
|
defer writeconcurrencylimiter.PutReader(wcr)
|
||||||
r = wcr
|
r = wcr
|
||||||
|
@ -138,9 +139,9 @@ var streamContextPool sync.Pool
|
||||||
var streamContextPoolCh = make(chan *streamContext, cgroup.AvailableCPUs())
|
var streamContextPoolCh = make(chan *streamContext, cgroup.AvailableCPUs())
|
||||||
|
|
||||||
type unmarshalWork struct {
|
type unmarshalWork struct {
|
||||||
rows Rows
|
rows vmimport.Rows
|
||||||
ctx *streamContext
|
ctx *streamContext
|
||||||
callback func(rows []Row) error
|
callback func(rows []vmimport.Row) error
|
||||||
reqBuf []byte
|
reqBuf []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -151,7 +152,7 @@ func (uw *unmarshalWork) reset() {
|
||||||
uw.reqBuf = uw.reqBuf[:0]
|
uw.reqBuf = uw.reqBuf[:0]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (uw *unmarshalWork) runCallback(rows []Row) {
|
func (uw *unmarshalWork) runCallback(rows []vmimport.Row) {
|
||||||
ctx := uw.ctx
|
ctx := uw.ctx
|
||||||
if err := uw.callback(rows); err != nil {
|
if err := uw.callback(rows); err != nil {
|
||||||
ctx.callbackErrLock.Lock()
|
ctx.callbackErrLock.Lock()
|
Loading…
Reference in a new issue