mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2025-04-30 16:10:59 +00:00
Compare commits
359 commits
pmm-6401-v
...
master
Author | SHA1 | Date | |
---|---|---|---|
![]() |
58a605317f | ||
![]() |
5e76845b5d | ||
![]() |
f587771c4a | ||
![]() |
6392c0a0cb | ||
![]() |
5188436b14 | ||
![]() |
acd06e37a8 | ||
![]() |
e50ccee5ac | ||
![]() |
92bf019c3d | ||
![]() |
4261b28c86 | ||
![]() |
589ee65c83 | ||
![]() |
46ec5e1d11 | ||
![]() |
41356d4e3e | ||
![]() |
7d1294e080 | ||
![]() |
045f33bcc3 | ||
![]() |
ee81975392 | ||
![]() |
039411e012 | ||
![]() |
cc274cf079 | ||
![]() |
28a7840ad2 | ||
![]() |
a1ade47591 | ||
![]() |
492def6d57 | ||
![]() |
7b675bfc13 | ||
![]() |
75a970fb5e | ||
![]() |
c1b9b2bd8b | ||
![]() |
5d12ba702e | ||
![]() |
d6efa6365d | ||
![]() |
767c0cc996 | ||
![]() |
f0f548231d | ||
![]() |
5147846d18 | ||
![]() |
dab3ed4056 | ||
![]() |
e0254b996a | ||
![]() |
1d30efa48f | ||
![]() |
bf88327121 | ||
![]() |
a6d68f859f | ||
![]() |
b911f721b3 | ||
![]() |
3f39bd08cf | ||
![]() |
cddf36af43 | ||
![]() |
0e555b421d | ||
![]() |
ec6f33f526 | ||
![]() |
e316ab878c | ||
![]() |
4aa0f93085 | ||
![]() |
764fc6010d | ||
![]() |
3ec70a4b6d | ||
![]() |
3b7039679f | ||
![]() |
f3197dc5d7 | ||
![]() |
9d82de9385 | ||
![]() |
c34f6db5c0 | ||
![]() |
8ad81220d3 | ||
![]() |
7455e6c0a5 | ||
![]() |
22c85a149e | ||
![]() |
1e4238ede7 | ||
![]() |
1d218f6aeb | ||
![]() |
ba4c9133ee | ||
![]() |
dcadc65681 | ||
![]() |
ae9126fae7 | ||
![]() |
bf3b98954e | ||
![]() |
814a37e57e | ||
![]() |
0afa51a156 | ||
![]() |
b89cce6bd6 | ||
![]() |
fdfa68bc2d | ||
![]() |
1572e1e5c3 | ||
![]() |
55d75b911c | ||
![]() |
f9e312f35a | ||
![]() |
0cfe28c2fc | ||
![]() |
1c39853164 | ||
![]() |
7390f99897 | ||
![]() |
2d279750a9 | ||
![]() |
e9c577132d | ||
![]() |
899d70a93e | ||
![]() |
1d13718fcf | ||
![]() |
fa4708178f | ||
![]() |
e0ff58ce5e | ||
![]() |
a9f03605d3 | ||
![]() |
19025d5a19 | ||
![]() |
1f8c115428 | ||
![]() |
1863c5b95b | ||
![]() |
7756046865 | ||
![]() |
7f2d194948 | ||
![]() |
f224b60410 | ||
![]() |
46d32af89a | ||
![]() |
3a7ea02d41 | ||
![]() |
84d2922f5e | ||
![]() |
39b27cb397 | ||
![]() |
b1523f650d | ||
![]() |
297bc28e3d | ||
![]() |
54bb834c16 | ||
![]() |
edee80f215 | ||
![]() |
5491d54c11 | ||
![]() |
14561a7ed3 | ||
![]() |
cd4ec0e739 | ||
![]() |
8e76b41081 | ||
![]() |
9751ea1098 | ||
![]() |
787e6a9b24 | ||
![]() |
73a69bd655 | ||
![]() |
ec6ed3cb48 | ||
![]() |
faf95b4a58 | ||
![]() |
a7106040de | ||
![]() |
792d0f8bb8 | ||
![]() |
8628f90e6b | ||
![]() |
89237a04b9 | ||
![]() |
d6c156727b | ||
![]() |
f1fdc8610d | ||
![]() |
611450a188 | ||
![]() |
3b1a41c014 | ||
![]() |
cd9bb6b315 | ||
![]() |
e58d3f03c7 | ||
![]() |
3e4c38c56c | ||
![]() |
c3becae96b | ||
![]() |
84ee713dc6 | ||
![]() |
7dc58894d1 | ||
![]() |
30376722c1 | ||
![]() |
6674691a58 | ||
![]() |
55e3ccb5f0 | ||
![]() |
2c97c8841c | ||
![]() |
f38736343d | ||
![]() |
33315f1ece | ||
![]() |
680cbef0cb | ||
![]() |
2d3e048f59 | ||
![]() |
024a40a799 | ||
![]() |
225c6b6f52 | ||
![]() |
0fee22e91a | ||
![]() |
96200a9ec4 | ||
![]() |
06c26315df | ||
![]() |
231810fe49 | ||
![]() |
60ef715c79 | ||
![]() |
8071dabe58 | ||
![]() |
3d9b160fce | ||
![]() |
5fa40ee631 | ||
![]() |
53814b1ea6 | ||
![]() |
9ca2a246a9 | ||
![]() |
4517425da8 | ||
![]() |
953ed46680 | ||
![]() |
795d3fe722 | ||
![]() |
536b40c06c | ||
![]() |
a113516588 | ||
![]() |
b68f9ea9e3 | ||
![]() |
fa6a32a39d | ||
![]() |
477635e00f | ||
![]() |
8e92cd3b2d | ||
![]() |
2ab53acce4 | ||
![]() |
2f4680d8f3 | ||
![]() |
1f1afeb06e | ||
![]() |
b2c075191e | ||
![]() |
1191c6453b | ||
![]() |
5d61122fd5 | ||
![]() |
e9c04879ce | ||
![]() |
5f4205a050 | ||
![]() |
7a46af3920 | ||
![]() |
ff967a8e65 | ||
![]() |
3108376d95 | ||
![]() |
494fe4403a | ||
![]() |
303b425fa3 | ||
![]() |
8f3efde55d | ||
![]() |
f16938bba9 | ||
![]() |
65ff04bc09 | ||
![]() |
e2715f94af | ||
![]() |
582160f566 | ||
![]() |
638f9839d5 | ||
![]() |
3f5bf4bd03 | ||
![]() |
038419663b | ||
![]() |
123f373537 | ||
![]() |
57121c828f | ||
![]() |
aa5edbc706 | ||
![]() |
f9d8c86b0a | ||
![]() |
b733fc5b83 | ||
![]() |
f2eaad62dc | ||
![]() |
adae788b18 | ||
![]() |
a65d10fcce | ||
![]() |
298f862fc0 | ||
![]() |
aff1580a1d | ||
![]() |
d4c0a42c1b | ||
![]() |
a9736a5bfb | ||
![]() |
2e4beeefb1 | ||
![]() |
635c5c9feb | ||
![]() |
4e1260e189 | ||
![]() |
ca3910748f | ||
![]() |
2f0796ff40 | ||
![]() |
cdba6dbc0e | ||
![]() |
f18daaeac5 | ||
![]() |
a9f124388f | ||
![]() |
4b2276608b | ||
![]() |
b352470ae1 | ||
![]() |
b3261a1b87 | ||
![]() |
6d5973dcb0 | ||
![]() |
74f17bb67e | ||
![]() |
bf024d3dce | ||
![]() |
5b87aff830 | ||
![]() |
34d35869fa | ||
![]() |
b1d1f1f461 | ||
![]() |
98f1e32e39 | ||
![]() |
edac875179 | ||
![]() |
0ff1a3b154 | ||
![]() |
bbe58cc37b | ||
![]() |
78dca6ee6e | ||
![]() |
12f26668a6 | ||
![]() |
6c90843aab | ||
![]() |
4aad4c64bb | ||
![]() |
5630c0108e | ||
![]() |
732a549cff | ||
![]() |
af637bc2a2 | ||
![]() |
6600916344 | ||
![]() |
e9cb95c5d4 | ||
![]() |
8617faa160 | ||
![]() |
fe888be58c | ||
![]() |
a38de1c242 | ||
![]() |
8dc0f6cfab | ||
![]() |
346db8a606 | ||
![]() |
b277a62e94 | ||
![]() |
e2535fcb28 | ||
![]() |
66e7b908ec | ||
![]() |
a2ba37be68 | ||
![]() |
004e5ff38b | ||
![]() |
4160fbecc0 | ||
![]() |
44844b7fbe | ||
![]() |
f60458d5fa | ||
![]() |
e242dd5bf2 | ||
![]() |
f863b331c1 | ||
![]() |
a98779770c | ||
![]() |
bcbcae2309 | ||
![]() |
5b8c10d08e | ||
![]() |
588fa4d90d | ||
![]() |
b9c777a578 | ||
![]() |
d8fe739aba | ||
![]() |
e5ddf475b8 | ||
![]() |
3c3c8668d6 | ||
![]() |
22d1b916bf | ||
![]() |
35b31f904d | ||
![]() |
7c05ec42fe | ||
![]() |
0e142e4e11 | ||
![]() |
ef16681dbf | ||
![]() |
45df1e1142 | ||
![]() |
0a49d8c930 | ||
![]() |
c896664b7a | ||
![]() |
c1663f2175 | ||
![]() |
75995fc4db | ||
![]() |
53a1c6162d | ||
![]() |
2bce56b348 | ||
![]() |
63222a512e | ||
![]() |
f152021521 | ||
![]() |
35319a414b | ||
![]() |
2083144058 | ||
![]() |
2f86ef95a1 | ||
![]() |
e5f4826964 | ||
![]() |
cd94593383 | ||
![]() |
5d7f68726d | ||
![]() |
a4c01c9c23 | ||
![]() |
e3349be39e | ||
![]() |
0e0432db6c | ||
![]() |
b43c28ccdf | ||
![]() |
0f3f49f9b9 | ||
![]() |
2880a290fc | ||
![]() |
e9c4769baf | ||
![]() |
e178f3cce5 | ||
![]() |
e09c3f7938 | ||
![]() |
30ed5d15b9 | ||
![]() |
28c9f617c2 | ||
![]() |
9537594971 | ||
![]() |
099b2fdba7 | ||
![]() |
9a3b5114db | ||
![]() |
34e1e18bcc | ||
![]() |
47201ace96 | ||
![]() |
e950846534 | ||
![]() |
7195862a51 | ||
![]() |
ca9bb48f3f | ||
![]() |
8239c119ca | ||
![]() |
c05ffa906d | ||
![]() |
30625e83b6 | ||
![]() |
4c9c2501f3 | ||
![]() |
3447cb0bd3 | ||
![]() |
985f7faad8 | ||
![]() |
3155542168 | ||
![]() |
756c93c66a | ||
![]() |
511517f491 | ||
![]() |
db66ab1852 | ||
![]() |
31f662a0f7 | ||
![]() |
7f69553230 | ||
![]() |
90d547dec0 | ||
![]() |
4587d092fd | ||
![]() |
bd11e00a59 | ||
![]() |
348991d1b3 | ||
![]() |
c35dfd4585 | ||
![]() |
baf701889a | ||
![]() |
b97cacad45 | ||
![]() |
acdd6faecf | ||
![]() |
ea4534d154 | ||
![]() |
dec237b7d6 | ||
![]() |
27d7d0b25c | ||
![]() |
f645479b5e | ||
![]() |
1aa14f3b6d | ||
![]() |
9f5ff82532 | ||
![]() |
3511e2e6af | ||
![]() |
127d4f37b8 | ||
![]() |
44a54e4590 | ||
![]() |
d4560ee015 | ||
![]() |
76d205feae | ||
![]() |
d852e5e0b4 | ||
![]() |
336f954056 | ||
![]() |
6837e0c7d3 | ||
![]() |
ee3ed8ab86 | ||
![]() |
c005cb89fc | ||
![]() |
771233ebcd | ||
![]() |
39082103a6 | ||
![]() |
a884949aba | ||
![]() |
d2f4698e3f | ||
![]() |
23c4e4cdb2 | ||
![]() |
13ff9a8ebd | ||
![]() |
73aae546e0 | ||
![]() |
174a6db19f | ||
![]() |
0e413a7efb | ||
![]() |
9769ad3a24 | ||
![]() |
8e773564b1 | ||
![]() |
fbcfb6d72e | ||
![]() |
3d9f2e3937 | ||
![]() |
2c7dd2b991 | ||
![]() |
0451a1c9e0 | ||
![]() |
c60b4175bb | ||
![]() |
f874a3aa7b | ||
![]() |
972f14d540 | ||
![]() |
f62b690599 | ||
![]() |
dc1f7ef0d0 | ||
![]() |
8b0129f29b | ||
![]() |
9548b7e442 | ||
![]() |
85f1bd172b | ||
![]() |
b6e4abb31e | ||
![]() |
8c079602c1 | ||
![]() |
974d504043 | ||
![]() |
c62ccf11ae | ||
![]() |
5301af33c0 | ||
![]() |
37ed1842ab | ||
![]() |
815bad3687 | ||
![]() |
6e34ea62c7 | ||
![]() |
7dfdee5709 | ||
![]() |
d48b70a5d3 | ||
![]() |
aad79e574a | ||
![]() |
b2d2315c39 | ||
![]() |
c218fa7b29 | ||
![]() |
dfc22950bd | ||
![]() |
4b52f7973d | ||
![]() |
b1fab92d1f | ||
![]() |
19165c436f | ||
![]() |
8d177f06da | ||
![]() |
ee66d601b4 | ||
![]() |
5e231fe07b | ||
![]() |
cb240eda70 | ||
![]() |
02bef62a66 | ||
![]() |
4d44c3e154 | ||
![]() |
cfd2c6e5e7 | ||
![]() |
d47d329ce7 | ||
![]() |
11436d5f00 | ||
![]() |
486b9e1c64 | ||
![]() |
18d6c715ac | ||
![]() |
6a1c70115a | ||
![]() |
9007a4803c | ||
![]() |
0873d1d8ab | ||
![]() |
edecc433ff | ||
![]() |
77b08cdbb0 | ||
![]() |
44b0466281 | ||
![]() |
179c530095 | ||
![]() |
c174a046e2 | ||
![]() |
67f8fa66ed |
2015 changed files with 105839 additions and 27215 deletions
.github
.gitignoreMakefileapp
vlinsert
datadog
elasticsearch
insertutil
internalinsert
journald
jsonline
loki
loki.goloki_json.goloki_json_test.goloki_json_timing_test.goloki_protobuf.goloki_protobuf_test.goloki_protobuf_timing_test.go
main.goopentelemetry
syslog
vlogscli
vlogsgenerator
vlselect
vlstorage
vmagent
README.md
csvimport
datadogsketches
datadogv1
datadogv2
graphite
influx
main.gonative
newrelic
opentelemetry
opentsdbhttp
prometheusimport
promremotewrite
remotewrite
client.goclient_test.gopendingseries.gorelabel.gorelabel_test.goremotewrite.goremotewrite_test.gostreamaggr.go
vmimport
vmalert-tool
vmalert
2
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
2
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
|
@ -8,7 +8,7 @@ body:
|
|||
Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade)
|
||||
to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
|
||||
and verify whether the bug is reproducible there.
|
||||
It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/troubleshooting/) first.
|
||||
It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) first.
|
||||
- type: textarea
|
||||
id: describe-the-bug
|
||||
attributes:
|
||||
|
|
6
.github/ISSUE_TEMPLATE/question.yml
vendored
6
.github/ISSUE_TEMPLATE/question.yml
vendored
|
@ -24,9 +24,9 @@ body:
|
|||
label: Troubleshooting docs
|
||||
description: I am familiar with the following troubleshooting docs
|
||||
options:
|
||||
- label: General - https://docs.victoriametrics.com/troubleshooting/
|
||||
- label: General - https://docs.victoriametrics.com/victoriametrics/troubleshooting/
|
||||
required: false
|
||||
- label: vmagent - https://docs.victoriametrics.com/vmagent/#troubleshooting
|
||||
- label: vmagent - https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
|
||||
required: false
|
||||
- label: vmalert - https://docs.victoriametrics.com/vmalert/#troubleshooting
|
||||
- label: vmalert - https://docs.victoriametrics.com/victoriametrics/vmalert/#troubleshooting
|
||||
required: false
|
||||
|
|
2
.github/pull_request_template.md
vendored
2
.github/pull_request_template.md
vendored
|
@ -6,4 +6,4 @@ Please provide a brief description of the changes you made. Be as specific as po
|
|||
|
||||
The following checks are **mandatory**:
|
||||
|
||||
- [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).
|
||||
- [ ] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).
|
||||
|
|
5
.github/workflows/docs.yaml
vendored
5
.github/workflows/docs.yaml
vendored
|
@ -40,9 +40,8 @@ jobs:
|
|||
- name: Copy docs
|
||||
id: update
|
||||
run: |
|
||||
rsync -zarv \
|
||||
--exclude="Makefile" \
|
||||
docs/ ../__vm-docs/content/victoriametrics
|
||||
find docs -type d -maxdepth 1 -mindepth 1 -exec \
|
||||
sh -c 'rsync -zarvh --delete {}/ ../__vm-docs/content/$(basename {})/' \;
|
||||
echo "SHORT_SHA=$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_OUTPUT
|
||||
working-directory: __vm
|
||||
|
||||
|
|
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -27,3 +27,4 @@ _site
|
|||
coverage.txt
|
||||
cspell.json
|
||||
*~
|
||||
deployment/docker/provisioning/plugins/
|
||||
|
|
24
Makefile
24
Makefile
|
@ -18,7 +18,7 @@ TAR_OWNERSHIP ?= --owner=1000 --group=1000
|
|||
.PHONY: $(MAKECMDGOALS)
|
||||
|
||||
include app/*/Makefile
|
||||
include cspell/Makefile
|
||||
include codespell/Makefile
|
||||
include docs/Makefile
|
||||
include deployment/*/Makefile
|
||||
include dashboards/Makefile
|
||||
|
@ -504,7 +504,7 @@ fmt:
|
|||
gofmt -l -w -s ./apptest
|
||||
|
||||
vet:
|
||||
go vet ./lib/...
|
||||
GOEXPERIMENT=synctest go vet ./lib/...
|
||||
go vet ./app/...
|
||||
go vet ./apptest/...
|
||||
|
||||
|
@ -513,35 +513,35 @@ check-all: fmt vet golangci-lint govulncheck
|
|||
clean-checkers: remove-golangci-lint remove-govulncheck
|
||||
|
||||
test:
|
||||
go test ./lib/... ./app/...
|
||||
GOEXPERIMENT=synctest go test ./lib/... ./app/...
|
||||
|
||||
test-race:
|
||||
go test -race ./lib/... ./app/...
|
||||
GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
|
||||
|
||||
test-pure:
|
||||
CGO_ENABLED=0 go test ./lib/... ./app/...
|
||||
GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
|
||||
|
||||
test-full:
|
||||
go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
|
||||
test-full-386:
|
||||
GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
|
||||
integration-test: victoria-metrics vmagent vmalert vmauth
|
||||
go test ./apptest/... -skip="^TestCluster.*"
|
||||
|
||||
benchmark:
|
||||
go test -bench=. ./lib/...
|
||||
GOEXPERIMENT=synctest go test -bench=. ./lib/...
|
||||
go test -bench=. ./app/...
|
||||
|
||||
benchmark-pure:
|
||||
CGO_ENABLED=0 go test -bench=. ./lib/...
|
||||
GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
|
||||
CGO_ENABLED=0 go test -bench=. ./app/...
|
||||
|
||||
vendor-update:
|
||||
go get -u ./lib/...
|
||||
go get -u ./app/...
|
||||
go mod tidy -compat=1.23
|
||||
go mod tidy -compat=1.24
|
||||
go mod vendor
|
||||
|
||||
app-local:
|
||||
|
@ -564,10 +564,10 @@ install-qtc:
|
|||
|
||||
|
||||
golangci-lint: install-golangci-lint
|
||||
golangci-lint run
|
||||
GOEXPERIMENT=synctest golangci-lint run
|
||||
|
||||
install-golangci-lint:
|
||||
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.5
|
||||
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.7
|
||||
|
||||
remove-golangci-lint:
|
||||
rm -rf `which golangci-lint`
|
||||
|
|
|
@ -3,7 +3,6 @@ package datadog
|
|||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
@ -11,20 +10,22 @@ import (
|
|||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
)
|
||||
|
||||
var (
|
||||
datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Datadog tags to be used as stream fields.")
|
||||
datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Datadog tags to ignore.")
|
||||
datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested via DataDog protocol. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#stream-fields")
|
||||
datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Comma-separated list of fields to ignore for logs ingested via DataDog protocol. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#dropping-fields")
|
||||
|
||||
maxRequestSize = flagutil.NewBytes("datadog.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single DataDog request")
|
||||
)
|
||||
|
||||
var parserPool fastjson.ParserPool
|
||||
|
@ -46,7 +47,6 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
|
|||
w.Header().Add("Content-Type", "application/json")
|
||||
startTime := time.Now()
|
||||
v2LogsRequestsTotal.Inc()
|
||||
reader := r.Body
|
||||
|
||||
var ts int64
|
||||
if tsValue := r.Header.Get("dd-message-timestamp"); tsValue != "" && tsValue != "0" {
|
||||
|
@ -61,25 +61,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
|
|||
ts = startTime.UnixNano()
|
||||
}
|
||||
|
||||
if r.Header.Get("Content-Encoding") == "gzip" {
|
||||
zr, err := common.GetGzipReader(reader)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read gzipped logs request: %s", err)
|
||||
return true
|
||||
}
|
||||
defer common.PutGzipReader(zr)
|
||||
reader = zr
|
||||
}
|
||||
|
||||
wcr := writeconcurrencylimiter.GetReader(reader)
|
||||
data, err := io.ReadAll(wcr)
|
||||
writeconcurrencylimiter.PutReader(wcr)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read request body: %s", err)
|
||||
return true
|
||||
}
|
||||
|
||||
cp, err := insertutils.GetCommonParams(r)
|
||||
cp, err := insertutil.GetCommonParams(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
|
@ -97,11 +79,15 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("datadog")
|
||||
err = readLogsRequest(ts, data, lmp)
|
||||
lmp.MustClose()
|
||||
encoding := r.Header.Get("Content-Encoding")
|
||||
err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
|
||||
lmp := cp.NewLogMessageProcessor("datadog", false)
|
||||
err := readLogsRequest(ts, data, lmp)
|
||||
lmp.MustClose()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf("cannot decode log message in /api/v2/logs request: %s, stream fields: %s", err, cp.StreamFields)
|
||||
httpserver.Errorf(w, r, "cannot read DataDog protocol data: %s", err)
|
||||
return true
|
||||
}
|
||||
|
||||
|
@ -121,7 +107,7 @@ var (
|
|||
// datadog message field has two formats:
|
||||
// - regular log message with string text
|
||||
// - nested json format for serverless plugins
|
||||
// which has folowing format:
|
||||
// which has the following format:
|
||||
// {"message": {"message": "text","lamdba": {"arn": "string","requestID": "string"}, "timestamp": int64} }
|
||||
//
|
||||
// See https://github.com/DataDog/datadog-lambda-extension/blob/28b90c7e4e985b72d60b5f5a5147c69c7ac693c4/bottlecap/src/logs/lambda/mod.rs#L24
|
||||
|
@ -184,7 +170,7 @@ func appendMsgFields(fields []logstorage.Field, v *fastjson.Value) ([]logstorage
|
|||
|
||||
// readLogsRequest parses data according to DataDog logs format
|
||||
// https://docs.datadoghq.com/api/latest/logs/#send-logs
|
||||
func readLogsRequest(ts int64, data []byte, lmp insertutils.LogMessageProcessor) error {
|
||||
func readLogsRequest(ts int64, data []byte, lmp insertutil.LogMessageProcessor) error {
|
||||
p := parserPool.Get()
|
||||
defer parserPool.Put(p)
|
||||
v, err := p.ParseBytes(data)
|
||||
|
|
|
@ -4,7 +4,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func TestReadLogsRequestFailure(t *testing.T) {
|
||||
|
@ -13,7 +13,7 @@ func TestReadLogsRequestFailure(t *testing.T) {
|
|||
|
||||
ts := time.Now().UnixNano()
|
||||
|
||||
lmp := &insertutils.TestLogMessageProcessor{}
|
||||
lmp := &insertutil.TestLogMessageProcessor{}
|
||||
if err := readLogsRequest(ts, []byte(data), lmp); err == nil {
|
||||
t.Fatalf("expecting non-empty error")
|
||||
}
|
||||
|
@ -37,7 +37,7 @@ func TestReadLogsRequestSuccess(t *testing.T) {
|
|||
for i := 0; i < rowsExpected; i++ {
|
||||
timestampsExpected = append(timestampsExpected, ts)
|
||||
}
|
||||
lmp := &insertutils.TestLogMessageProcessor{}
|
||||
lmp := &insertutil.TestLogMessageProcessor{}
|
||||
if err := readLogsRequest(ts, []byte(data), lmp); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
|
|
@ -10,14 +10,14 @@ import (
|
|||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
)
|
||||
|
||||
|
@ -90,7 +90,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
|||
startTime := time.Now()
|
||||
bulkRequestsTotal.Inc()
|
||||
|
||||
cp, err := insertutils.GetCommonParams(r)
|
||||
cp, err := insertutil.GetCommonParams(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
|
@ -99,10 +99,10 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
|||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
lmp := cp.NewLogMessageProcessor("elasticsearch_bulk")
|
||||
isGzip := r.Header.Get("Content-Encoding") == "gzip"
|
||||
lmp := cp.NewLogMessageProcessor("elasticsearch_bulk", true)
|
||||
encoding := r.Header.Get("Content-Encoding")
|
||||
streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
|
||||
n, err := readBulkRequest(streamName, r.Body, isGzip, cp.TimeField, cp.MsgFields, lmp)
|
||||
n, err := readBulkRequest(streamName, r.Body, encoding, cp.TimeField, cp.MsgFields, lmp)
|
||||
lmp.MustClose()
|
||||
if err != nil {
|
||||
logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
|
||||
|
@ -131,22 +131,19 @@ var (
|
|||
bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
|
||||
)
|
||||
|
||||
func readBulkRequest(streamName string, r io.Reader, isGzip bool, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (int, error) {
|
||||
func readBulkRequest(streamName string, r io.Reader, encoding string, timeField string, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {
|
||||
// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
|
||||
|
||||
if isGzip {
|
||||
zr, err := common.GetGzipReader(r)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot read gzipped _bulk request: %w", err)
|
||||
}
|
||||
defer common.PutGzipReader(zr)
|
||||
r = zr
|
||||
reader, err := protoparserutil.GetUncompressedReader(r, encoding)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot decode Elasticsearch protocol data: %w", err)
|
||||
}
|
||||
defer protoparserutil.PutUncompressedReader(reader)
|
||||
|
||||
wcr := writeconcurrencylimiter.GetReader(r)
|
||||
wcr := writeconcurrencylimiter.GetReader(reader)
|
||||
defer writeconcurrencylimiter.PutReader(wcr)
|
||||
|
||||
lr := insertutils.NewLineReader(streamName, wcr)
|
||||
lr := insertutil.NewLineReader(streamName, wcr)
|
||||
|
||||
n := 0
|
||||
for {
|
||||
|
@ -159,7 +156,7 @@ func readBulkRequest(streamName string, r io.Reader, isGzip bool, timeField stri
|
|||
}
|
||||
}
|
||||
|
||||
func readBulkLine(lr *insertutils.LineReader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (bool, error) {
|
||||
func readBulkLine(lr *insertutil.LineReader, timeField string, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
|
||||
var line []byte
|
||||
|
||||
// Read the command, must be "create" or "index"
|
||||
|
@ -231,7 +228,7 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
|
|||
}
|
||||
if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
|
||||
// Try parsing timestamp in seconds or milliseconds
|
||||
return insertutils.ParseUnixTimestamp(s)
|
||||
return insertutil.ParseUnixTimestamp(s)
|
||||
}
|
||||
if len(s) == len("YYYY-MM-DD") {
|
||||
t, err := time.Parse("2006-01-02", s)
|
||||
|
|
|
@ -2,20 +2,24 @@ package elasticsearch
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"github.com/golang/snappy"
|
||||
"github.com/klauspost/compress/gzip"
|
||||
"github.com/klauspost/compress/zlib"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func TestReadBulkRequest_Failure(t *testing.T) {
|
||||
f := func(data string) {
|
||||
t.Helper()
|
||||
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
r := bytes.NewBufferString(data)
|
||||
rows, err := readBulkRequest("test", r, false, "_time", []string{"_msg"}, tlp)
|
||||
rows, err := readBulkRequest("test", r, "", "_time", []string{"_msg"}, tlp)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-empty error")
|
||||
}
|
||||
|
@ -33,15 +37,15 @@ foobar`)
|
|||
}
|
||||
|
||||
func TestReadBulkRequest_Success(t *testing.T) {
|
||||
f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
|
||||
f := func(data, encoding, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
msgFields := []string{"non_existing_foo", msgField, "non_exiting_bar"}
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
|
||||
// Read the request without compression
|
||||
r := bytes.NewBufferString(data)
|
||||
rows, err := readBulkRequest("test", r, false, timeField, msgFields, tlp)
|
||||
rows, err := readBulkRequest("test", r, "", timeField, msgFields, tlp)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
@ -53,10 +57,12 @@ func TestReadBulkRequest_Success(t *testing.T) {
|
|||
}
|
||||
|
||||
// Read the request with compression
|
||||
tlp = &insertutils.TestLogMessageProcessor{}
|
||||
compressedData := compressData(data)
|
||||
r = bytes.NewBufferString(compressedData)
|
||||
rows, err = readBulkRequest("test", r, true, timeField, msgFields, tlp)
|
||||
tlp = &insertutil.TestLogMessageProcessor{}
|
||||
if encoding != "" {
|
||||
data = compressData(data, encoding)
|
||||
}
|
||||
r = bytes.NewBufferString(data)
|
||||
rows, err = readBulkRequest("test", r, encoding, timeField, msgFields, tlp)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
@ -69,9 +75,9 @@ func TestReadBulkRequest_Success(t *testing.T) {
|
|||
}
|
||||
|
||||
// Verify an empty data
|
||||
f("", "_time", "_msg", nil, "")
|
||||
f("\n", "_time", "_msg", nil, "")
|
||||
f("\n\n", "_time", "_msg", nil, "")
|
||||
f("", "gzip", "_time", "_msg", nil, "")
|
||||
f("\n", "gzip", "_time", "_msg", nil, "")
|
||||
f("\n\n", "gzip", "_time", "_msg", nil, "")
|
||||
|
||||
// Verify non-empty data
|
||||
data := `{"create":{"_index":"filebeat-8.8.0"}}
|
||||
|
@ -82,20 +88,35 @@ func TestReadBulkRequest_Success(t *testing.T) {
|
|||
{"message":"xyz","@timestamp":"1686026893735","x":"y"}
|
||||
{"create":{"_index":"filebeat-8.8.0"}}
|
||||
{"message":"qwe rty","@timestamp":"1686026893"}
|
||||
{"create":{"_index":"filebeat-8.8.0"}}
|
||||
{"message":"qwe rty float","@timestamp":"1686026123.62"}
|
||||
`
|
||||
timeField := "@timestamp"
|
||||
msgField := "message"
|
||||
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
|
||||
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000, 1686026123620000000}
|
||||
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"_msg":"baz"}
|
||||
{"_msg":"xyz","x":"y"}
|
||||
{"_msg":"qwe rty"}`
|
||||
f(data, timeField, msgField, timestampsExpected, resultExpected)
|
||||
{"_msg":"qwe rty"}
|
||||
{"_msg":"qwe rty float"}`
|
||||
f(data, "zstd", timeField, msgField, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
func compressData(s string) string {
|
||||
func compressData(s string, encoding string) string {
|
||||
var bb bytes.Buffer
|
||||
zw := gzip.NewWriter(&bb)
|
||||
var zw io.WriteCloser
|
||||
switch encoding {
|
||||
case "gzip":
|
||||
zw = gzip.NewWriter(&bb)
|
||||
case "zstd":
|
||||
zw, _ = zstd.NewWriter(&bb)
|
||||
case "snappy":
|
||||
zw = snappy.NewBufferedWriter(&bb)
|
||||
case "deflate":
|
||||
zw = zlib.NewWriter(&bb)
|
||||
default:
|
||||
panic(fmt.Errorf("%q encoding is not supported", encoding))
|
||||
}
|
||||
if _, err := zw.Write([]byte(s)); err != nil {
|
||||
panic(fmt.Errorf("unexpected error when compressing data: %w", err))
|
||||
}
|
||||
|
|
|
@ -5,20 +5,29 @@ import (
|
|||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
)
|
||||
|
||||
func BenchmarkReadBulkRequest(b *testing.B) {
|
||||
b.Run("gzip:off", func(b *testing.B) {
|
||||
benchmarkReadBulkRequest(b, false)
|
||||
b.Run("encoding:none", func(b *testing.B) {
|
||||
benchmarkReadBulkRequest(b, "")
|
||||
})
|
||||
b.Run("gzip:on", func(b *testing.B) {
|
||||
benchmarkReadBulkRequest(b, true)
|
||||
b.Run("encoding:gzip", func(b *testing.B) {
|
||||
benchmarkReadBulkRequest(b, "gzip")
|
||||
})
|
||||
b.Run("encoding:zstd", func(b *testing.B) {
|
||||
benchmarkReadBulkRequest(b, "zstd")
|
||||
})
|
||||
b.Run("encoding:deflate", func(b *testing.B) {
|
||||
benchmarkReadBulkRequest(b, "deflate")
|
||||
})
|
||||
b.Run("encoding:snappy", func(b *testing.B) {
|
||||
benchmarkReadBulkRequest(b, "snappy")
|
||||
})
|
||||
}
|
||||
|
||||
func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
|
||||
func benchmarkReadBulkRequest(b *testing.B, encoding string) {
|
||||
data := `{"create":{"_index":"filebeat-8.8.0"}}
|
||||
{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
|
||||
{"create":{"_index":"filebeat-8.8.0"}}
|
||||
|
@ -26,14 +35,14 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
|
|||
{"create":{"_index":"filebeat-8.8.0"}}
|
||||
{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
|
||||
`
|
||||
if isGzip {
|
||||
data = compressData(data)
|
||||
if encoding != "" {
|
||||
data = compressData(data, encoding)
|
||||
}
|
||||
dataBytes := bytesutil.ToUnsafeBytes(data)
|
||||
|
||||
timeField := "@timestamp"
|
||||
msgFields := []string{"message"}
|
||||
blp := &insertutils.BenchmarkLogMessageProcessor{}
|
||||
blp := &insertutil.BenchmarkLogMessageProcessor{}
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(data)))
|
||||
|
@ -41,7 +50,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
|
|||
r := &bytes.Reader{}
|
||||
for pb.Next() {
|
||||
r.Reset(dataBytes)
|
||||
_, err := readBulkRequest("test", r, isGzip, timeField, msgFields, blp)
|
||||
_, err := readBulkRequest("test", r, encoding, timeField, msgFields, blp)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
package insertutils
|
||||
package insertutil
|
||||
|
||||
import (
|
||||
"flag"
|
||||
|
@ -13,7 +13,7 @@ import (
|
|||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
|
@ -49,13 +49,13 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
|||
}
|
||||
|
||||
timeField := "_time"
|
||||
if tf := httputils.GetRequestValue(r, "_time_field", "VL-Time-Field"); tf != "" {
|
||||
if tf := httputil.GetRequestValue(r, "_time_field", "VL-Time-Field"); tf != "" {
|
||||
timeField = tf
|
||||
}
|
||||
|
||||
msgFields := httputils.GetArray(r, "_msg_field", "VL-Msg-Field")
|
||||
streamFields := httputils.GetArray(r, "_stream_fields", "VL-Stream-Fields")
|
||||
ignoreFields := httputils.GetArray(r, "ignore_fields", "VL-Ignore-Fields")
|
||||
msgFields := httputil.GetArray(r, "_msg_field", "VL-Msg-Field")
|
||||
streamFields := httputil.GetArray(r, "_stream_fields", "VL-Stream-Fields")
|
||||
ignoreFields := httputil.GetArray(r, "ignore_fields", "VL-Ignore-Fields")
|
||||
|
||||
extraFields, err := getExtraFields(r)
|
||||
if err != nil {
|
||||
|
@ -63,7 +63,7 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
|||
}
|
||||
|
||||
debug := false
|
||||
if dv := httputils.GetRequestValue(r, "debug", "VL-Debug"); dv != "" {
|
||||
if dv := httputil.GetRequestValue(r, "debug", "VL-Debug"); dv != "" {
|
||||
debug, err = strconv.ParseBool(dv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse debug=%q: %w", dv, err)
|
||||
|
@ -92,7 +92,7 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
|||
}
|
||||
|
||||
func getExtraFields(r *http.Request) ([]logstorage.Field, error) {
|
||||
efs := httputils.GetArray(r, "extra_fields", "VL-Extra-Fields")
|
||||
efs := httputil.GetArray(r, "extra_fields", "VL-Extra-Fields")
|
||||
if len(efs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
@ -141,7 +141,7 @@ type LogMessageProcessor interface {
|
|||
//
|
||||
// If streamFields is non-nil, then the given streamFields must be used as log stream fields instead of pre-configured fields.
|
||||
//
|
||||
// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
|
||||
// The LogMessageProcessor implementation cannot hold references to fields, since the caller can reuse them.
|
||||
AddRow(timestamp int64, fields, streamFields []logstorage.Field)
|
||||
|
||||
// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
|
||||
|
@ -191,9 +191,6 @@ func (lmp *logMessageProcessor) initPeriodicFlush() {
|
|||
//
|
||||
// If streamFields is non-nil, then it is used as log stream fields instead of the pre-configured stream fields.
|
||||
func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
|
||||
lmp.mu.Lock()
|
||||
defer lmp.mu.Unlock()
|
||||
|
||||
lmp.rowsIngestedTotal.Inc()
|
||||
n := logstorage.EstimatedJSONRowLen(fields)
|
||||
lmp.bytesIngestedTotal.Add(n)
|
||||
|
@ -205,7 +202,47 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
|
|||
return
|
||||
}
|
||||
|
||||
lmp.mu.Lock()
|
||||
defer lmp.mu.Unlock()
|
||||
|
||||
lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields, streamFields)
|
||||
|
||||
if lmp.cp.Debug {
|
||||
s := lmp.lr.GetRowString(0)
|
||||
lmp.lr.ResetKeepSettings()
|
||||
logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
|
||||
rowsDroppedTotalDebug.Inc()
|
||||
return
|
||||
}
|
||||
if lmp.lr.NeedFlush() {
|
||||
lmp.flushLocked()
|
||||
}
|
||||
}
|
||||
|
||||
// InsertRowProcessor is used by native data ingestion protocol parser.
|
||||
type InsertRowProcessor interface {
|
||||
// AddInsertRow must add r to the underlying storage.
|
||||
AddInsertRow(r *logstorage.InsertRow)
|
||||
}
|
||||
|
||||
// AddInsertRow adds r to lmp.
|
||||
func (lmp *logMessageProcessor) AddInsertRow(r *logstorage.InsertRow) {
|
||||
lmp.rowsIngestedTotal.Inc()
|
||||
n := logstorage.EstimatedJSONRowLen(r.Fields)
|
||||
lmp.bytesIngestedTotal.Add(n)
|
||||
|
||||
if len(r.Fields) > *MaxFieldsPerLine {
|
||||
line := logstorage.MarshalFieldsToJSON(nil, r.Fields)
|
||||
logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(r.Fields), *MaxFieldsPerLine, line)
|
||||
rowsDroppedTotalTooManyFields.Inc()
|
||||
return
|
||||
}
|
||||
|
||||
lmp.mu.Lock()
|
||||
defer lmp.mu.Unlock()
|
||||
|
||||
lmp.lr.MustAddInsertRow(r)
|
||||
|
||||
if lmp.cp.Debug {
|
||||
s := lmp.lr.GetRowString(0)
|
||||
lmp.lr.ResetKeepSettings()
|
||||
|
@ -238,7 +275,7 @@ func (lmp *logMessageProcessor) MustClose() {
|
|||
// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
|
||||
//
|
||||
// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
|
||||
func (cp *CommonParams) NewLogMessageProcessor(protocolName string) LogMessageProcessor {
|
||||
func (cp *CommonParams) NewLogMessageProcessor(protocolName string, isStreamMode bool) LogMessageProcessor {
|
||||
lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.ExtraFields, *defaultMsgValue)
|
||||
rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
|
||||
bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
|
||||
|
@ -251,7 +288,10 @@ func (cp *CommonParams) NewLogMessageProcessor(protocolName string) LogMessagePr
|
|||
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
lmp.initPeriodicFlush()
|
||||
|
||||
if isStreamMode {
|
||||
lmp.initPeriodicFlush()
|
||||
}
|
||||
|
||||
return lmp
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package insertutils
|
||||
package insertutil
|
||||
|
||||
import (
|
||||
"flag"
|
|
@ -1,4 +1,4 @@
|
|||
package insertutils
|
||||
package insertutil
|
||||
|
||||
import (
|
||||
"bytes"
|
|
@ -1,4 +1,4 @@
|
|||
package insertutils
|
||||
package insertutil
|
||||
|
||||
import (
|
||||
"bytes"
|
|
@ -1,4 +1,4 @@
|
|||
package insertutils
|
||||
package insertutil
|
||||
|
||||
import (
|
||||
"fmt"
|
|
@ -1,8 +1,10 @@
|
|||
package insertutils
|
||||
package insertutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
|
@ -49,6 +51,35 @@ func parseTimestamp(s string) (int64, error) {
|
|||
|
||||
// ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
|
||||
func ParseUnixTimestamp(s string) (int64, error) {
|
||||
if strings.IndexByte(s, '.') >= 0 {
|
||||
// Parse timestamp as floating-point value
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
|
||||
}
|
||||
if f < (1<<31) && f >= (-1<<31) {
|
||||
// The timestamp is in seconds.
|
||||
return int64(f * 1e9), nil
|
||||
}
|
||||
if f < 1e3*(1<<31) && f >= 1e3*(-1<<31) {
|
||||
// The timestamp is in milliseconds.
|
||||
return int64(f * 1e6), nil
|
||||
}
|
||||
if f < 1e6*(1<<31) && f >= 1e6*(-1<<31) {
|
||||
// The timestamp is in microseconds.
|
||||
return int64(f * 1e3), nil
|
||||
}
|
||||
// The timestamp is in nanoseconds
|
||||
if f > math.MaxInt64 {
|
||||
return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
|
||||
}
|
||||
if f < math.MinInt64 {
|
||||
return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
|
||||
}
|
||||
return int64(f), nil
|
||||
}
|
||||
|
||||
// Parse timestamp as integer
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
|
|
@ -1,4 +1,4 @@
|
|||
package insertutils
|
||||
package insertutil
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
@ -6,6 +6,63 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
)
|
||||
|
||||
func TestParseUnixTimestamp_Success(t *testing.T) {
|
||||
f := func(s string, timestampExpected int64) {
|
||||
t.Helper()
|
||||
|
||||
timestamp, err := ParseUnixTimestamp(s)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error in ParseUnixTimestamp(%q): %s", s, err)
|
||||
}
|
||||
if timestamp != timestampExpected {
|
||||
t.Fatalf("unexpected timestamp returned from ParseUnixTimestamp(%q); got %d; want %d", s, timestamp, timestampExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f("0", 0)
|
||||
|
||||
// nanoseconds
|
||||
f("-1234567890123456789", -1234567890123456789)
|
||||
f("1234567890123456789", 1234567890123456789)
|
||||
|
||||
// microseconds
|
||||
f("-1234567890123456", -1234567890123456000)
|
||||
f("1234567890123456", 1234567890123456000)
|
||||
f("1234567890123456.789", 1234567890123456768)
|
||||
|
||||
// milliseconds
|
||||
f("-1234567890123", -1234567890123000000)
|
||||
f("1234567890123", 1234567890123000000)
|
||||
f("1234567890123.456", 1234567890123456000)
|
||||
|
||||
// seconds
|
||||
f("-1234567890", -1234567890000000000)
|
||||
f("1234567890", 1234567890000000000)
|
||||
f("-1234567890.123456", -1234567890123456000)
|
||||
}
|
||||
|
||||
func TestParseUnixTimestamp_Failure(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
||||
_, err := ParseUnixTimestamp(s)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error in ParseUnixTimestamp(%q)", s)
|
||||
}
|
||||
}
|
||||
|
||||
// non-numeric timestamp
|
||||
f("")
|
||||
f("foobar")
|
||||
f("foo.bar")
|
||||
|
||||
// too big timestamp
|
||||
f("12345678901234567890")
|
||||
f("-12345678901234567890")
|
||||
f("12345678901234567890.235424")
|
||||
f("-12345678901234567890.235424")
|
||||
}
|
||||
|
||||
func TestExtractTimestampFromFields_Success(t *testing.T) {
|
||||
f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
|
||||
t.Helper()
|
96
app/vlinsert/internalinsert/internalinsert.go
Normal file
96
app/vlinsert/internalinsert/internalinsert.go
Normal file
|
@ -0,0 +1,96 @@
|
|||
package internalinsert
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
)
|
||||
|
||||
var (
|
||||
disableInsert = flag.Bool("internalinsert.disable", false, "Whether to disable /internal/insert HTTP endpoint")
|
||||
maxRequestSize = flagutil.NewBytes("internalinsert.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single request, which can be accepted at /internal/insert HTTP endpoint")
|
||||
)
|
||||
|
||||
// RequestHandler processes /internal/insert requests.
|
||||
func RequestHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if *disableInsert {
|
||||
httpserver.Errorf(w, r, "requests to /internal/insert are disabled with -internalinsert.disable command-line flag")
|
||||
return
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
if r.Method != "POST" {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
version := r.FormValue("version")
|
||||
if version != netinsert.ProtocolVersion {
|
||||
httpserver.Errorf(w, r, "unsupported protocol version=%q; want %q", version, netinsert.ProtocolVersion)
|
||||
return
|
||||
}
|
||||
|
||||
requestsTotal.Inc()
|
||||
|
||||
cp, err := insertutil.GetCommonParams(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
|
||||
encoding := r.Header.Get("Content-Encoding")
|
||||
err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
|
||||
lmp := cp.NewLogMessageProcessor("internalinsert", false)
|
||||
irp := lmp.(insertutil.InsertRowProcessor)
|
||||
err := parseData(irp, data)
|
||||
lmp.MustClose()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
errorsTotal.Inc()
|
||||
httpserver.Errorf(w, r, "cannot parse internal insert request: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
requestDuration.UpdateDuration(startTime)
|
||||
}
|
||||
|
||||
func parseData(irp insertutil.InsertRowProcessor, data []byte) error {
|
||||
r := logstorage.GetInsertRow()
|
||||
src := data
|
||||
i := 0
|
||||
for len(src) > 0 {
|
||||
tail, err := r.UnmarshalInplace(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse row #%d: %s", i, err)
|
||||
}
|
||||
src = tail
|
||||
i++
|
||||
|
||||
irp.AddInsertRow(r)
|
||||
}
|
||||
logstorage.PutInsertRow(r)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
|
||||
|
||||
requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/internal/insert"}`)
|
||||
)
|
|
@ -5,7 +5,6 @@ import (
|
|||
"encoding/binary"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"slices"
|
||||
|
@ -13,39 +12,37 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
const (
|
||||
journaldEntryMaxNameLen = 64
|
||||
)
|
||||
// See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
|
||||
const journaldEntryMaxNameLen = 64
|
||||
|
||||
var allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*`)
|
||||
|
||||
var (
|
||||
bodyBufferPool bytesutil.ByteBufferPool
|
||||
allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*`)
|
||||
)
|
||||
|
||||
var (
|
||||
journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Journal fields to be used as stream fields. "+
|
||||
"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
|
||||
journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Journal fields to ignore. "+
|
||||
"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
|
||||
journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Journal field to be used as time field. "+
|
||||
"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
|
||||
journaldTenantID = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint.")
|
||||
journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#stream-fields")
|
||||
journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Comma-separated list of fields to ignore for logs ingested over journald protocol. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#dropping-fields")
|
||||
journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Field to use as a log timestamp for logs ingested via journald protocol. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#time-field")
|
||||
journaldTenantID = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#multitenancy")
|
||||
journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include journal entry fields, which with double underscores.")
|
||||
|
||||
maxRequestSize = flagutil.NewBytes("journald.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single journald request")
|
||||
)
|
||||
|
||||
func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
|
||||
cp, err := insertutils.GetCommonParams(r)
|
||||
func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
|
||||
cp, err := insertutil.GetCommonParams(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -89,43 +86,29 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
|
|||
startTime := time.Now()
|
||||
requestsJournaldTotal.Inc()
|
||||
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
|
||||
reader := r.Body
|
||||
var err error
|
||||
|
||||
wcr := writeconcurrencylimiter.GetReader(reader)
|
||||
data, err := io.ReadAll(wcr)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read request body: %s", err)
|
||||
return
|
||||
}
|
||||
writeconcurrencylimiter.PutReader(wcr)
|
||||
bb := bodyBufferPool.Get()
|
||||
defer bodyBufferPool.Put(bb)
|
||||
if r.Header.Get("Content-Encoding") == "zstd" {
|
||||
bb.B, err = zstd.Decompress(bb.B[:0], data)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot decompress zstd-encoded request with length %d: %s", len(data), err)
|
||||
return
|
||||
}
|
||||
data = bb.B
|
||||
}
|
||||
cp, err := getCommonParams(r)
|
||||
if err != nil {
|
||||
errorsTotal.Inc()
|
||||
httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("journald")
|
||||
err = parseJournaldRequest(data, lmp, cp)
|
||||
lmp.MustClose()
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
errorsTotal.Inc()
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
|
||||
encoding := r.Header.Get("Content-Encoding")
|
||||
err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
|
||||
lmp := cp.NewLogMessageProcessor("journald", false)
|
||||
err := parseJournaldRequest(data, lmp, cp)
|
||||
lmp.MustClose()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
errorsTotal.Inc()
|
||||
httpserver.Errorf(w, r, "cannot parse Journald protobuf request: %s", err)
|
||||
httpserver.Errorf(w, r, "cannot read journald protocol data: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -148,7 +131,7 @@ var (
|
|||
)
|
||||
|
||||
// See https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
|
||||
func parseJournaldRequest(data []byte, lmp insertutils.LogMessageProcessor, cp *insertutils.CommonParams) error {
|
||||
func parseJournaldRequest(data []byte, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
|
||||
var fields []logstorage.Field
|
||||
var ts int64
|
||||
var size uint64
|
||||
|
@ -198,7 +181,7 @@ func parseJournaldRequest(data []byte, lmp insertutils.LogMessageProcessor, cp *
|
|||
if err != nil {
|
||||
return fmt.Errorf("failed to extract binary field %q value size: %w", name, err)
|
||||
}
|
||||
// skip binary data sise
|
||||
// skip binary data size
|
||||
data = data[idx:]
|
||||
if size == 0 {
|
||||
return fmt.Errorf("unexpected zero binary data size decoded %d", size)
|
||||
|
@ -218,7 +201,6 @@ func parseJournaldRequest(data []byte, lmp insertutils.LogMessageProcessor, cp *
|
|||
}
|
||||
data = data[1:]
|
||||
}
|
||||
// https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
|
||||
if len(name) > journaldEntryMaxNameLen {
|
||||
return fmt.Errorf("journald entry name should not exceed %d symbols, got: %q", journaldEntryMaxNameLen, name)
|
||||
}
|
||||
|
|
|
@ -3,14 +3,14 @@ package journald
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func TestPushJournaldOk(t *testing.T) {
|
||||
f := func(src string, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
cp := &insertutils.CommonParams{
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
cp := &insertutil.CommonParams{
|
||||
TimeField: "__REALTIME_TIMESTAMP",
|
||||
MsgFields: []string{"MESSAGE"},
|
||||
}
|
||||
|
@ -44,8 +44,8 @@ func TestPushJournaldOk(t *testing.T) {
|
|||
func TestPushJournald_Failure(t *testing.T) {
|
||||
f := func(data string) {
|
||||
t.Helper()
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
cp := &insertutils.CommonParams{
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
cp := &insertutil.CommonParams{
|
||||
TimeField: "__REALTIME_TIMESTAMP",
|
||||
MsgFields: []string{"MESSAGE"},
|
||||
}
|
||||
|
|
|
@ -6,12 +6,12 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -28,7 +28,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
requestsTotal.Inc()
|
||||
|
||||
cp, err := insertutils.GetCommonParams(r)
|
||||
cp, err := insertutil.GetCommonParams(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
|
@ -38,18 +38,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
reader := r.Body
|
||||
if r.Header.Get("Content-Encoding") == "gzip" {
|
||||
zr, err := common.GetGzipReader(reader)
|
||||
if err != nil {
|
||||
logger.Errorf("cannot read gzipped jsonline request: %s", err)
|
||||
return
|
||||
}
|
||||
defer common.PutGzipReader(zr)
|
||||
reader = zr
|
||||
encoding := r.Header.Get("Content-Encoding")
|
||||
reader, err := protoparserutil.GetUncompressedReader(r.Body, encoding)
|
||||
if err != nil {
|
||||
logger.Errorf("cannot decode jsonline request: %s", err)
|
||||
return
|
||||
}
|
||||
defer protoparserutil.PutUncompressedReader(reader)
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("jsonline")
|
||||
lmp := cp.NewLogMessageProcessor("jsonline", true)
|
||||
streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
|
||||
processStreamInternal(streamName, reader, cp.TimeField, cp.MsgFields, lmp)
|
||||
lmp.MustClose()
|
||||
|
@ -57,11 +54,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
|
|||
requestDuration.UpdateDuration(startTime)
|
||||
}
|
||||
|
||||
func processStreamInternal(streamName string, r io.Reader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) {
|
||||
func processStreamInternal(streamName string, r io.Reader, timeField string, msgFields []string, lmp insertutil.LogMessageProcessor) {
|
||||
wcr := writeconcurrencylimiter.GetReader(r)
|
||||
defer writeconcurrencylimiter.PutReader(wcr)
|
||||
|
||||
lr := insertutils.NewLineReader(streamName, wcr)
|
||||
lr := insertutil.NewLineReader(streamName, wcr)
|
||||
|
||||
n := 0
|
||||
for {
|
||||
|
@ -78,7 +75,7 @@ func processStreamInternal(streamName string, r io.Reader, timeField string, msg
|
|||
}
|
||||
}
|
||||
|
||||
func readLine(lr *insertutils.LineReader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (bool, error) {
|
||||
func readLine(lr *insertutil.LineReader, timeField string, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
|
||||
var line []byte
|
||||
for len(line) == 0 {
|
||||
if !lr.NextLine() {
|
||||
|
@ -94,7 +91,7 @@ func readLine(lr *insertutils.LineReader, timeField string, msgFields []string,
|
|||
if err := p.ParseLogMessage(line); err != nil {
|
||||
return true, fmt.Errorf("cannot parse json-encoded line: %w; line contents: %q", err, line)
|
||||
}
|
||||
ts, err := insertutils.ExtractTimestampFromFields(timeField, p.Fields)
|
||||
ts, err := insertutil.ExtractTimestampFromFields(timeField, p.Fields)
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("cannot get timestamp from json-encoded line: %w; line contents: %q", err, line)
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ import (
|
|||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func TestProcessStreamInternal(t *testing.T) {
|
||||
|
@ -12,7 +12,7 @@ func TestProcessStreamInternal(t *testing.T) {
|
|||
t.Helper()
|
||||
|
||||
msgFields := []string{msgField}
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
r := bytes.NewBufferString(data)
|
||||
processStreamInternal("test", r, timeField, msgFields, tlp)
|
||||
|
||||
|
|
|
@ -1,12 +1,18 @@
|
|||
package loki
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
)
|
||||
|
||||
var disableMessageParsing = flag.Bool("loki.disableMessageParsing", false, "Whether to disable automatic parsing of JSON-encoded log fields inside Loki log message into distinct log fields")
|
||||
|
||||
// RequestHandler processes Loki insert requests
|
||||
func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
switch path {
|
||||
|
@ -35,8 +41,17 @@ func handleInsert(r *http.Request, w http.ResponseWriter) {
|
|||
}
|
||||
}
|
||||
|
||||
func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
|
||||
cp, err := insertutils.GetCommonParams(r)
|
||||
type commonParams struct {
|
||||
cp *insertutil.CommonParams
|
||||
|
||||
// Whether to parse JSON inside plaintext log message.
|
||||
//
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8486
|
||||
parseMessage bool
|
||||
}
|
||||
|
||||
func getCommonParams(r *http.Request) (*commonParams, error) {
|
||||
cp, err := insertutil.GetCommonParams(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -55,5 +70,17 @@ func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
|
|||
|
||||
}
|
||||
|
||||
return cp, nil
|
||||
parseMessage := !*disableMessageParsing
|
||||
if rv := httputil.GetRequestValue(r, "disable_message_parsing", "VL-Loki-Disable-Message-Parsing"); rv != "" {
|
||||
bv, err := strconv.ParseBool(rv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse dusable_message_parsing=%q: %s", rv, err)
|
||||
}
|
||||
parseMessage = !bv
|
||||
}
|
||||
|
||||
return &commonParams{
|
||||
cp: cp,
|
||||
parseMessage: parseMessage,
|
||||
}, nil
|
||||
}
|
||||
|
|
|
@ -2,47 +2,28 @@ package loki
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
)
|
||||
|
||||
var maxRequestSize = flagutil.NewBytes("loki.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single Loki request")
|
||||
|
||||
var parserPool fastjson.ParserPool
|
||||
|
||||
func handleJSON(r *http.Request, w http.ResponseWriter) {
|
||||
startTime := time.Now()
|
||||
requestsJSONTotal.Inc()
|
||||
reader := r.Body
|
||||
if r.Header.Get("Content-Encoding") == "gzip" {
|
||||
zr, err := common.GetGzipReader(reader)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
|
||||
return
|
||||
}
|
||||
defer common.PutGzipReader(zr)
|
||||
reader = zr
|
||||
}
|
||||
|
||||
wcr := writeconcurrencylimiter.GetReader(reader)
|
||||
data, err := io.ReadAll(wcr)
|
||||
writeconcurrencylimiter.PutReader(wcr)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read request body: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
cp, err := getCommonParams(r)
|
||||
if err != nil {
|
||||
|
@ -53,12 +34,17 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
|
|||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
lmp := cp.NewLogMessageProcessor("loki_json")
|
||||
useDefaultStreamFields := len(cp.StreamFields) == 0
|
||||
err = parseJSONRequest(data, lmp, useDefaultStreamFields)
|
||||
lmp.MustClose()
|
||||
|
||||
encoding := r.Header.Get("Content-Encoding")
|
||||
err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
|
||||
lmp := cp.cp.NewLogMessageProcessor("loki_json", false)
|
||||
useDefaultStreamFields := len(cp.cp.StreamFields) == 0
|
||||
err := parseJSONRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
|
||||
lmp.MustClose()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse Loki json request: %s; data=%s", err, data)
|
||||
httpserver.Errorf(w, r, "cannot read Loki json data: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -66,6 +52,9 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
|
|||
// There is no need in updating requestJSONDuration for request errors,
|
||||
// since their timings are usually much smaller than the timing for successful request parsing.
|
||||
requestJSONDuration.UpdateDuration(startTime)
|
||||
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
var (
|
||||
|
@ -73,9 +62,10 @@ var (
|
|||
requestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
)
|
||||
|
||||
func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
|
||||
func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
|
||||
p := parserPool.Get()
|
||||
defer parserPool.Put(p)
|
||||
|
||||
v, err := p.ParseBytes(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse JSON request body: %w", err)
|
||||
|
@ -90,11 +80,20 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
|
|||
return fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
|
||||
}
|
||||
|
||||
fields := getFields()
|
||||
defer putFields(fields)
|
||||
|
||||
var msgParser *logstorage.JSONParser
|
||||
if parseMessage {
|
||||
msgParser = logstorage.GetJSONParser()
|
||||
defer logstorage.PutJSONParser(msgParser)
|
||||
}
|
||||
|
||||
currentTimestamp := time.Now().UnixNano()
|
||||
var commonFields []logstorage.Field
|
||||
|
||||
for _, stream := range streams {
|
||||
// populate common labels from `stream` dict
|
||||
commonFields = commonFields[:0]
|
||||
fields.fields = fields.fields[:0]
|
||||
labelsV := stream.Get("stream")
|
||||
var labels *fastjson.Object
|
||||
if labelsV != nil {
|
||||
|
@ -110,7 +109,7 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
|
|||
err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
|
||||
return
|
||||
}
|
||||
commonFields = append(commonFields, logstorage.Field{
|
||||
fields.fields = append(fields.fields, logstorage.Field{
|
||||
Name: bytesutil.ToUnsafeString(k),
|
||||
Value: bytesutil.ToUnsafeString(vStr),
|
||||
})
|
||||
|
@ -129,8 +128,10 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
|
|||
return fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
|
||||
}
|
||||
|
||||
fields := commonFields
|
||||
commonFieldsLen := len(fields.fields)
|
||||
for _, line := range lines {
|
||||
fields.fields = fields.fields[:commonFieldsLen]
|
||||
|
||||
lineA, err := line.Array()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
|
||||
|
@ -152,17 +153,6 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
|
|||
ts = currentTimestamp
|
||||
}
|
||||
|
||||
// parse log message
|
||||
msg, err := lineA[1].StringBytes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
|
||||
}
|
||||
|
||||
fields = append(fields[:len(commonFields)], logstorage.Field{
|
||||
Name: "_msg",
|
||||
Value: bytesutil.ToUnsafeString(msg),
|
||||
})
|
||||
|
||||
// parse structured metadata - see https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
|
||||
if len(lineA) > 2 {
|
||||
structuredMetadata, err := lineA[2].Object()
|
||||
|
@ -177,7 +167,7 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
|
|||
return
|
||||
}
|
||||
|
||||
fields = append(fields, logstorage.Field{
|
||||
fields.fields = append(fields.fields, logstorage.Field{
|
||||
Name: bytesutil.ToUnsafeString(k),
|
||||
Value: bytesutil.ToUnsafeString(vStr),
|
||||
})
|
||||
|
@ -186,39 +176,51 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
|
|||
return fmt.Errorf("error when parsing `structuredMetadata` object: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// parse log message
|
||||
msg, err := lineA[1].StringBytes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
|
||||
}
|
||||
allowMsgRenaming := false
|
||||
fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, bytesutil.ToUnsafeString(msg))
|
||||
|
||||
var streamFields []logstorage.Field
|
||||
if useDefaultStreamFields {
|
||||
streamFields = commonFields
|
||||
streamFields = fields.fields[:commonFieldsLen]
|
||||
}
|
||||
lmp.AddRow(ts, fields, streamFields)
|
||||
if allowMsgRenaming {
|
||||
logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
|
||||
}
|
||||
lmp.AddRow(ts, fields.fields, streamFields)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func addMsgField(dst []logstorage.Field, msgParser *logstorage.JSONParser, msg string) ([]logstorage.Field, bool) {
|
||||
if msgParser == nil || len(msg) < 2 || msg[0] != '{' || msg[len(msg)-1] != '}' {
|
||||
return append(dst, logstorage.Field{
|
||||
Name: "_msg",
|
||||
Value: msg,
|
||||
}), false
|
||||
}
|
||||
if msgParser != nil && len(msg) >= 2 && msg[0] == '{' && msg[len(msg)-1] == '}' {
|
||||
if err := msgParser.ParseLogMessage(bytesutil.ToUnsafeBytes(msg)); err == nil {
|
||||
return append(dst, msgParser.Fields...), true
|
||||
}
|
||||
}
|
||||
return append(dst, logstorage.Field{
|
||||
Name: "_msg",
|
||||
Value: msg,
|
||||
}), false
|
||||
}
|
||||
|
||||
func parseLokiTimestamp(s string) (int64, error) {
|
||||
if s == "" {
|
||||
// Special case - an empty timestamp must be substituted with the current time by the caller.
|
||||
return 0, nil
|
||||
}
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
// Fall back to parsing floating-point value
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if f > math.MaxInt64 {
|
||||
return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
|
||||
}
|
||||
if f < math.MinInt64 {
|
||||
return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
|
||||
}
|
||||
n = int64(f)
|
||||
}
|
||||
if n < 0 {
|
||||
return 0, fmt.Errorf("too small timestamp in nanoseconds: %d; must be bigger than 0", n)
|
||||
}
|
||||
return n, nil
|
||||
return insertutil.ParseUnixTimestamp(s)
|
||||
}
|
||||
|
|
|
@ -3,15 +3,15 @@ package loki
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func TestParseJSONRequest_Failure(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
if err := parseJSONRequest([]byte(s), tlp, false); err == nil {
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err == nil {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
}
|
||||
if err := tlp.Verify(nil, ""); err != nil {
|
||||
|
@ -63,9 +63,9 @@ func TestParseJSONRequest_Success(t *testing.T) {
|
|||
f := func(s string, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
|
||||
if err := parseJSONRequest([]byte(s), tlp, false); err != nil {
|
||||
if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
|
@ -89,9 +89,9 @@ func TestParseJSONRequest_Success(t *testing.T) {
|
|||
"label2": "value2"
|
||||
},"values":[
|
||||
["1577836800000000001", "foo bar"],
|
||||
["1477836900005000002", "abc"],
|
||||
["1686026123.62", "abc"],
|
||||
["147.78369e9", "foobar"]
|
||||
]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
|
||||
]}]}`, []int64{1577836800000000001, 1686026123620000000, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
|
||||
{"label1":"value1","label2":"value2","_msg":"abc"}
|
||||
{"label1":"value1","label2":"value2","_msg":"foobar"}`)
|
||||
|
||||
|
@ -122,6 +122,48 @@ func TestParseJSONRequest_Success(t *testing.T) {
|
|||
{"x":"y","_msg":"yx"}`)
|
||||
|
||||
// values with metadata
|
||||
f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar","metadata_1":"md_value"}`)
|
||||
f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"metadata_1":"md_value","_msg":"foo bar"}`)
|
||||
f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
|
||||
}
|
||||
|
||||
func TestParseJSONRequest_ParseMessage(t *testing.T) {
|
||||
f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
|
||||
if err := parseJSONRequest([]byte(s), tlp, msgFields, false, true); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
f(`{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"foo": "bar",
|
||||
"a": "b"
|
||||
},
|
||||
"values": [
|
||||
["1577836800000000001", "{\"user_id\":\"123\"}"],
|
||||
["1577836900005000002", "abc", {"trace_id":"pqw"}],
|
||||
["1577836900005000003", "{def}"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"x": "y"
|
||||
},
|
||||
"values": [
|
||||
["1877836900005000004", "{\"trace_id\":\"111\",\"parent_id\":\"abc\"}"]
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
|
||||
{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
|
||||
{"foo":"bar","a":"b","_msg":"{def}"}
|
||||
{"x":"y","_msg":"111","parent_id":"abc"}`)
|
||||
}
|
||||
|
|
|
@ -6,7 +6,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func BenchmarkParseJSONRequest(b *testing.B) {
|
||||
|
@ -22,13 +22,13 @@ func BenchmarkParseJSONRequest(b *testing.B) {
|
|||
}
|
||||
|
||||
func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
|
||||
blp := &insertutils.BenchmarkLogMessageProcessor{}
|
||||
blp := &insertutil.BenchmarkLogMessageProcessor{}
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(streams * rows))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
data := getJSONBody(streams, rows, labels)
|
||||
for pb.Next() {
|
||||
if err := parseJSONRequest(data, blp, false); err != nil {
|
||||
if err := parseJSONRequest(data, blp, nil, false, true); err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,38 +2,27 @@ package loki
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/golang/snappy"
|
||||
)
|
||||
|
||||
var (
|
||||
bytesBufPool bytesutil.ByteBufferPool
|
||||
pushReqsPool sync.Pool
|
||||
)
|
||||
|
||||
func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
||||
startTime := time.Now()
|
||||
requestsProtobufTotal.Inc()
|
||||
wcr := writeconcurrencylimiter.GetReader(r.Body)
|
||||
data, err := io.ReadAll(wcr)
|
||||
writeconcurrencylimiter.PutReader(wcr)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read request body: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
cp, err := getCommonParams(r)
|
||||
if err != nil {
|
||||
|
@ -44,12 +33,22 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
|||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
lmp := cp.NewLogMessageProcessor("loki_protobuf")
|
||||
useDefaultStreamFields := len(cp.StreamFields) == 0
|
||||
err = parseProtobufRequest(data, lmp, useDefaultStreamFields)
|
||||
lmp.MustClose()
|
||||
|
||||
encoding := r.Header.Get("Content-Encoding")
|
||||
if encoding == "" {
|
||||
// Loki protocol uses snappy compression by default.
|
||||
// See https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
|
||||
encoding = "snappy"
|
||||
}
|
||||
err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
|
||||
lmp := cp.cp.NewLogMessageProcessor("loki_protobuf", false)
|
||||
useDefaultStreamFields := len(cp.cp.StreamFields) == 0
|
||||
err := parseProtobufRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
|
||||
lmp.MustClose()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
|
||||
httpserver.Errorf(w, r, "cannot read Loki protobuf data: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -57,6 +56,9 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
|||
// There is no need in updating requestProtobufDuration for request errors,
|
||||
// since their timings are usually much smaller than the timing for successful request parsing.
|
||||
requestProtobufDuration.UpdateDuration(startTime)
|
||||
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
var (
|
||||
|
@ -64,20 +66,11 @@ var (
|
|||
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
)
|
||||
|
||||
func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
|
||||
bb := bytesBufPool.Get()
|
||||
defer bytesBufPool.Put(bb)
|
||||
|
||||
buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
|
||||
}
|
||||
bb.B = buf
|
||||
|
||||
func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
|
||||
req := getPushRequest()
|
||||
defer putPushRequest(req)
|
||||
|
||||
err = req.UnmarshalProtobuf(bb.B)
|
||||
err := req.UnmarshalProtobuf(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse request body: %w", err)
|
||||
}
|
||||
|
@ -85,8 +78,15 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
|
|||
fields := getFields()
|
||||
defer putFields(fields)
|
||||
|
||||
var msgParser *logstorage.JSONParser
|
||||
if parseMessage {
|
||||
msgParser = logstorage.GetJSONParser()
|
||||
defer logstorage.PutJSONParser(msgParser)
|
||||
}
|
||||
|
||||
streams := req.Streams
|
||||
currentTimestamp := time.Now().UnixNano()
|
||||
|
||||
for i := range streams {
|
||||
stream := &streams[i]
|
||||
// st.Labels contains labels for the stream.
|
||||
|
@ -109,10 +109,8 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
|
|||
})
|
||||
}
|
||||
|
||||
fields.fields = append(fields.fields, logstorage.Field{
|
||||
Name: "_msg",
|
||||
Value: e.Line,
|
||||
})
|
||||
allowMsgRenaming := false
|
||||
fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, e.Line)
|
||||
|
||||
ts := e.Timestamp.UnixNano()
|
||||
if ts == 0 {
|
||||
|
@ -123,6 +121,9 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
|
|||
if useDefaultStreamFields {
|
||||
streamFields = fields.fields[:commonFieldsLen]
|
||||
}
|
||||
if allowMsgRenaming {
|
||||
logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
|
||||
}
|
||||
lmp.AddRow(ts, fields.fields, streamFields)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,9 +6,8 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/golang/snappy"
|
||||
)
|
||||
|
||||
type testLogMessageProcessor struct {
|
||||
|
@ -53,7 +52,7 @@ func TestParseProtobufRequest_Success(t *testing.T) {
|
|||
t.Helper()
|
||||
|
||||
tlp := &testLogMessageProcessor{}
|
||||
if err := parseJSONRequest([]byte(s), tlp, false); err != nil {
|
||||
if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if len(tlp.pr.Streams) != len(timestampsExpected) {
|
||||
|
@ -61,10 +60,9 @@ func TestParseProtobufRequest_Success(t *testing.T) {
|
|||
}
|
||||
|
||||
data := tlp.pr.MarshalProtobuf(nil)
|
||||
encodedData := snappy.Encode(nil, data)
|
||||
|
||||
tlp2 := &insertutils.TestLogMessageProcessor{}
|
||||
if err := parseProtobufRequest(encodedData, tlp2, false); err != nil {
|
||||
tlp2 := &insertutil.TestLogMessageProcessor{}
|
||||
if err := parseProtobufRequest(data, tlp2, nil, false, false); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
|
@ -90,7 +88,7 @@ func TestParseProtobufRequest_Success(t *testing.T) {
|
|||
["1577836800000000001", "foo bar"],
|
||||
["1477836900005000002", "abc"],
|
||||
["147.78369e9", "foobar"]
|
||||
]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
|
||||
]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
|
||||
{"label1":"value1","label2":"value2","_msg":"abc"}
|
||||
{"label1":"value1","label2":"value2","_msg":"foobar"}`)
|
||||
|
||||
|
@ -121,6 +119,57 @@ func TestParseProtobufRequest_Success(t *testing.T) {
|
|||
{"x":"y","_msg":"yx"}`)
|
||||
}
|
||||
|
||||
func TestParseProtobufRequest_ParseMessage(t *testing.T) {
|
||||
f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
tlp := &testLogMessageProcessor{}
|
||||
if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if len(tlp.pr.Streams) != len(timestampsExpected) {
|
||||
t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
|
||||
}
|
||||
|
||||
data := tlp.pr.MarshalProtobuf(nil)
|
||||
|
||||
tlp2 := &insertutil.TestLogMessageProcessor{}
|
||||
if err := parseProtobufRequest(data, tlp2, msgFields, false, true); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
f(`{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"foo": "bar",
|
||||
"a": "b"
|
||||
},
|
||||
"values": [
|
||||
["1577836800000000001", "{\"user_id\":\"123\"}"],
|
||||
["1577836900005000002", "abc", {"trace_id":"pqw"}],
|
||||
["1577836900005000003", "{def}"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"x": "y"
|
||||
},
|
||||
"values": [
|
||||
["1877836900005000004", "{\"trace_id\":\"432\",\"parent_id\":\"qwerty\"}"]
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
|
||||
{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
|
||||
{"foo":"bar","a":"b","_msg":"{def}"}
|
||||
{"x":"y","_msg":"432","parent_id":"qwerty"}`)
|
||||
}
|
||||
|
||||
func TestParsePromLabels_Success(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
|
|
@ -6,9 +6,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/golang/snappy"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
)
|
||||
|
||||
|
@ -25,13 +23,13 @@ func BenchmarkParseProtobufRequest(b *testing.B) {
|
|||
}
|
||||
|
||||
func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
|
||||
blp := &insertutils.BenchmarkLogMessageProcessor{}
|
||||
blp := &insertutil.BenchmarkLogMessageProcessor{}
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(streams * rows))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
body := getProtobufBody(streams, rows, labels)
|
||||
for pb.Next() {
|
||||
if err := parseProtobufRequest(body, blp, false); err != nil {
|
||||
if err := parseProtobufRequest(body, blp, nil, false, true); err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
}
|
||||
|
@ -78,8 +76,5 @@ func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
|
|||
Streams: streams,
|
||||
}
|
||||
|
||||
body := pr.MarshalProtobuf(nil)
|
||||
encodedBody := snappy.Encode(nil, body)
|
||||
|
||||
return encodedBody
|
||||
return pr.MarshalProtobuf(nil)
|
||||
}
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/internalinsert"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
|
||||
|
@ -28,6 +29,11 @@ func Stop() {
|
|||
func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
path := r.URL.Path
|
||||
|
||||
if path == "/internal/insert" {
|
||||
internalinsert.RequestHandler(w, r)
|
||||
return true
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(path, "/insert/") {
|
||||
// Skip requests, which do not start with /insert/, since these aren't our requests.
|
||||
return false
|
||||
|
|
|
@ -2,21 +2,22 @@ package opentelemetry
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
var maxRequestSize = flagutil.NewBytes("opentelemetry.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single OpenTelemetry request")
|
||||
|
||||
// RequestHandler processes Opentelemetry insert requests
|
||||
func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
switch path {
|
||||
|
@ -37,26 +38,8 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
|||
func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
||||
startTime := time.Now()
|
||||
requestsProtobufTotal.Inc()
|
||||
reader := r.Body
|
||||
if r.Header.Get("Content-Encoding") == "gzip" {
|
||||
zr, err := common.GetGzipReader(reader)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
|
||||
return
|
||||
}
|
||||
defer common.PutGzipReader(zr)
|
||||
reader = zr
|
||||
}
|
||||
|
||||
wcr := writeconcurrencylimiter.GetReader(reader)
|
||||
data, err := io.ReadAll(wcr)
|
||||
writeconcurrencylimiter.PutReader(wcr)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read request body: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
cp, err := insertutils.GetCommonParams(r)
|
||||
cp, err := insertutil.GetCommonParams(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
|
||||
return
|
||||
|
@ -66,12 +49,16 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
|||
return
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf")
|
||||
useDefaultStreamFields := len(cp.StreamFields) == 0
|
||||
err = pushProtobufRequest(data, lmp, useDefaultStreamFields)
|
||||
lmp.MustClose()
|
||||
encoding := r.Header.Get("Content-Encoding")
|
||||
err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
|
||||
lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf", false)
|
||||
useDefaultStreamFields := len(cp.StreamFields) == 0
|
||||
err := pushProtobufRequest(data, lmp, useDefaultStreamFields)
|
||||
lmp.MustClose()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse OpenTelemetry protobuf request: %s", err)
|
||||
httpserver.Errorf(w, r, "cannot read OpenTelemetry protocol data: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -88,7 +75,7 @@ var (
|
|||
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
)
|
||||
|
||||
func pushProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
|
||||
func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, useDefaultStreamFields bool) error {
|
||||
var req pb.ExportLogsServiceRequest
|
||||
if err := req.UnmarshalProtobuf(data); err != nil {
|
||||
errorsTotal.Inc()
|
||||
|
@ -112,7 +99,7 @@ func pushProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDe
|
|||
return nil
|
||||
}
|
||||
|
||||
func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) []logstorage.Field {
|
||||
func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutil.LogMessageProcessor, useDefaultStreamFields bool) []logstorage.Field {
|
||||
fields := commonFields
|
||||
for _, lr := range sc.LogRecords {
|
||||
fields = fields[:len(commonFields)]
|
||||
|
|
|
@ -3,7 +3,7 @@ package opentelemetry
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
|
||||
)
|
||||
|
||||
|
@ -15,7 +15,7 @@ func TestPushProtoOk(t *testing.T) {
|
|||
}
|
||||
|
||||
pData := lr.MarshalProtobuf(nil)
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
if err := pushProtobufRequest(pData, tlp, false); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
@ -24,6 +24,7 @@ func TestPushProtoOk(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// single line without resource attributes
|
||||
f([]pb.ResourceLogs{
|
||||
{
|
||||
|
@ -39,6 +40,27 @@ func TestPushProtoOk(t *testing.T) {
|
|||
[]int64{1234},
|
||||
`{"_msg":"log-line-message","severity":"Trace"}`,
|
||||
)
|
||||
|
||||
// severities mapping
|
||||
f([]pb.ResourceLogs{
|
||||
{
|
||||
ScopeLogs: []pb.ScopeLogs{
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 13, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 24, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]int64{1234, 1234, 1234},
|
||||
`{"_msg":"log-line-message","severity":"Trace"}
|
||||
{"_msg":"log-line-message","severity":"Warn"}
|
||||
{"_msg":"log-line-message","severity":"Fatal4"}`,
|
||||
)
|
||||
|
||||
// multi-line with resource attributes
|
||||
f([]pb.ResourceLogs{
|
||||
{
|
||||
|
@ -58,7 +80,7 @@ func TestPushProtoOk(t *testing.T) {
|
|||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 21, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 25, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
|
||||
},
|
||||
},
|
||||
|
@ -106,19 +128,21 @@ func TestPushProtoOk(t *testing.T) {
|
|||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
|
||||
{ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
|
||||
{TraceID: "1234", SpanID: "45", ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
|
||||
{TraceID: "4bf92f3577b34da6a3ce929d0e0e4736", SpanID: "00f067aa0ba902b7", ObservedTimeUnixNano: 3333, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-2")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]int64{1234, 1235, 2345, 2346, 2347, 2348},
|
||||
[]int64{1234, 1235, 2345, 2346, 2347, 2348, 3333},
|
||||
`{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message","severity":"Trace"}
|
||||
{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Debug"}
|
||||
{"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
|
||||
{"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
|
||||
{"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
|
||||
{"_msg":"log-line-resource-scope-1-1-1","severity":"Info4"}`,
|
||||
{"_msg":"log-line-resource-scope-1-1-1","trace_id":"1234","span_id":"45","severity":"Info4"}
|
||||
{"_msg":"log-line-resource-scope-1-1-2","trace_id":"4bf92f3577b34da6a3ce929d0e0e4736","span_id":"00f067aa0ba902b7","severity":"Unspecified"}`,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
@ -4,7 +4,7 @@ import (
|
|||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
|
||||
)
|
||||
|
||||
|
@ -21,7 +21,7 @@ func BenchmarkParseProtobufRequest(b *testing.B) {
|
|||
}
|
||||
|
||||
func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
|
||||
blp := &insertutils.BenchmarkLogMessageProcessor{}
|
||||
blp := &insertutil.BenchmarkLogMessageProcessor{}
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(streams * rows))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
|
|
|
@ -16,9 +16,7 @@ import (
|
|||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/gzip"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
|
||||
|
@ -27,7 +25,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
@ -274,14 +272,14 @@ func runTCPListener(addr string, argIdx int) {
|
|||
|
||||
func checkCompressMethod(compressMethod, addr, protocol string) {
|
||||
switch compressMethod {
|
||||
case "", "none", "gzip", "deflate":
|
||||
case "", "none", "zstd", "gzip", "deflate":
|
||||
return
|
||||
default:
|
||||
logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
|
||||
logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'zstd', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
|
||||
}
|
||||
}
|
||||
|
||||
func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
|
||||
func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
|
||||
gomaxprocs := cgroup.AvailableCPUs()
|
||||
var wg sync.WaitGroup
|
||||
localAddr := ln.LocalAddr()
|
||||
|
@ -289,7 +287,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
|
|||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
|
||||
cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
|
||||
var bb bytesutil.ByteBuffer
|
||||
bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
|
||||
for {
|
||||
|
@ -314,7 +312,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
|
|||
}
|
||||
bb.B = bb.B[:n]
|
||||
udpRequestsTotal.Inc()
|
||||
if err := processStream("udp", bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
|
||||
if err := processStream("udp", bb.NewReader(), encoding, useLocalTimestamp, cp); err != nil {
|
||||
logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
|
||||
}
|
||||
}
|
||||
|
@ -323,7 +321,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
|
|||
wg.Wait()
|
||||
}
|
||||
|
||||
func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
|
||||
func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
|
||||
var cm ingestserver.ConnsMap
|
||||
cm.Init("syslog")
|
||||
|
||||
|
@ -353,8 +351,8 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
|
|||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
|
||||
if err := processStream("tcp", c, compressMethod, useLocalTimestamp, cp); err != nil {
|
||||
cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
|
||||
if err := processStream("tcp", c, encoding, useLocalTimestamp, cp); err != nil {
|
||||
logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
|
||||
}
|
||||
|
||||
|
@ -369,52 +367,29 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
|
|||
}
|
||||
|
||||
// processStream parses a stream of syslog messages from r and ingests them into vlstorage.
|
||||
func processStream(protocol string, r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
|
||||
func processStream(protocol string, r io.Reader, encoding string, useLocalTimestamp bool, cp *insertutil.CommonParams) error {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("syslog_" + protocol)
|
||||
err := processStreamInternal(r, compressMethod, useLocalTimestamp, lmp)
|
||||
lmp := cp.NewLogMessageProcessor("syslog_"+protocol, true)
|
||||
err := processStreamInternal(r, encoding, useLocalTimestamp, lmp)
|
||||
lmp.MustClose()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
|
||||
switch compressMethod {
|
||||
case "", "none":
|
||||
case "gzip":
|
||||
zr, err := common.GetGzipReader(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read gzipped data: %w", err)
|
||||
}
|
||||
r = zr
|
||||
case "deflate":
|
||||
zr, err := common.GetZlibReader(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read deflated data: %w", err)
|
||||
}
|
||||
r = zr
|
||||
default:
|
||||
logger.Panicf("BUG: unsupported compressMethod=%q; supported values: none, gzip, deflate", compressMethod)
|
||||
func processStreamInternal(r io.Reader, encoding string, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
|
||||
reader, err := protoparserutil.GetUncompressedReader(r, encoding)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot decode syslog data: %w", err)
|
||||
}
|
||||
defer protoparserutil.PutUncompressedReader(reader)
|
||||
|
||||
err := processUncompressedStream(r, useLocalTimestamp, lmp)
|
||||
|
||||
switch compressMethod {
|
||||
case "gzip":
|
||||
zr := r.(*gzip.Reader)
|
||||
common.PutGzipReader(zr)
|
||||
case "deflate":
|
||||
zr := r.(io.ReadCloser)
|
||||
common.PutZlibReader(zr)
|
||||
}
|
||||
|
||||
return err
|
||||
return processUncompressedStream(reader, useLocalTimestamp, lmp)
|
||||
}
|
||||
|
||||
func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
|
||||
func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
|
||||
wcr := writeconcurrencylimiter.GetReader(r)
|
||||
defer writeconcurrencylimiter.PutReader(wcr)
|
||||
|
||||
|
@ -499,7 +474,7 @@ again:
|
|||
slr.err = fmt.Errorf("cannot parse message length from %q: %w", msgLenStr, err)
|
||||
return false
|
||||
}
|
||||
if maxMsgLen := insertutils.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
|
||||
if maxMsgLen := insertutil.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
|
||||
slr.err = fmt.Errorf("cannot read message longer than %d bytes; msgLen=%d", maxMsgLen, msgLen)
|
||||
return false
|
||||
}
|
||||
|
@ -551,7 +526,7 @@ func putSyslogLineReader(slr *syslogLineReader) {
|
|||
|
||||
var syslogLineReaderPool sync.Pool
|
||||
|
||||
func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
|
||||
func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
|
||||
p := logstorage.GetSyslogParser(currentYear, timezone)
|
||||
lineStr := bytesutil.ToUnsafeString(line)
|
||||
p.Parse(lineStr)
|
||||
|
@ -560,7 +535,7 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
|
|||
if useLocalTimestamp {
|
||||
ts = time.Now().UnixNano()
|
||||
} else {
|
||||
nsecs, err := insertutils.ExtractTimestampFromFields("timestamp", p.Fields)
|
||||
nsecs, err := insertutil.ExtractTimestampFromFields("timestamp", p.Fields)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
|
||||
}
|
||||
|
|
|
@ -6,7 +6,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func TestSyslogLineReader_Success(t *testing.T) {
|
||||
|
@ -84,7 +84,7 @@ func TestProcessStreamInternal_Success(t *testing.T) {
|
|||
globalTimezone = time.UTC
|
||||
globalCurrentYear.Store(int64(currentYear))
|
||||
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
r := bytes.NewBufferString(data)
|
||||
if err := processStreamInternal(r, "", false, tlp); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
|
@ -113,7 +113,7 @@ func TestProcessStreamInternal_Failure(t *testing.T) {
|
|||
MustInit()
|
||||
defer MustStop()
|
||||
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
r := bytes.NewBufferString(data)
|
||||
if err := processStreamInternal(r, "", false, tlp); err == nil {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
|
|
|
@ -154,7 +154,7 @@ func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
|
|||
}
|
||||
value, ok := t.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected token read for oject value: %v; want string", t)
|
||||
return nil, fmt.Errorf("unexpected token read for object value: %v; want string", t)
|
||||
}
|
||||
|
||||
fields = append(fields, logstorage.Field{
|
||||
|
|
|
@ -31,7 +31,7 @@ var (
|
|||
totalStreams = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
|
||||
"during data generation")
|
||||
logsPerStream = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
|
||||
constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
|
||||
constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constant values to generate per each log entry; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||
varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||
|
@ -177,7 +177,10 @@ func generateAndPushLogs(cfg *workerConfig, workerID int) {
|
|||
sw := &statWriter{
|
||||
w: pw,
|
||||
}
|
||||
bw := bufio.NewWriter(sw)
|
||||
|
||||
// The 1MB write buffer increases data ingestion performance by reducing the number of send() syscalls
|
||||
bw := bufio.NewWriterSize(sw, 1024*1024)
|
||||
|
||||
doneCh := make(chan struct{})
|
||||
go func() {
|
||||
generateLogs(bw, workerID, cfg.activeStreams, cfg.totalStreams)
|
||||
|
|
324
app/vlselect/internalselect/internalselect.go
Normal file
324
app/vlselect/internalselect/internalselect.go
Normal file
|
@ -0,0 +1,324 @@
|
|||
package internalselect
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
)
|
||||
|
||||
var disableSelect = flag.Bool("internalselect.disable", false, "Whether to disable /internal/select/* HTTP endpoints")
|
||||
|
||||
// RequestHandler processes requests to /internal/select/*
|
||||
func RequestHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
if *disableSelect {
|
||||
httpserver.Errorf(w, r, "requests to /internal/select/* are disabled with -internalselect.disable command-line flag")
|
||||
return
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
path := r.URL.Path
|
||||
rh := requestHandlers[path]
|
||||
if rh == nil {
|
||||
httpserver.Errorf(w, r, "unsupported endpoint requested: %s", path)
|
||||
return
|
||||
}
|
||||
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_requests_total{path=%q}`, path)).Inc()
|
||||
if err := rh(ctx, w, r); err != nil && !netutil.IsTrivialNetworkError(err) {
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_request_errors_total{path=%q}`, path)).Inc()
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
// The return is skipped intentionally in order to track the duration of failed queries.
|
||||
}
|
||||
metrics.GetOrCreateSummary(fmt.Sprintf(`vl_http_request_duration_seconds{path=%q}`, path)).UpdateDuration(startTime)
|
||||
}
|
||||
|
||||
var requestHandlers = map[string]func(ctx context.Context, w http.ResponseWriter, r *http.Request) error{
|
||||
"/internal/select/query": processQueryRequest,
|
||||
"/internal/select/field_names": processFieldNamesRequest,
|
||||
"/internal/select/field_values": processFieldValuesRequest,
|
||||
"/internal/select/stream_field_names": processStreamFieldNamesRequest,
|
||||
"/internal/select/stream_field_values": processStreamFieldValuesRequest,
|
||||
"/internal/select/streams": processStreamsRequest,
|
||||
"/internal/select/stream_ids": processStreamIDsRequest,
|
||||
}
|
||||
|
||||
func processQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
|
||||
cp, err := getCommonParams(r, netselect.QueryProtocolVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
|
||||
var wLock sync.Mutex
|
||||
var dataLenBuf []byte
|
||||
|
||||
sendBuf := func(bb *bytesutil.ByteBuffer) error {
|
||||
if len(bb.B) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
data := bb.B
|
||||
if !cp.DisableCompression {
|
||||
bufLen := len(bb.B)
|
||||
bb.B = zstd.CompressLevel(bb.B, bb.B, 1)
|
||||
data = bb.B[bufLen:]
|
||||
}
|
||||
|
||||
wLock.Lock()
|
||||
dataLenBuf = encoding.MarshalUint64(dataLenBuf[:0], uint64(len(data)))
|
||||
_, err := w.Write(dataLenBuf)
|
||||
if err == nil {
|
||||
_, err = w.Write(data)
|
||||
}
|
||||
wLock.Unlock()
|
||||
|
||||
// Reset the sent buf
|
||||
bb.Reset()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
var bufs atomicutil.Slice[bytesutil.ByteBuffer]
|
||||
|
||||
var errGlobalLock sync.Mutex
|
||||
var errGlobal error
|
||||
|
||||
writeBlock := func(workerID uint, db *logstorage.DataBlock) {
|
||||
if errGlobal != nil {
|
||||
return
|
||||
}
|
||||
|
||||
bb := bufs.Get(workerID)
|
||||
|
||||
bb.B = db.Marshal(bb.B)
|
||||
|
||||
if len(bb.B) < 1024*1024 {
|
||||
// Fast path - the bb is too small to be sent to the client yet.
|
||||
return
|
||||
}
|
||||
|
||||
// Slow path - the bb must be sent to the client.
|
||||
if err := sendBuf(bb); err != nil {
|
||||
errGlobalLock.Lock()
|
||||
if errGlobal != nil {
|
||||
errGlobal = err
|
||||
}
|
||||
errGlobalLock.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
if err := vlstorage.RunQuery(ctx, cp.TenantIDs, cp.Query, writeBlock); err != nil {
|
||||
return err
|
||||
}
|
||||
if errGlobal != nil {
|
||||
return errGlobal
|
||||
}
|
||||
|
||||
// Send the remaining data
|
||||
for _, bb := range bufs.All() {
|
||||
if err := sendBuf(bb); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func processFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
|
||||
cp, err := getCommonParams(r, netselect.FieldNamesProtocolVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fieldNames, err := vlstorage.GetFieldNames(ctx, cp.TenantIDs, cp.Query)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain field names: %w", err)
|
||||
}
|
||||
|
||||
return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
|
||||
}
|
||||
|
||||
func processFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
|
||||
cp, err := getCommonParams(r, netselect.FieldValuesProtocolVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fieldName := r.FormValue("field")
|
||||
|
||||
limit, err := getInt64FromRequest(r, "limit")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fieldValues, err := vlstorage.GetFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain field values: %w", err)
|
||||
}
|
||||
|
||||
return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
|
||||
}
|
||||
|
||||
func processStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
|
||||
cp, err := getCommonParams(r, netselect.StreamFieldNamesProtocolVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fieldNames, err := vlstorage.GetStreamFieldNames(ctx, cp.TenantIDs, cp.Query)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain stream field names: %w", err)
|
||||
}
|
||||
|
||||
return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
|
||||
}
|
||||
|
||||
func processStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
|
||||
cp, err := getCommonParams(r, netselect.StreamFieldValuesProtocolVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fieldName := r.FormValue("field")
|
||||
|
||||
limit, err := getInt64FromRequest(r, "limit")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fieldValues, err := vlstorage.GetStreamFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain stream field values: %w", err)
|
||||
}
|
||||
|
||||
return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
|
||||
}
|
||||
|
||||
func processStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
|
||||
cp, err := getCommonParams(r, netselect.StreamsProtocolVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
limit, err := getInt64FromRequest(r, "limit")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
streams, err := vlstorage.GetStreams(ctx, cp.TenantIDs, cp.Query, uint64(limit))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain streams: %w", err)
|
||||
}
|
||||
|
||||
return writeValuesWithHits(w, streams, cp.DisableCompression)
|
||||
}
|
||||
|
||||
func processStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
|
||||
cp, err := getCommonParams(r, netselect.StreamIDsProtocolVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
limit, err := getInt64FromRequest(r, "limit")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
streamIDs, err := vlstorage.GetStreamIDs(ctx, cp.TenantIDs, cp.Query, uint64(limit))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain streams: %w", err)
|
||||
}
|
||||
|
||||
return writeValuesWithHits(w, streamIDs, cp.DisableCompression)
|
||||
}
|
||||
|
||||
type commonParams struct {
|
||||
TenantIDs []logstorage.TenantID
|
||||
Query *logstorage.Query
|
||||
|
||||
DisableCompression bool
|
||||
}
|
||||
|
||||
func getCommonParams(r *http.Request, expectedProtocolVersion string) (*commonParams, error) {
|
||||
version := r.FormValue("version")
|
||||
if version != expectedProtocolVersion {
|
||||
return nil, fmt.Errorf("unexpected version=%q; want %q", version, expectedProtocolVersion)
|
||||
}
|
||||
|
||||
tenantIDsStr := r.FormValue("tenant_ids")
|
||||
tenantIDs, err := logstorage.UnmarshalTenantIDs([]byte(tenantIDsStr))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot unmarshal tenant_ids=%q: %w", tenantIDsStr, err)
|
||||
}
|
||||
|
||||
timestamp, err := getInt64FromRequest(r, "timestamp")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
qStr := r.FormValue("query")
|
||||
q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot unmarshal query=%q: %w", qStr, err)
|
||||
}
|
||||
|
||||
s := r.FormValue("disable_compression")
|
||||
disableCompression, err := strconv.ParseBool(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse disable_compression=%q: %w", s, err)
|
||||
}
|
||||
|
||||
cp := &commonParams{
|
||||
TenantIDs: tenantIDs,
|
||||
Query: q,
|
||||
|
||||
DisableCompression: disableCompression,
|
||||
}
|
||||
return cp, nil
|
||||
}
|
||||
|
||||
func writeValuesWithHits(w http.ResponseWriter, vhs []logstorage.ValueWithHits, disableCompression bool) error {
|
||||
var b []byte
|
||||
for i := range vhs {
|
||||
b = vhs[i].Marshal(b)
|
||||
}
|
||||
|
||||
if !disableCompression {
|
||||
b = zstd.CompressLevel(nil, b, 1)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
|
||||
if _, err := w.Write(b); err != nil {
|
||||
return fmt.Errorf("cannot send response to the client: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getInt64FromRequest(r *http.Request, argName string) (int64, error) {
|
||||
s := r.FormValue(argName)
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse %s=%q: %w", argName, s, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
|
@ -1,47 +0,0 @@
|
|||
package logsql
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func getBufferedWriter(w io.Writer) *bufferedWriter {
|
||||
v := bufferedWriterPool.Get()
|
||||
if v == nil {
|
||||
return &bufferedWriter{
|
||||
bw: bufio.NewWriter(w),
|
||||
}
|
||||
}
|
||||
bw := v.(*bufferedWriter)
|
||||
bw.bw.Reset(w)
|
||||
return bw
|
||||
}
|
||||
|
||||
func putBufferedWriter(bw *bufferedWriter) {
|
||||
bw.reset()
|
||||
bufferedWriterPool.Put(bw)
|
||||
}
|
||||
|
||||
var bufferedWriterPool sync.Pool
|
||||
|
||||
type bufferedWriter struct {
|
||||
mu sync.Mutex
|
||||
bw *bufio.Writer
|
||||
}
|
||||
|
||||
func (bw *bufferedWriter) reset() {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
func (bw *bufferedWriter) WriteIgnoreErrors(p []byte) {
|
||||
bw.mu.Lock()
|
||||
_, _ = bw.bw.Write(p)
|
||||
bw.mu.Unlock()
|
||||
}
|
||||
|
||||
func (bw *bufferedWriter) FlushIgnoreErrors() {
|
||||
bw.mu.Lock()
|
||||
_ = bw.bw.Flush()
|
||||
bw.mu.Unlock()
|
||||
}
|
|
@ -3,6 +3,7 @@ package logsql
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
@ -11,15 +12,17 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
|
@ -35,32 +38,35 @@ func ProcessFacetsRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
|
|||
return
|
||||
}
|
||||
|
||||
limit, err := httputils.GetInt(r, "limit")
|
||||
limit, err := httputil.GetInt(r, "limit")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
maxValuesPerField, err := httputils.GetInt(r, "max_values_per_field")
|
||||
maxValuesPerField, err := httputil.GetInt(r, "max_values_per_field")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
maxValueLen, err := httputils.GetInt(r, "max_value_len")
|
||||
maxValueLen, err := httputil.GetInt(r, "max_value_len")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
keepConstFields := httputils.GetBool(r, "keep_const_fields")
|
||||
keepConstFields := httputil.GetBool(r, "keep_const_fields")
|
||||
|
||||
q.DropAllPipes()
|
||||
q.AddFacetsPipe(limit, maxValuesPerField, maxValueLen, keepConstFields)
|
||||
|
||||
var mLock sync.Mutex
|
||||
m := make(map[string][]facetEntry)
|
||||
writeBlock := func(_ uint, _ []int64, columns []logstorage.BlockColumn) {
|
||||
if len(columns) == 0 || len(columns[0].Values) == 0 {
|
||||
writeBlock := func(_ uint, db *logstorage.DataBlock) {
|
||||
rowsCount := db.RowsCount()
|
||||
if rowsCount == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
columns := db.Columns
|
||||
if len(columns) != 3 {
|
||||
logger.Panicf("BUG: expecting 3 columns; got %d columns", len(columns))
|
||||
}
|
||||
|
@ -141,7 +147,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
|
|||
fields := r.Form["field"]
|
||||
|
||||
// Obtain limit on the number of top fields entries.
|
||||
fieldsLimit, err := httputils.GetInt(r, "fields_limit")
|
||||
fieldsLimit, err := httputil.GetInt(r, "fields_limit")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
|
@ -156,17 +162,19 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
|
|||
|
||||
var mLock sync.Mutex
|
||||
m := make(map[string]*hitsSeries)
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
if len(columns) == 0 || len(columns[0].Values) == 0 {
|
||||
writeBlock := func(_ uint, db *logstorage.DataBlock) {
|
||||
rowsCount := db.RowsCount()
|
||||
if rowsCount == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
columns := db.Columns
|
||||
timestampValues := columns[0].Values
|
||||
hitsValues := columns[len(columns)-1].Values
|
||||
columns = columns[1 : len(columns)-1]
|
||||
|
||||
bb := blockResultPool.Get()
|
||||
for i := range timestamps {
|
||||
for i := 0; i < rowsCount; i++ {
|
||||
timestampStr := strings.Clone(timestampValues[i])
|
||||
hitsStr := strings.Clone(hitsValues[i])
|
||||
hits, err := strconv.ParseUint(hitsStr, 10, 64)
|
||||
|
@ -205,6 +213,8 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
|
|||
WriteHitsSeries(w, m)
|
||||
}
|
||||
|
||||
var blockResultPool bytesutil.ByteBufferPool
|
||||
|
||||
func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hitsSeries {
|
||||
if fieldsLimit <= 0 || fieldsLimit >= len(m) {
|
||||
return m
|
||||
|
@ -310,7 +320,7 @@ func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *ht
|
|||
}
|
||||
|
||||
// Parse limit query arg
|
||||
limit, err := httputils.GetInt(r, "limit")
|
||||
limit, err := httputil.GetInt(r, "limit")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
|
@ -370,7 +380,7 @@ func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter,
|
|||
}
|
||||
|
||||
// Parse limit query arg
|
||||
limit, err := httputils.GetInt(r, "limit")
|
||||
limit, err := httputil.GetInt(r, "limit")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
|
@ -401,7 +411,7 @@ func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http
|
|||
}
|
||||
|
||||
// Parse limit query arg
|
||||
limit, err := httputils.GetInt(r, "limit")
|
||||
limit, err := httputil.GetInt(r, "limit")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
|
@ -432,7 +442,7 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
|
|||
}
|
||||
|
||||
// Parse limit query arg
|
||||
limit, err := httputils.GetInt(r, "limit")
|
||||
limit, err := httputil.GetInt(r, "limit")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
|
@ -470,21 +480,21 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
|
|||
return
|
||||
}
|
||||
|
||||
refreshIntervalMsecs, err := httputils.GetDuration(r, "refresh_interval", 1000)
|
||||
refreshIntervalMsecs, err := httputil.GetDuration(r, "refresh_interval", 1000)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
refreshInterval := time.Millisecond * time.Duration(refreshIntervalMsecs)
|
||||
|
||||
startOffsetMsecs, err := httputils.GetDuration(r, "start_offset", 5*1000)
|
||||
startOffsetMsecs, err := httputil.GetDuration(r, "start_offset", 5*1000)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
startOffset := startOffsetMsecs * 1e6
|
||||
|
||||
offsetMsecs, err := httputils.GetDuration(r, "offset", 1000)
|
||||
offsetMsecs, err := httputil.GetDuration(r, "offset", 1000)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
|
@ -536,7 +546,7 @@ var liveTailRequests = metrics.NewCounter(`vl_live_tailing_requests`)
|
|||
const tailOffsetNsecs = 5e9
|
||||
|
||||
type logRow struct {
|
||||
timestamp int64
|
||||
timestamp string
|
||||
fields []logstorage.Field
|
||||
}
|
||||
|
||||
|
@ -552,7 +562,7 @@ type tailProcessor struct {
|
|||
mu sync.Mutex
|
||||
|
||||
perStreamRows map[string][]logRow
|
||||
lastTimestamps map[string]int64
|
||||
lastTimestamps map[string]string
|
||||
|
||||
err error
|
||||
}
|
||||
|
@ -562,12 +572,12 @@ func newTailProcessor(cancel func()) *tailProcessor {
|
|||
cancel: cancel,
|
||||
|
||||
perStreamRows: make(map[string][]logRow),
|
||||
lastTimestamps: make(map[string]int64),
|
||||
lastTimestamps: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
if len(timestamps) == 0 {
|
||||
func (tp *tailProcessor) writeBlock(_ uint, db *logstorage.DataBlock) {
|
||||
if db.RowsCount() == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -579,14 +589,8 @@ func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logsto
|
|||
}
|
||||
|
||||
// Make sure columns contain _time field, since it is needed for proper tail work.
|
||||
hasTime := false
|
||||
for _, c := range columns {
|
||||
if c.Name == "_time" {
|
||||
hasTime = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasTime {
|
||||
timestamps, ok := db.GetTimestamps()
|
||||
if !ok {
|
||||
tp.err = fmt.Errorf("missing _time field")
|
||||
tp.cancel()
|
||||
return
|
||||
|
@ -595,8 +599,8 @@ func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logsto
|
|||
// Copy block rows to tp.perStreamRows
|
||||
for i, timestamp := range timestamps {
|
||||
streamID := ""
|
||||
fields := make([]logstorage.Field, len(columns))
|
||||
for j, c := range columns {
|
||||
fields := make([]logstorage.Field, len(db.Columns))
|
||||
for j, c := range db.Columns {
|
||||
name := strings.Clone(c.Name)
|
||||
value := strings.Clone(c.Values[i])
|
||||
|
||||
|
@ -688,12 +692,15 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
|
|||
m := make(map[string]*statsSeries)
|
||||
var mLock sync.Mutex
|
||||
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
writeBlock := func(_ uint, db *logstorage.DataBlock) {
|
||||
rowsCount := db.RowsCount()
|
||||
|
||||
columns := db.Columns
|
||||
clonedColumnNames := make([]string, len(columns))
|
||||
for i, c := range columns {
|
||||
clonedColumnNames[i] = strings.Clone(c.Name)
|
||||
}
|
||||
for i := range timestamps {
|
||||
for i := 0; i < rowsCount; i++ {
|
||||
// Do not move q.GetTimestamp() outside writeBlock, since ts
|
||||
// must be initialized to query timestamp for every processed log row.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8312
|
||||
|
@ -802,12 +809,14 @@ func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *htt
|
|||
var rowsLock sync.Mutex
|
||||
|
||||
timestamp := q.GetTimestamp()
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
writeBlock := func(_ uint, db *logstorage.DataBlock) {
|
||||
rowsCount := db.RowsCount()
|
||||
columns := db.Columns
|
||||
clonedColumnNames := make([]string, len(columns))
|
||||
for i, c := range columns {
|
||||
clonedColumnNames[i] = strings.Clone(c.Name)
|
||||
}
|
||||
for i := range timestamps {
|
||||
for i := 0; i < rowsCount; i++ {
|
||||
labels := make([]logstorage.Field, 0, len(byFields))
|
||||
for j, c := range columns {
|
||||
if slices.Contains(byFields, c.Name) {
|
||||
|
@ -863,17 +872,27 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
|
|||
}
|
||||
|
||||
// Parse limit query arg
|
||||
limit, err := httputils.GetInt(r, "limit")
|
||||
limit, err := httputil.GetInt(r, "limit")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
|
||||
bw := getBufferedWriter(w)
|
||||
sw := &syncWriter{
|
||||
w: w,
|
||||
}
|
||||
|
||||
var bwShards atomicutil.Slice[bufferedWriter]
|
||||
bwShards.Init = func(shard *bufferedWriter) {
|
||||
shard.sw = sw
|
||||
}
|
||||
defer func() {
|
||||
bw.FlushIgnoreErrors()
|
||||
putBufferedWriter(bw)
|
||||
shards := bwShards.All()
|
||||
for _, shard := range shards {
|
||||
shard.FlushIgnoreErrors()
|
||||
}
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/stream+json")
|
||||
|
||||
if limit > 0 {
|
||||
|
@ -883,32 +902,34 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
|
|||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
bb := blockResultPool.Get()
|
||||
b := bb.B
|
||||
bw := bwShards.Get(0)
|
||||
for i := range rows {
|
||||
b = logstorage.MarshalFieldsToJSON(b[:0], rows[i].fields)
|
||||
b = append(b, '\n')
|
||||
bw.WriteIgnoreErrors(b)
|
||||
bw.buf = logstorage.MarshalFieldsToJSON(bw.buf, rows[i].fields)
|
||||
bw.buf = append(bw.buf, '\n')
|
||||
if len(bw.buf) > 16*1024 {
|
||||
bw.FlushIgnoreErrors()
|
||||
}
|
||||
}
|
||||
bb.B = b
|
||||
blockResultPool.Put(bb)
|
||||
return
|
||||
}
|
||||
|
||||
q.AddPipeLimit(uint64(limit))
|
||||
}
|
||||
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
if len(columns) == 0 || len(columns[0].Values) == 0 {
|
||||
writeBlock := func(workerID uint, db *logstorage.DataBlock) {
|
||||
rowsCount := db.RowsCount()
|
||||
if rowsCount == 0 {
|
||||
return
|
||||
}
|
||||
columns := db.Columns
|
||||
|
||||
bb := blockResultPool.Get()
|
||||
for i := range timestamps {
|
||||
WriteJSONRow(bb, columns, i)
|
||||
bw := bwShards.Get(workerID)
|
||||
for i := 0; i < rowsCount; i++ {
|
||||
WriteJSONRow(bw, columns, i)
|
||||
if len(bw.buf) > 16*1024 {
|
||||
bw.FlushIgnoreErrors()
|
||||
}
|
||||
}
|
||||
bw.WriteIgnoreErrors(bb.B)
|
||||
blockResultPool.Put(bb)
|
||||
}
|
||||
|
||||
if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
|
||||
|
@ -917,14 +938,37 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
|
|||
}
|
||||
}
|
||||
|
||||
var blockResultPool bytesutil.ByteBufferPool
|
||||
|
||||
type row struct {
|
||||
timestamp int64
|
||||
fields []logstorage.Field
|
||||
type syncWriter struct {
|
||||
mu sync.Mutex
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
|
||||
func (sw *syncWriter) Write(p []byte) (int, error) {
|
||||
sw.mu.Lock()
|
||||
n, err := sw.w.Write(p)
|
||||
sw.mu.Unlock()
|
||||
return n, err
|
||||
}
|
||||
|
||||
type bufferedWriter struct {
|
||||
buf []byte
|
||||
sw *syncWriter
|
||||
}
|
||||
|
||||
func (bw *bufferedWriter) Write(p []byte) (int, error) {
|
||||
bw.buf = append(bw.buf, p...)
|
||||
|
||||
// Do not send bw.buf to bw.sw here, since the data at bw.buf may be incomplete (it must end with '\n')
|
||||
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (bw *bufferedWriter) FlushIgnoreErrors() {
|
||||
_, _ = bw.sw.Write(bw.buf)
|
||||
bw.buf = bw.buf[:0]
|
||||
}
|
||||
|
||||
func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]logRow, error) {
|
||||
limitUpper := 2 * limit
|
||||
q.AddPipeLimit(uint64(limitUpper))
|
||||
|
||||
|
@ -993,7 +1037,7 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
|
|||
}
|
||||
}
|
||||
|
||||
func getLastNRows(rows []row, limit int) []row {
|
||||
func getLastNRows(rows []logRow, limit int) []logRow {
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
return rows[i].timestamp < rows[j].timestamp
|
||||
})
|
||||
|
@ -1003,18 +1047,31 @@ func getLastNRows(rows []row, limit int) []row {
|
|||
return rows
|
||||
}
|
||||
|
||||
func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
|
||||
func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]logRow, error) {
|
||||
ctxWithCancel, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
var rows []row
|
||||
var missingTimeColumn atomic.Bool
|
||||
var rows []logRow
|
||||
var rowsLock sync.Mutex
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
writeBlock := func(_ uint, db *logstorage.DataBlock) {
|
||||
if missingTimeColumn.Load() {
|
||||
return
|
||||
}
|
||||
|
||||
columns := db.Columns
|
||||
clonedColumnNames := make([]string, len(columns))
|
||||
for i, c := range columns {
|
||||
clonedColumnNames[i] = strings.Clone(c.Name)
|
||||
}
|
||||
|
||||
timestamps, ok := db.GetTimestamps()
|
||||
if !ok {
|
||||
missingTimeColumn.Store(true)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
|
||||
for i, timestamp := range timestamps {
|
||||
fields := make([]logstorage.Field, len(columns))
|
||||
for j := range columns {
|
||||
|
@ -1024,7 +1081,7 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
|
|||
}
|
||||
|
||||
rowsLock.Lock()
|
||||
rows = append(rows, row{
|
||||
rows = append(rows, logRow{
|
||||
timestamp: timestamp,
|
||||
fields: fields,
|
||||
})
|
||||
|
@ -1035,11 +1092,13 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
|
|||
cancel()
|
||||
}
|
||||
}
|
||||
if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock); err != nil {
|
||||
return nil, err
|
||||
err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock)
|
||||
|
||||
if missingTimeColumn.Load() {
|
||||
return nil, fmt.Errorf("missing _time column in the result for the query [%s]", q)
|
||||
}
|
||||
|
||||
return rows, nil
|
||||
return rows, err
|
||||
}
|
||||
|
||||
func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID, error) {
|
||||
|
@ -1098,20 +1157,22 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
|
|||
}
|
||||
|
||||
// Parse optional extra_filters
|
||||
extraFiltersStr := r.FormValue("extra_filters")
|
||||
extraFilters, err := parseExtraFilters(extraFiltersStr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
for _, extraFiltersStr := range r.Form["extra_filters"] {
|
||||
extraFilters, err := parseExtraFilters(extraFiltersStr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
q.AddExtraFilters(extraFilters)
|
||||
}
|
||||
q.AddExtraFilters(extraFilters)
|
||||
|
||||
// Parse optional extra_stream_filters
|
||||
extraStreamFiltersStr := r.FormValue("extra_stream_filters")
|
||||
extraStreamFilters, err := parseExtraStreamFilters(extraStreamFiltersStr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
for _, extraStreamFiltersStr := range r.Form["extra_stream_filters"] {
|
||||
extraStreamFilters, err := parseExtraStreamFilters(extraStreamFiltersStr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
q.AddExtraFilters(extraStreamFilters)
|
||||
}
|
||||
q.AddExtraFilters(extraStreamFilters)
|
||||
|
||||
return q, tenantIDs, nil
|
||||
}
|
||||
|
|
|
@ -9,10 +9,11 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/internalselect"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -71,7 +72,8 @@ var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
|
|||
// RequestHandler handles select requests for VictoriaLogs
|
||||
func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
path := r.URL.Path
|
||||
if !strings.HasPrefix(path, "/select/") {
|
||||
|
||||
if !strings.HasPrefix(path, "/select/") && !strings.HasPrefix(path, "/internal/select/") {
|
||||
// Skip requests, which do not start with /select/, since these aren't our requests.
|
||||
return false
|
||||
}
|
||||
|
@ -98,25 +100,75 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
if path == "/select/logsql/tail" {
|
||||
logsqlTailRequests.Inc()
|
||||
// Process live tailing request without timeout, since it is OK to run live tailing requests for very long time.
|
||||
// Also do not apply concurrency limit to tail requests, since these limits are intended for non-tail requests.
|
||||
logsql.ProcessLiveTailRequest(ctx, w, r)
|
||||
return true
|
||||
}
|
||||
|
||||
// Limit the number of concurrent queries, which can consume big amounts of CPU time.
|
||||
startTime := time.Now()
|
||||
ctx := r.Context()
|
||||
d := getMaxQueryDuration(r)
|
||||
ctxWithTimeout, cancel := context.WithTimeout(ctx, d)
|
||||
defer cancel()
|
||||
|
||||
stopCh := ctxWithTimeout.Done()
|
||||
if !incRequestConcurrency(ctxWithTimeout, w, r) {
|
||||
return true
|
||||
}
|
||||
defer decRequestConcurrency()
|
||||
|
||||
if strings.HasPrefix(path, "/internal/select/") {
|
||||
// Process internal request from vlselect without timeout (e.g. use ctx instead of ctxWithTimeout),
|
||||
// since the timeout must be controlled by the vlselect.
|
||||
internalselect.RequestHandler(ctx, w, r)
|
||||
return true
|
||||
}
|
||||
|
||||
ok := processSelectRequest(ctxWithTimeout, w, r, path)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
logRequestErrorIfNeeded(ctxWithTimeout, w, r, startTime)
|
||||
return true
|
||||
}
|
||||
|
||||
func logRequestErrorIfNeeded(ctx context.Context, w http.ResponseWriter, r *http.Request, startTime time.Time) {
|
||||
err := ctx.Err()
|
||||
switch err {
|
||||
case nil:
|
||||
// nothing to do
|
||||
case context.Canceled:
|
||||
// do not log canceled requests, since they are expected and legal.
|
||||
case context.DeadlineExceeded:
|
||||
err = &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
|
||||
"to increase -search.maxQueryDuration=%s; to pass bigger value to 'timeout' query arg", time.Since(startTime).Seconds(), maxQueryDuration),
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
default:
|
||||
httpserver.Errorf(w, r, "unexpected error: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func incRequestConcurrency(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {
|
||||
startTime := time.Now()
|
||||
stopCh := ctx.Done()
|
||||
select {
|
||||
case concurrencyLimitCh <- struct{}{}:
|
||||
defer func() { <-concurrencyLimitCh }()
|
||||
return true
|
||||
default:
|
||||
// Sleep for a while until giving up. This should resolve short bursts in requests.
|
||||
concurrencyLimitReached.Inc()
|
||||
select {
|
||||
case concurrencyLimitCh <- struct{}{}:
|
||||
defer func() { <-concurrencyLimitCh }()
|
||||
return true
|
||||
case <-stopCh:
|
||||
switch ctxWithTimeout.Err() {
|
||||
switch ctx.Err() {
|
||||
case context.Canceled:
|
||||
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
|
||||
requestURI := httpserver.GetRequestURI(r)
|
||||
|
@ -129,49 +181,18 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
|||
"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
|
||||
"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration=%s; to increase -search.maxConcurrentRequests; "+
|
||||
"to pass bigger value to 'timeout' query arg",
|
||||
d.Seconds(), *maxConcurrentRequests, maxQueueDuration, maxQueryDuration),
|
||||
time.Since(startTime).Seconds(), *maxConcurrentRequests, maxQueueDuration, maxQueryDuration),
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
}
|
||||
return true
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if path == "/select/logsql/tail" {
|
||||
logsqlTailRequests.Inc()
|
||||
// Process live tailing request without timeout (e.g. use ctx instead of ctxWithTimeout),
|
||||
// since it is OK to run live tailing requests for very long time.
|
||||
logsql.ProcessLiveTailRequest(ctx, w, r)
|
||||
return true
|
||||
}
|
||||
|
||||
ok := processSelectRequest(ctxWithTimeout, w, r, path)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
err := ctxWithTimeout.Err()
|
||||
switch err {
|
||||
case nil:
|
||||
// nothing to do
|
||||
case context.Canceled:
|
||||
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
|
||||
requestURI := httpserver.GetRequestURI(r)
|
||||
logger.Infof("client has canceled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
|
||||
time.Since(startTime).Seconds(), remoteAddr, requestURI)
|
||||
case context.DeadlineExceeded:
|
||||
err = &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
|
||||
"to increase -search.maxQueryDuration=%s; to pass bigger value to 'timeout' query arg", d.Seconds(), maxQueryDuration),
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
default:
|
||||
httpserver.Errorf(w, r, "unexpected error: %s", err)
|
||||
}
|
||||
|
||||
return true
|
||||
func decRequestConcurrency() {
|
||||
<-concurrencyLimitCh
|
||||
}
|
||||
|
||||
func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
|
||||
|
@ -240,7 +261,7 @@ func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
|
|||
|
||||
// getMaxQueryDuration returns the maximum duration for query from r.
|
||||
func getMaxQueryDuration(r *http.Request) time.Duration {
|
||||
dms, err := httputils.GetDuration(r, "timeout", 0)
|
||||
dms, err := httputil.GetDuration(r, "timeout", 0)
|
||||
if err != nil {
|
||||
dms = 0
|
||||
}
|
||||
|
|
|
@ -5,9 +5,13 @@ menu:
|
|||
docs:
|
||||
parent: 'victoriametrics'
|
||||
weight: 23
|
||||
tags:
|
||||
- metrics
|
||||
aliases:
|
||||
- /ExtendedPromQL.html
|
||||
- /MetricsQL.html
|
||||
- /metricsql/index.html
|
||||
- /metricsql/
|
||||
---
|
||||
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
|
||||
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
|
||||
|
@ -336,7 +340,7 @@ See also [increases_over_time](#increases_over_time).
|
|||
|
||||
`default_rollup(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
value on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
Compared to [last_over_time](#last_over_time) it accounts for [staleness markers](https://docs.victoriametrics.com/vmagent/#prometheus-staleness-markers) to detect stale series.
|
||||
Compared to [last_over_time](#last_over_time) it accounts for [staleness markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers) to detect stale series.
|
||||
|
||||
If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
|
||||
passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
|
||||
|
@ -921,7 +925,7 @@ See also [count_eq_over_time](#count_eq_over_time).
|
|||
#### stale_samples_over_time
|
||||
|
||||
`stale_samples_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number
|
||||
of [staleness markers](https://docs.victoriametrics.com/vmagent/#prometheus-staleness-markers) on the given lookbehind window `d`
|
||||
of [staleness markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers) on the given lookbehind window `d`
|
||||
per each time series matching the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
File diff suppressed because one or more lines are too long
1
app/vlselect/vmui/assets/index-Brup_hCI.css
Normal file
1
app/vlselect/vmui/assets/index-Brup_hCI.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
206
app/vlselect/vmui/assets/index-Wi3UrT0H.js
Normal file
206
app/vlselect/vmui/assets/index-Wi3UrT0H.js
Normal file
File diff suppressed because one or more lines are too long
76
app/vlselect/vmui/assets/vendor-C-vZmbyg.js
Normal file
76
app/vlselect/vmui/assets/vendor-C-vZmbyg.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
@ -35,10 +35,10 @@
|
|||
<meta property="og:title" content="UI for VictoriaLogs">
|
||||
<meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
|
||||
<meta property="og:description" content="Explore your log data with VictoriaLogs UI">
|
||||
<script type="module" crossorigin src="./assets/index-C68hz-qY.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-DojlIpLz.js">
|
||||
<script type="module" crossorigin src="./assets/index-Wi3UrT0H.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-C-vZmbyg.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-B_R5bdPN.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-Brup_hCI.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
|
|
|
@ -10,11 +10,14 @@ import (
|
|||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -38,15 +41,59 @@ var (
|
|||
minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
|
||||
"the storage stops accepting new data")
|
||||
|
||||
forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
|
||||
forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge . It overrides -httpAuth.* . "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/#forced-merge")
|
||||
forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush . It overrides -httpAuth.* . "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/#forced-flush")
|
||||
|
||||
storageNodeAddrs = flagutil.NewArrayString("storageNode", "Comma-separated list of TCP addresses for storage nodes to route the ingested logs to and to send select queries to. "+
|
||||
"If the list is empty, then the ingested logs are stored and queried locally from -storageDataPath")
|
||||
insertConcurrency = flag.Int("insert.concurrency", 2, "The average number of concurrent data ingestion requests, which can be sent to every -storageNode")
|
||||
insertDisableCompression = flag.Bool("insert.disableCompression", false, "Whether to disable compression when sending the ingested data to -storageNode nodes. "+
|
||||
"Disabled compression reduces CPU usage at the cost of higher network usage")
|
||||
selectDisableCompression = flag.Bool("select.disableCompression", false, "Whether to disable compression for select query responses received from -storageNode nodes. "+
|
||||
"Disabled compression reduces CPU usage at the cost of higher network usage")
|
||||
|
||||
storageNodeUsername = flagutil.NewArrayString("storageNode.username", "Optional basic auth username to use for the corresponding -storageNode")
|
||||
storageNodePassword = flagutil.NewArrayString("storageNode.password", "Optional basic auth password to use for the corresponding -storageNode")
|
||||
storageNodePasswordFile = flagutil.NewArrayString("storageNode.passwordFile", "Optional path to basic auth password to use for the corresponding -storageNode. "+
|
||||
"The file is re-read every second")
|
||||
storageNodeBearerToken = flagutil.NewArrayString("storageNode.bearerToken", "Optional bearer auth token to use for the corresponding -storageNode")
|
||||
storageNodeBearerTokenFile = flagutil.NewArrayString("storageNode.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -storageNode. "+
|
||||
"The token is re-read from the file every second")
|
||||
|
||||
storageNodeTLS = flagutil.NewArrayBool("storageNode.tls", "Whether to use TLS (HTTPS) protocol for communicating with the corresponding -storageNode. "+
|
||||
"By default communication is performed via HTTP")
|
||||
storageNodeTLSCAFile = flagutil.NewArrayString("storageNode.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -storageNode. "+
|
||||
"By default, system CA is used")
|
||||
storageNodeTLSCertFile = flagutil.NewArrayString("storageNode.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
|
||||
"to the corresponding -storageNode")
|
||||
storageNodeTLSKeyFile = flagutil.NewArrayString("storageNode.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -storageNode")
|
||||
storageNodeTLSServerName = flagutil.NewArrayString("storageNode.tlsServerName", "Optional TLS server name to use for connections to the corresponding -storageNode. "+
|
||||
"By default, the server name from -storageNode is used")
|
||||
storageNodeTLSInsecureSkipVerify = flagutil.NewArrayBool("storageNode.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -storageNode")
|
||||
)
|
||||
|
||||
var localStorage *logstorage.Storage
|
||||
var localStorageMetrics *metrics.Set
|
||||
|
||||
var netstorageInsert *netinsert.Storage
|
||||
var netstorageSelect *netselect.Storage
|
||||
|
||||
// Init initializes vlstorage.
|
||||
//
|
||||
// Stop must be called when vlstorage is no longer needed
|
||||
func Init() {
|
||||
if strg != nil {
|
||||
logger.Panicf("BUG: Init() has been already called")
|
||||
if len(*storageNodeAddrs) == 0 {
|
||||
initLocalStorage()
|
||||
} else {
|
||||
initNetworkStorage()
|
||||
}
|
||||
}
|
||||
|
||||
func initLocalStorage() {
|
||||
if localStorage != nil {
|
||||
logger.Panicf("BUG: initLocalStorage() has been already called")
|
||||
}
|
||||
|
||||
if retentionPeriod.Duration() < 24*time.Hour {
|
||||
|
@ -63,60 +110,157 @@ func Init() {
|
|||
}
|
||||
logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
|
||||
startTime := time.Now()
|
||||
strg = logstorage.MustOpenStorage(*storageDataPath, cfg)
|
||||
localStorage = logstorage.MustOpenStorage(*storageDataPath, cfg)
|
||||
|
||||
var ss logstorage.StorageStats
|
||||
strg.UpdateStats(&ss)
|
||||
localStorage.UpdateStats(&ss)
|
||||
logger.Infof("successfully opened storage in %.3f seconds; smallParts: %d; bigParts: %d; smallPartBlocks: %d; bigPartBlocks: %d; smallPartRows: %d; bigPartRows: %d; "+
|
||||
"smallPartSize: %d bytes; bigPartSize: %d bytes",
|
||||
time.Since(startTime).Seconds(), ss.SmallParts, ss.BigParts, ss.SmallPartBlocks, ss.BigPartBlocks, ss.SmallPartRowsCount, ss.BigPartRowsCount,
|
||||
ss.CompressedSmallPartSize, ss.CompressedBigPartSize)
|
||||
|
||||
// register storage metrics
|
||||
storageMetrics = metrics.NewSet()
|
||||
storageMetrics.RegisterMetricsWriter(func(w io.Writer) {
|
||||
writeStorageMetrics(w, strg)
|
||||
// register local storage metrics
|
||||
localStorageMetrics = metrics.NewSet()
|
||||
localStorageMetrics.RegisterMetricsWriter(func(w io.Writer) {
|
||||
writeStorageMetrics(w, localStorage)
|
||||
})
|
||||
metrics.RegisterSet(storageMetrics)
|
||||
metrics.RegisterSet(localStorageMetrics)
|
||||
}
|
||||
|
||||
func initNetworkStorage() {
|
||||
if netstorageInsert != nil || netstorageSelect != nil {
|
||||
logger.Panicf("BUG: initNetworkStorage() has been already called")
|
||||
}
|
||||
|
||||
authCfgs := make([]*promauth.Config, len(*storageNodeAddrs))
|
||||
isTLSs := make([]bool, len(*storageNodeAddrs))
|
||||
for i := range authCfgs {
|
||||
authCfgs[i] = newAuthConfigForStorageNode(i)
|
||||
isTLSs[i] = storageNodeTLS.GetOptionalArg(i)
|
||||
}
|
||||
|
||||
logger.Infof("starting insert service for nodes %s", *storageNodeAddrs)
|
||||
netstorageInsert = netinsert.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *insertConcurrency, *insertDisableCompression)
|
||||
|
||||
logger.Infof("initializing select service for nodes %s", *storageNodeAddrs)
|
||||
netstorageSelect = netselect.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *selectDisableCompression)
|
||||
|
||||
logger.Infof("initialized all the network services")
|
||||
}
|
||||
|
||||
func newAuthConfigForStorageNode(argIdx int) *promauth.Config {
|
||||
username := storageNodeUsername.GetOptionalArg(argIdx)
|
||||
password := storageNodePassword.GetOptionalArg(argIdx)
|
||||
passwordFile := storageNodePasswordFile.GetOptionalArg(argIdx)
|
||||
var basicAuthCfg *promauth.BasicAuthConfig
|
||||
if username != "" || password != "" || passwordFile != "" {
|
||||
basicAuthCfg = &promauth.BasicAuthConfig{
|
||||
Username: username,
|
||||
Password: promauth.NewSecret(password),
|
||||
PasswordFile: passwordFile,
|
||||
}
|
||||
}
|
||||
|
||||
token := storageNodeBearerToken.GetOptionalArg(argIdx)
|
||||
tokenFile := storageNodeBearerTokenFile.GetOptionalArg(argIdx)
|
||||
|
||||
tlsCfg := &promauth.TLSConfig{
|
||||
CAFile: storageNodeTLSCAFile.GetOptionalArg(argIdx),
|
||||
CertFile: storageNodeTLSCertFile.GetOptionalArg(argIdx),
|
||||
KeyFile: storageNodeTLSKeyFile.GetOptionalArg(argIdx),
|
||||
ServerName: storageNodeTLSServerName.GetOptionalArg(argIdx),
|
||||
InsecureSkipVerify: storageNodeTLSInsecureSkipVerify.GetOptionalArg(argIdx),
|
||||
}
|
||||
|
||||
opts := &promauth.Options{
|
||||
BasicAuth: basicAuthCfg,
|
||||
BearerToken: token,
|
||||
BearerTokenFile: tokenFile,
|
||||
TLSConfig: tlsCfg,
|
||||
}
|
||||
ac, err := opts.NewConfig()
|
||||
if err != nil {
|
||||
logger.Panicf("FATAL: cannot populate auth config for storage node #%d: %s", argIdx, err)
|
||||
}
|
||||
|
||||
return ac
|
||||
}
|
||||
|
||||
// Stop stops vlstorage.
|
||||
func Stop() {
|
||||
metrics.UnregisterSet(storageMetrics, true)
|
||||
storageMetrics = nil
|
||||
if localStorage != nil {
|
||||
metrics.UnregisterSet(localStorageMetrics, true)
|
||||
localStorageMetrics = nil
|
||||
|
||||
strg.MustClose()
|
||||
strg = nil
|
||||
localStorage.MustClose()
|
||||
localStorage = nil
|
||||
} else {
|
||||
netstorageInsert.MustStop()
|
||||
netstorageInsert = nil
|
||||
|
||||
netstorageSelect.MustStop()
|
||||
netstorageSelect = nil
|
||||
}
|
||||
}
|
||||
|
||||
// RequestHandler is a storage request handler.
|
||||
func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
path := r.URL.Path
|
||||
if path == "/internal/force_merge" {
|
||||
if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
|
||||
return true
|
||||
}
|
||||
// Run force merge in background
|
||||
partitionNamePrefix := r.FormValue("partition_prefix")
|
||||
go func() {
|
||||
activeForceMerges.Inc()
|
||||
defer activeForceMerges.Dec()
|
||||
logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
|
||||
startTime := time.Now()
|
||||
strg.MustForceMerge(partitionNamePrefix)
|
||||
logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
|
||||
}()
|
||||
return true
|
||||
switch path {
|
||||
case "/internal/force_merge":
|
||||
return processForceMerge(w, r)
|
||||
case "/internal/force_flush":
|
||||
return processForceFlush(w, r)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var strg *logstorage.Storage
|
||||
var storageMetrics *metrics.Set
|
||||
func processForceMerge(w http.ResponseWriter, r *http.Request) bool {
|
||||
if localStorage == nil {
|
||||
// Force merge isn't supported by non-local storage
|
||||
return false
|
||||
}
|
||||
|
||||
// CanWriteData returns non-nil error if it cannot write data to vlstorage.
|
||||
if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Run force merge in background
|
||||
partitionNamePrefix := r.FormValue("partition_prefix")
|
||||
go func() {
|
||||
activeForceMerges.Inc()
|
||||
defer activeForceMerges.Dec()
|
||||
logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
|
||||
startTime := time.Now()
|
||||
localStorage.MustForceMerge(partitionNamePrefix)
|
||||
logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
func processForceFlush(w http.ResponseWriter, r *http.Request) bool {
|
||||
if localStorage == nil {
|
||||
// Force merge isn't supported by non-local storage
|
||||
return false
|
||||
}
|
||||
|
||||
if !httpserver.CheckAuthFlag(w, r, forceFlushAuthKey) {
|
||||
return true
|
||||
}
|
||||
|
||||
logger.Infof("flushing storage to make pending data available for reading")
|
||||
localStorage.DebugFlush()
|
||||
return true
|
||||
}
|
||||
|
||||
// CanWriteData returns non-nil error if it cannot write data to vlstorage
|
||||
func CanWriteData() error {
|
||||
if strg.IsReadOnly() {
|
||||
if localStorage == nil {
|
||||
// The data can be always written in non-local mode.
|
||||
return nil
|
||||
}
|
||||
|
||||
if localStorage.IsReadOnly() {
|
||||
return &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
|
||||
"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
|
||||
|
@ -130,50 +274,77 @@ func CanWriteData() error {
|
|||
//
|
||||
// It is advised to call CanWriteData() before calling MustAddRows()
|
||||
func MustAddRows(lr *logstorage.LogRows) {
|
||||
strg.MustAddRows(lr)
|
||||
if localStorage != nil {
|
||||
// Store lr in the local storage.
|
||||
localStorage.MustAddRows(lr)
|
||||
} else {
|
||||
// Store lr across the remote storage nodes.
|
||||
lr.ForEachRow(netstorageInsert.AddRow)
|
||||
}
|
||||
}
|
||||
|
||||
// RunQuery runs the given q and calls writeBlock for the returned data blocks
|
||||
func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteBlockFunc) error {
|
||||
return strg.RunQuery(ctx, tenantIDs, q, writeBlock)
|
||||
func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
|
||||
if localStorage != nil {
|
||||
return localStorage.RunQuery(ctx, tenantIDs, q, writeBlock)
|
||||
}
|
||||
return netstorageSelect.RunQuery(ctx, tenantIDs, q, writeBlock)
|
||||
}
|
||||
|
||||
// GetFieldNames executes q and returns field names seen in results.
|
||||
func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
|
||||
return strg.GetFieldNames(ctx, tenantIDs, q)
|
||||
if localStorage != nil {
|
||||
return localStorage.GetFieldNames(ctx, tenantIDs, q)
|
||||
}
|
||||
return netstorageSelect.GetFieldNames(ctx, tenantIDs, q)
|
||||
}
|
||||
|
||||
// GetFieldValues executes q and returns unique values for the fieldName seen in results.
|
||||
//
|
||||
// If limit > 0, then up to limit unique values are returned.
|
||||
func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
return strg.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
|
||||
if localStorage != nil {
|
||||
return localStorage.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
|
||||
}
|
||||
return netstorageSelect.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
|
||||
}
|
||||
|
||||
// GetStreamFieldNames executes q and returns stream field names seen in results.
|
||||
func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
|
||||
return strg.GetStreamFieldNames(ctx, tenantIDs, q)
|
||||
if localStorage != nil {
|
||||
return localStorage.GetStreamFieldNames(ctx, tenantIDs, q)
|
||||
}
|
||||
return netstorageSelect.GetStreamFieldNames(ctx, tenantIDs, q)
|
||||
}
|
||||
|
||||
// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
|
||||
//
|
||||
// If limit > 0, then up to limit unique stream field values are returned.
|
||||
func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
return strg.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
|
||||
if localStorage != nil {
|
||||
return localStorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
|
||||
}
|
||||
return netstorageSelect.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
|
||||
}
|
||||
|
||||
// GetStreams executes q and returns streams seen in query results.
|
||||
//
|
||||
// If limit > 0, then up to limit unique streams are returned.
|
||||
func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
return strg.GetStreams(ctx, tenantIDs, q, limit)
|
||||
if localStorage != nil {
|
||||
return localStorage.GetStreams(ctx, tenantIDs, q, limit)
|
||||
}
|
||||
return netstorageSelect.GetStreams(ctx, tenantIDs, q, limit)
|
||||
}
|
||||
|
||||
// GetStreamIDs executes q and returns streamIDs seen in query results.
|
||||
//
|
||||
// If limit > 0, then up to limit unique streamIDs are returned.
|
||||
func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
return strg.GetStreamIDs(ctx, tenantIDs, q, limit)
|
||||
if localStorage != nil {
|
||||
return localStorage.GetStreamIDs(ctx, tenantIDs, q, limit)
|
||||
}
|
||||
return netstorageSelect.GetStreamIDs(ctx, tenantIDs, q, limit)
|
||||
}
|
||||
|
||||
func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
|
||||
|
|
369
app/vlstorage/netinsert/netinsert.go
Normal file
369
app/vlstorage/netinsert/netinsert.go
Normal file
|
@ -0,0 +1,369 @@
|
|||
package netinsert
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/valyala/fastrand"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
||||
)
|
||||
|
||||
// the maximum size of a single data block sent to storage node.
|
||||
const maxInsertBlockSize = 2 * 1024 * 1024
|
||||
|
||||
// ProtocolVersion is the version of the data ingestion protocol.
|
||||
//
|
||||
// It must be changed every time the data encoding at /internal/insert HTTP endpoint is changed.
|
||||
const ProtocolVersion = "v1"
|
||||
|
||||
// Storage is a network storage for sending data to remote storage nodes in the cluster.
|
||||
type Storage struct {
|
||||
sns []*storageNode
|
||||
|
||||
disableCompression bool
|
||||
|
||||
srt *streamRowsTracker
|
||||
|
||||
pendingDataBuffers chan *bytesutil.ByteBuffer
|
||||
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
type storageNode struct {
|
||||
// scheme is http or https scheme to communicate with addr
|
||||
scheme string
|
||||
|
||||
// addr is TCP address of storage node to send the ingested data to
|
||||
addr string
|
||||
|
||||
// s is a storage, which holds the given storageNode
|
||||
s *Storage
|
||||
|
||||
// c is an http client used for sending data blocks to addr.
|
||||
c *http.Client
|
||||
|
||||
// ac is auth config used for setting request headers such as Authorization and Host.
|
||||
ac *promauth.Config
|
||||
|
||||
// pendingData contains pending data, which must be sent to the storage node at the addr.
|
||||
pendingDataMu sync.Mutex
|
||||
pendingData *bytesutil.ByteBuffer
|
||||
pendingDataLastFlush time.Time
|
||||
|
||||
// the unix timestamp until the storageNode is disabled for data writing.
|
||||
disabledUntil atomic.Uint64
|
||||
}
|
||||
|
||||
func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
|
||||
tr := httputil.NewTransport(false, "vlinsert_backend")
|
||||
tr.TLSHandshakeTimeout = 20 * time.Second
|
||||
tr.DisableCompression = true
|
||||
|
||||
scheme := "http"
|
||||
if isTLS {
|
||||
scheme = "https"
|
||||
}
|
||||
|
||||
sn := &storageNode{
|
||||
scheme: scheme,
|
||||
addr: addr,
|
||||
s: s,
|
||||
c: &http.Client{
|
||||
Transport: ac.NewRoundTripper(tr),
|
||||
},
|
||||
ac: ac,
|
||||
|
||||
pendingData: &bytesutil.ByteBuffer{},
|
||||
}
|
||||
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
sn.backgroundFlusher()
|
||||
}()
|
||||
|
||||
return sn
|
||||
}
|
||||
|
||||
func (sn *storageNode) backgroundFlusher() {
|
||||
t := time.NewTicker(time.Second)
|
||||
defer t.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-sn.s.stopCh:
|
||||
return
|
||||
case <-t.C:
|
||||
sn.flushPendingData()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sn *storageNode) flushPendingData() {
|
||||
sn.pendingDataMu.Lock()
|
||||
if time.Since(sn.pendingDataLastFlush) < time.Second {
|
||||
// nothing to flush
|
||||
sn.pendingDataMu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
pendingData := sn.grabPendingDataForFlushLocked()
|
||||
sn.pendingDataMu.Unlock()
|
||||
|
||||
sn.mustSendInsertRequest(pendingData)
|
||||
}
|
||||
|
||||
func (sn *storageNode) addRow(r *logstorage.InsertRow) {
|
||||
bb := bbPool.Get()
|
||||
b := bb.B
|
||||
|
||||
b = r.Marshal(b)
|
||||
|
||||
if len(b) > maxInsertBlockSize {
|
||||
logger.Warnf("skipping too long log entry, since its length exceeds %d bytes; the actual log entry length is %d bytes; log entry contents: %s", maxInsertBlockSize, len(b), b)
|
||||
return
|
||||
}
|
||||
|
||||
var pendingData *bytesutil.ByteBuffer
|
||||
sn.pendingDataMu.Lock()
|
||||
if sn.pendingData.Len()+len(b) > maxInsertBlockSize {
|
||||
pendingData = sn.grabPendingDataForFlushLocked()
|
||||
}
|
||||
sn.pendingData.MustWrite(b)
|
||||
sn.pendingDataMu.Unlock()
|
||||
|
||||
bb.B = b
|
||||
bbPool.Put(bb)
|
||||
|
||||
if pendingData != nil {
|
||||
sn.mustSendInsertRequest(pendingData)
|
||||
}
|
||||
}
|
||||
|
||||
var bbPool bytesutil.ByteBufferPool
|
||||
|
||||
func (sn *storageNode) grabPendingDataForFlushLocked() *bytesutil.ByteBuffer {
|
||||
sn.pendingDataLastFlush = time.Now()
|
||||
pendingData := sn.pendingData
|
||||
sn.pendingData = <-sn.s.pendingDataBuffers
|
||||
|
||||
return pendingData
|
||||
}
|
||||
|
||||
func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer) {
|
||||
defer func() {
|
||||
pendingData.Reset()
|
||||
sn.s.pendingDataBuffers <- pendingData
|
||||
}()
|
||||
|
||||
err := sn.sendInsertRequest(pendingData)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !errors.Is(err, errTemporarilyDisabled) {
|
||||
logger.Warnf("%s; re-routing the data block to the remaining nodes", err)
|
||||
}
|
||||
for !sn.s.sendInsertRequestToAnyNode(pendingData) {
|
||||
logger.Errorf("cannot send pending data to all storage nodes, since all of them are unavailable; re-trying to send the data in a second")
|
||||
|
||||
t := timerpool.Get(time.Second)
|
||||
select {
|
||||
case <-sn.s.stopCh:
|
||||
timerpool.Put(t)
|
||||
logger.Errorf("dropping %d bytes of data, since there are no available storage nodes", pendingData.Len())
|
||||
return
|
||||
case <-t.C:
|
||||
timerpool.Put(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sn *storageNode) sendInsertRequest(pendingData *bytesutil.ByteBuffer) error {
|
||||
dataLen := pendingData.Len()
|
||||
if dataLen == 0 {
|
||||
// Nothing to send.
|
||||
return nil
|
||||
}
|
||||
|
||||
if sn.disabledUntil.Load() > fasttime.UnixTimestamp() {
|
||||
return errTemporarilyDisabled
|
||||
}
|
||||
|
||||
ctx, cancel := contextutil.NewStopChanContext(sn.s.stopCh)
|
||||
defer cancel()
|
||||
|
||||
var body io.Reader
|
||||
if !sn.s.disableCompression {
|
||||
bb := zstdBufPool.Get()
|
||||
defer zstdBufPool.Put(bb)
|
||||
|
||||
bb.B = zstd.CompressLevel(bb.B[:0], pendingData.B, 1)
|
||||
body = bb.NewReader()
|
||||
} else {
|
||||
body = pendingData.NewReader()
|
||||
}
|
||||
|
||||
reqURL := sn.getRequestURL("/internal/insert")
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", reqURL, body)
|
||||
if err != nil {
|
||||
logger.Panicf("BUG: unexpected error when creating an http request: %s", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
if !sn.s.disableCompression {
|
||||
req.Header.Set("Content-Encoding", "zstd")
|
||||
}
|
||||
if err := sn.ac.SetHeaders(req, true); err != nil {
|
||||
return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
|
||||
}
|
||||
|
||||
resp, err := sn.c.Do(req)
|
||||
if err != nil {
|
||||
// Disable sn for data writing for 10 seconds.
|
||||
sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
|
||||
|
||||
return fmt.Errorf("cannot send data block with the length %d to %q: %s", pendingData.Len(), reqURL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode/100 == 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
respBody = []byte(fmt.Sprintf("%s", err))
|
||||
}
|
||||
|
||||
// Disable sn for data writing for 10 seconds.
|
||||
sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
|
||||
|
||||
return fmt.Errorf("unexpected status code returned when sending data block to %q: %d; want 2xx; response body: %q", reqURL, resp.StatusCode, respBody)
|
||||
}
|
||||
|
||||
func (sn *storageNode) getRequestURL(path string) string {
|
||||
return fmt.Sprintf("%s://%s%s?version=%s", sn.scheme, sn.addr, path, url.QueryEscape(ProtocolVersion))
|
||||
}
|
||||
|
||||
var zstdBufPool bytesutil.ByteBufferPool
|
||||
|
||||
// NewStorage returns new Storage for the given addrs with the given authCfgs.
|
||||
//
|
||||
// The concurrency is the average number of concurrent connections per every addr.
|
||||
//
|
||||
// If disableCompression is set, then the data is sent uncompressed to the remote storage.
|
||||
//
|
||||
// Call MustStop on the returned storage when it is no longer needed.
|
||||
func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, concurrency int, disableCompression bool) *Storage {
|
||||
pendingDataBuffers := make(chan *bytesutil.ByteBuffer, concurrency*len(addrs))
|
||||
for i := 0; i < cap(pendingDataBuffers); i++ {
|
||||
pendingDataBuffers <- &bytesutil.ByteBuffer{}
|
||||
}
|
||||
|
||||
s := &Storage{
|
||||
disableCompression: disableCompression,
|
||||
pendingDataBuffers: pendingDataBuffers,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
sns := make([]*storageNode, len(addrs))
|
||||
for i, addr := range addrs {
|
||||
sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
|
||||
}
|
||||
s.sns = sns
|
||||
|
||||
s.srt = newStreamRowsTracker(len(sns))
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// MustStop stops the s.
|
||||
func (s *Storage) MustStop() {
|
||||
close(s.stopCh)
|
||||
s.wg.Wait()
|
||||
s.sns = nil
|
||||
}
|
||||
|
||||
// AddRow adds the given log row into s.
|
||||
func (s *Storage) AddRow(streamHash uint64, r *logstorage.InsertRow) {
|
||||
idx := s.srt.getNodeIdx(streamHash)
|
||||
sn := s.sns[idx]
|
||||
sn.addRow(r)
|
||||
}
|
||||
|
||||
func (s *Storage) sendInsertRequestToAnyNode(pendingData *bytesutil.ByteBuffer) bool {
|
||||
startIdx := int(fastrand.Uint32n(uint32(len(s.sns))))
|
||||
for i := range s.sns {
|
||||
idx := (startIdx + i) % len(s.sns)
|
||||
sn := s.sns[idx]
|
||||
err := sn.sendInsertRequest(pendingData)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
if !errors.Is(err, errTemporarilyDisabled) {
|
||||
logger.Warnf("cannot send pending data to the storage node %q: %s; trying to send it to another storage node", sn.addr, err)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var errTemporarilyDisabled = fmt.Errorf("writing to the node is temporarily disabled")
|
||||
|
||||
type streamRowsTracker struct {
|
||||
mu sync.Mutex
|
||||
|
||||
nodesCount int64
|
||||
rowsPerStream map[uint64]uint64
|
||||
}
|
||||
|
||||
func newStreamRowsTracker(nodesCount int) *streamRowsTracker {
|
||||
return &streamRowsTracker{
|
||||
nodesCount: int64(nodesCount),
|
||||
rowsPerStream: make(map[uint64]uint64),
|
||||
}
|
||||
}
|
||||
|
||||
func (srt *streamRowsTracker) getNodeIdx(streamHash uint64) uint64 {
|
||||
if srt.nodesCount == 1 {
|
||||
// Fast path for a single node.
|
||||
return 0
|
||||
}
|
||||
|
||||
srt.mu.Lock()
|
||||
defer srt.mu.Unlock()
|
||||
|
||||
streamRows := srt.rowsPerStream[streamHash] + 1
|
||||
srt.rowsPerStream[streamHash] = streamRows
|
||||
|
||||
if streamRows <= 1000 {
|
||||
// Write the initial rows for the stream to a single storage node for better locality.
|
||||
// This should work great for log streams containing small number of logs, since will be distributed
|
||||
// evenly among available storage nodes because they have different streamHash.
|
||||
return streamHash % uint64(srt.nodesCount)
|
||||
}
|
||||
|
||||
// The log stream contains more than 1000 rows. Distribute them among storage nodes at random
|
||||
// in order to improve query performance over this stream (the data for the log stream
|
||||
// can be processed in parallel on all the storage nodes).
|
||||
//
|
||||
// The random distribution is preferred over round-robin distribution in order to avoid possible
|
||||
// dependency between the order of the ingested logs and the number of storage nodes,
|
||||
// which may lead to non-uniform distribution of logs among storage nodes.
|
||||
return uint64(fastrand.Uint32n(uint32(srt.nodesCount)))
|
||||
}
|
57
app/vlstorage/netinsert/netinsert_test.go
Normal file
57
app/vlstorage/netinsert/netinsert_test.go
Normal file
|
@ -0,0 +1,57 @@
|
|||
package netinsert
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
)
|
||||
|
||||
func TestStreamRowsTracker(t *testing.T) {
|
||||
f := func(rowsCount, streamsCount, nodesCount int) {
|
||||
t.Helper()
|
||||
|
||||
// generate stream hashes
|
||||
streamHashes := make([]uint64, streamsCount)
|
||||
for i := range streamHashes {
|
||||
streamHashes[i] = xxhash.Sum64([]byte(fmt.Sprintf("stream %d.", i)))
|
||||
}
|
||||
|
||||
srt := newStreamRowsTracker(nodesCount)
|
||||
|
||||
rng := rand.New(rand.NewSource(0))
|
||||
rowsPerNode := make([]uint64, nodesCount)
|
||||
for i := 0; i < rowsCount; i++ {
|
||||
streamIdx := rng.Intn(streamsCount)
|
||||
h := streamHashes[streamIdx]
|
||||
nodeIdx := srt.getNodeIdx(h)
|
||||
rowsPerNode[nodeIdx]++
|
||||
}
|
||||
|
||||
// Verify that rows are uniformly distributed among nodes.
|
||||
expectedRowsPerNode := float64(rowsCount) / float64(nodesCount)
|
||||
for nodeIdx, nodeRows := range rowsPerNode {
|
||||
if math.Abs(float64(nodeRows)-expectedRowsPerNode)/expectedRowsPerNode > 0.15 {
|
||||
t.Fatalf("non-uniform distribution of rows among nodes; node %d has %d rows, while it must have %v rows; rowsPerNode=%d",
|
||||
nodeIdx, nodeRows, expectedRowsPerNode, rowsPerNode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rowsCount := 10000
|
||||
streamsCount := 9
|
||||
nodesCount := 2
|
||||
f(rowsCount, streamsCount, nodesCount)
|
||||
|
||||
rowsCount = 10000
|
||||
streamsCount = 100
|
||||
nodesCount = 2
|
||||
f(rowsCount, streamsCount, nodesCount)
|
||||
|
||||
rowsCount = 100000
|
||||
streamsCount = 1000
|
||||
nodesCount = 9
|
||||
f(rowsCount, streamsCount, nodesCount)
|
||||
}
|
469
app/vlstorage/netselect/netselect.go
Normal file
469
app/vlstorage/netselect/netselect.go
Normal file
|
@ -0,0 +1,469 @@
|
|||
package netselect
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
|
||||
)
|
||||
|
||||
const (
|
||||
// FieldNamesProtocolVersion is the version of the protocol used for /internal/select/field_names HTTP endpoint.
|
||||
//
|
||||
// It must be updated every time the protocol changes.
|
||||
FieldNamesProtocolVersion = "v1"
|
||||
|
||||
// FieldValuesProtocolVersion is the version of the protocol used for /internal/select/field_values HTTP endpoint.
|
||||
//
|
||||
// It must be updated every time the protocol changes.
|
||||
FieldValuesProtocolVersion = "v1"
|
||||
|
||||
// StreamFieldNamesProtocolVersion is the version of the protocol used for /internal/select/stream_field_names HTTP endpoint.
|
||||
//
|
||||
// It must be updated every time the protocol changes.
|
||||
StreamFieldNamesProtocolVersion = "v1"
|
||||
|
||||
// StreamFieldValuesProtocolVersion is the version of the protocol used for /internal/select/stream_field_values HTTP endpoint.
|
||||
//
|
||||
// It must be updated every time the protocol changes.
|
||||
StreamFieldValuesProtocolVersion = "v1"
|
||||
|
||||
// StreamsProtocolVersion is the version of the protocol used for /internal/select/streams HTTP endpoint.
|
||||
//
|
||||
// It must be updated every time the protocol changes.
|
||||
StreamsProtocolVersion = "v1"
|
||||
|
||||
// StreamIDsProtocolVersion is the version of the protocol used for /internal/select/stream_ids HTTP endpoint.
|
||||
//
|
||||
// It must be updated every time the protocol changes.
|
||||
StreamIDsProtocolVersion = "v1"
|
||||
|
||||
// QueryProtocolVersion is the version of the protocol used for /internal/select/query HTTP endpoint.
|
||||
//
|
||||
// It must be updated every time the protocol changes.
|
||||
QueryProtocolVersion = "v1"
|
||||
)
|
||||
|
||||
// Storage is a network storage for querying remote storage nodes in the cluster.
|
||||
type Storage struct {
|
||||
sns []*storageNode
|
||||
|
||||
disableCompression bool
|
||||
}
|
||||
|
||||
type storageNode struct {
|
||||
// scheme is http or https scheme to communicate with addr
|
||||
scheme string
|
||||
|
||||
// addr is TCP address of the storage node to query
|
||||
addr string
|
||||
|
||||
// s is a storage, which holds the given storageNode
|
||||
s *Storage
|
||||
|
||||
// c is an http client used for querying storage node at addr.
|
||||
c *http.Client
|
||||
|
||||
// ac is auth config used for setting request headers such as Authorization and Host.
|
||||
ac *promauth.Config
|
||||
}
|
||||
|
||||
func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
|
||||
tr := httputil.NewTransport(false, "vlselect_backend")
|
||||
tr.TLSHandshakeTimeout = 20 * time.Second
|
||||
tr.DisableCompression = true
|
||||
|
||||
scheme := "http"
|
||||
if isTLS {
|
||||
scheme = "https"
|
||||
}
|
||||
|
||||
sn := &storageNode{
|
||||
scheme: scheme,
|
||||
addr: addr,
|
||||
s: s,
|
||||
c: &http.Client{
|
||||
Transport: ac.NewRoundTripper(tr),
|
||||
},
|
||||
ac: ac,
|
||||
}
|
||||
return sn
|
||||
}
|
||||
|
||||
func (sn *storageNode) runQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, processBlock func(db *logstorage.DataBlock)) error {
|
||||
args := sn.getCommonArgs(QueryProtocolVersion, tenantIDs, q)
|
||||
|
||||
reqURL := sn.getRequestURL("/internal/select/query", args)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
|
||||
if err != nil {
|
||||
logger.Panicf("BUG: unexpected error when creating a request: %s", err)
|
||||
}
|
||||
if err := sn.ac.SetHeaders(req, true); err != nil {
|
||||
return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
|
||||
}
|
||||
|
||||
// send the request to the storage node
|
||||
resp, err := sn.c.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
responseBody = []byte(err.Error())
|
||||
}
|
||||
return fmt.Errorf("unexpected status code for the request to %q: %d; want %d; response: %q", reqURL, resp.StatusCode, http.StatusOK, responseBody)
|
||||
}
|
||||
|
||||
// read the response
|
||||
var dataLenBuf [8]byte
|
||||
var buf []byte
|
||||
var db logstorage.DataBlock
|
||||
var valuesBuf []string
|
||||
for {
|
||||
if _, err := io.ReadFull(resp.Body, dataLenBuf[:]); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
// The end of response stream
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("cannot read block size from %q: %w", reqURL, err)
|
||||
}
|
||||
blockLen := encoding.UnmarshalUint64(dataLenBuf[:])
|
||||
if blockLen > math.MaxInt {
|
||||
return fmt.Errorf("too big data block: %d bytes; mustn't exceed %v bytes", blockLen, math.MaxInt)
|
||||
}
|
||||
|
||||
buf = slicesutil.SetLength(buf, int(blockLen))
|
||||
if _, err := io.ReadFull(resp.Body, buf); err != nil {
|
||||
return fmt.Errorf("cannot read block with size of %d bytes from %q: %w", blockLen, reqURL, err)
|
||||
}
|
||||
|
||||
src := buf
|
||||
if !sn.s.disableCompression {
|
||||
bufLen := len(buf)
|
||||
var err error
|
||||
buf, err = zstd.Decompress(buf, buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot decompress data block: %w", err)
|
||||
}
|
||||
src = buf[bufLen:]
|
||||
}
|
||||
|
||||
for len(src) > 0 {
|
||||
tail, vb, err := db.UnmarshalInplace(src, valuesBuf[:0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot unmarshal data block received from %q: %w", reqURL, err)
|
||||
}
|
||||
valuesBuf = vb
|
||||
src = tail
|
||||
|
||||
processBlock(&db)
|
||||
|
||||
clear(valuesBuf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sn *storageNode) getFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
|
||||
args := sn.getCommonArgs(FieldNamesProtocolVersion, tenantIDs, q)
|
||||
|
||||
return sn.getValuesWithHits(ctx, "/internal/select/field_names", args)
|
||||
}
|
||||
|
||||
func (sn *storageNode) getFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
args := sn.getCommonArgs(FieldValuesProtocolVersion, tenantIDs, q)
|
||||
args.Set("field", fieldName)
|
||||
args.Set("limit", fmt.Sprintf("%d", limit))
|
||||
|
||||
return sn.getValuesWithHits(ctx, "/internal/select/field_values", args)
|
||||
}
|
||||
|
||||
func (sn *storageNode) getStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
|
||||
args := sn.getCommonArgs(StreamFieldNamesProtocolVersion, tenantIDs, q)
|
||||
|
||||
return sn.getValuesWithHits(ctx, "/internal/select/stream_field_names", args)
|
||||
}
|
||||
|
||||
func (sn *storageNode) getStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
args := sn.getCommonArgs(StreamFieldValuesProtocolVersion, tenantIDs, q)
|
||||
args.Set("field", fieldName)
|
||||
args.Set("limit", fmt.Sprintf("%d", limit))
|
||||
|
||||
return sn.getValuesWithHits(ctx, "/internal/select/stream_field_values", args)
|
||||
}
|
||||
|
||||
func (sn *storageNode) getStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
args := sn.getCommonArgs(StreamsProtocolVersion, tenantIDs, q)
|
||||
args.Set("limit", fmt.Sprintf("%d", limit))
|
||||
|
||||
return sn.getValuesWithHits(ctx, "/internal/select/streams", args)
|
||||
}
|
||||
|
||||
func (sn *storageNode) getStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
args := sn.getCommonArgs(StreamIDsProtocolVersion, tenantIDs, q)
|
||||
args.Set("limit", fmt.Sprintf("%d", limit))
|
||||
|
||||
return sn.getValuesWithHits(ctx, "/internal/select/stream_ids", args)
|
||||
}
|
||||
|
||||
func (sn *storageNode) getCommonArgs(version string, tenantIDs []logstorage.TenantID, q *logstorage.Query) url.Values {
|
||||
args := url.Values{}
|
||||
args.Set("version", version)
|
||||
args.Set("tenant_ids", string(logstorage.MarshalTenantIDs(nil, tenantIDs)))
|
||||
args.Set("query", q.String())
|
||||
args.Set("timestamp", fmt.Sprintf("%d", q.GetTimestamp()))
|
||||
args.Set("disable_compression", fmt.Sprintf("%v", sn.s.disableCompression))
|
||||
return args
|
||||
}
|
||||
|
||||
func (sn *storageNode) getValuesWithHits(ctx context.Context, path string, args url.Values) ([]logstorage.ValueWithHits, error) {
|
||||
data, err := sn.executeRequestAt(ctx, path, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return unmarshalValuesWithHits(data)
|
||||
}
|
||||
|
||||
func (sn *storageNode) executeRequestAt(ctx context.Context, path string, args url.Values) ([]byte, error) {
|
||||
reqURL := sn.getRequestURL(path, args)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
|
||||
if err != nil {
|
||||
logger.Panicf("BUG: unexpected error when creating a request: %s", err)
|
||||
}
|
||||
|
||||
// send the request to the storage node
|
||||
resp, err := sn.c.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
responseBody = []byte(err.Error())
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected status code for the request to %q: %d; want %d; response: %q", reqURL, resp.StatusCode, http.StatusOK, responseBody)
|
||||
}
|
||||
|
||||
// read the response
|
||||
var bb bytesutil.ByteBuffer
|
||||
if _, err := bb.ReadFrom(resp.Body); err != nil {
|
||||
return nil, fmt.Errorf("cannot read response from %q: %w", reqURL, err)
|
||||
}
|
||||
|
||||
if sn.s.disableCompression {
|
||||
return bb.B, nil
|
||||
}
|
||||
|
||||
bbLen := len(bb.B)
|
||||
bb.B, err = zstd.Decompress(bb.B, bb.B)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bb.B[bbLen:], nil
|
||||
}
|
||||
|
||||
func (sn *storageNode) getRequestURL(path string, args url.Values) string {
|
||||
return fmt.Sprintf("%s://%s%s?%s", sn.scheme, sn.addr, path, args.Encode())
|
||||
}
|
||||
|
||||
// NewStorage returns new Storage for the given addrs and the given authCfgs.
|
||||
//
|
||||
// If disableCompression is set, then uncompressed responses are received from storage nodes.
|
||||
//
|
||||
// Call MustStop on the returned storage when it is no longer needed.
|
||||
func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, disableCompression bool) *Storage {
|
||||
s := &Storage{
|
||||
disableCompression: disableCompression,
|
||||
}
|
||||
|
||||
sns := make([]*storageNode, len(addrs))
|
||||
for i, addr := range addrs {
|
||||
sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
|
||||
}
|
||||
s.sns = sns
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// MustStop stops the s.
|
||||
func (s *Storage) MustStop() {
|
||||
s.sns = nil
|
||||
}
|
||||
|
||||
// RunQuery runs the given q and calls writeBlock for the returned data blocks
|
||||
func (s *Storage) RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
|
||||
nqr, err := logstorage.NewNetQueryRunner(ctx, tenantIDs, q, s.RunQuery, writeBlock)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
search := func(stopCh <-chan struct{}, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
|
||||
return s.runQuery(stopCh, tenantIDs, q, writeBlock)
|
||||
}
|
||||
|
||||
concurrency := q.GetConcurrency()
|
||||
return nqr.Run(ctx, concurrency, search)
|
||||
}
|
||||
|
||||
func (s *Storage) runQuery(stopCh <-chan struct{}, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
|
||||
ctxWithCancel, cancel := contextutil.NewStopChanContext(stopCh)
|
||||
defer cancel()
|
||||
|
||||
errs := make([]error, len(s.sns))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := range s.sns {
|
||||
wg.Add(1)
|
||||
go func(nodeIdx int) {
|
||||
defer wg.Done()
|
||||
sn := s.sns[nodeIdx]
|
||||
err := sn.runQuery(ctxWithCancel, tenantIDs, q, func(db *logstorage.DataBlock) {
|
||||
writeBlock(uint(nodeIdx), db)
|
||||
})
|
||||
if err != nil {
|
||||
// Cancel the remaining parallel queries
|
||||
cancel()
|
||||
}
|
||||
|
||||
errs[nodeIdx] = err
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return getFirstNonCancelError(errs)
|
||||
}
|
||||
|
||||
// GetFieldNames executes q and returns field names seen in results.
|
||||
func (s *Storage) GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
|
||||
return s.getValuesWithHits(ctx, 0, false, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
|
||||
return sn.getFieldNames(ctx, tenantIDs, q)
|
||||
})
|
||||
}
|
||||
|
||||
// GetFieldValues executes q and returns unique values for the fieldName seen in results.
|
||||
//
|
||||
// If limit > 0, then up to limit unique values are returned.
|
||||
func (s *Storage) GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
|
||||
return sn.getFieldValues(ctx, tenantIDs, q, fieldName, limit)
|
||||
})
|
||||
}
|
||||
|
||||
// GetStreamFieldNames executes q and returns stream field names seen in results.
|
||||
func (s *Storage) GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
|
||||
return s.getValuesWithHits(ctx, 0, false, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
|
||||
return sn.getStreamFieldNames(ctx, tenantIDs, q)
|
||||
})
|
||||
}
|
||||
|
||||
// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
|
||||
//
|
||||
// If limit > 0, then up to limit unique stream field values are returned.
|
||||
func (s *Storage) GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
|
||||
return sn.getStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
|
||||
})
|
||||
}
|
||||
|
||||
// GetStreams executes q and returns streams seen in query results.
|
||||
//
|
||||
// If limit > 0, then up to limit unique streams are returned.
|
||||
func (s *Storage) GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
|
||||
return sn.getStreams(ctx, tenantIDs, q, limit)
|
||||
})
|
||||
}
|
||||
|
||||
// GetStreamIDs executes q and returns streamIDs seen in query results.
|
||||
//
|
||||
// If limit > 0, then up to limit unique streamIDs are returned.
|
||||
func (s *Storage) GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
|
||||
return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
|
||||
return sn.getStreamIDs(ctx, tenantIDs, q, limit)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Storage) getValuesWithHits(ctx context.Context, limit uint64, resetHitsOnLimitExceeded bool,
|
||||
callback func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error)) ([]logstorage.ValueWithHits, error) {
|
||||
|
||||
ctxWithCancel, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
results := make([][]logstorage.ValueWithHits, len(s.sns))
|
||||
errs := make([]error, len(s.sns))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := range s.sns {
|
||||
wg.Add(1)
|
||||
go func(nodeIdx int) {
|
||||
defer wg.Done()
|
||||
|
||||
sn := s.sns[nodeIdx]
|
||||
vhs, err := callback(ctxWithCancel, sn)
|
||||
results[nodeIdx] = vhs
|
||||
errs[nodeIdx] = err
|
||||
|
||||
if err != nil {
|
||||
// Cancel the remaining parallel requests
|
||||
cancel()
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if err := getFirstNonCancelError(errs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
vhs := logstorage.MergeValuesWithHits(results, limit, resetHitsOnLimitExceeded)
|
||||
|
||||
return vhs, nil
|
||||
}
|
||||
|
||||
func getFirstNonCancelError(errs []error) error {
|
||||
for _, err := range errs {
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func unmarshalValuesWithHits(src []byte) ([]logstorage.ValueWithHits, error) {
|
||||
var vhs []logstorage.ValueWithHits
|
||||
for len(src) > 0 {
|
||||
var vh logstorage.ValueWithHits
|
||||
tail, err := vh.UnmarshalInplace(src)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot unmarshal ValueWithHits #%d: %w", len(vhs), err)
|
||||
}
|
||||
src = tail
|
||||
|
||||
// Clone vh.Value, since it points to src.
|
||||
vh.Value = strings.Clone(vh.Value)
|
||||
|
||||
vhs = append(vhs, vh)
|
||||
}
|
||||
|
||||
return vhs, nil
|
||||
}
|
|
@ -1,3 +1,3 @@
|
|||
See vmagent docs [here](https://docs.victoriametrics.com/vmagent/).
|
||||
See vmagent docs [here](https://docs.victoriametrics.com/victoriametrics/vmagent/).
|
||||
|
||||
vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmagent.md).
|
||||
vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmagent.md).
|
||||
|
|
|
@ -7,9 +7,9 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -22,16 +22,16 @@ var (
|
|||
|
||||
// InsertHandler processes csv data from req.
|
||||
func InsertHandler(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return stream.Parse(req, func(rows []parser.Row) error {
|
||||
return stream.Parse(req, func(rows []csvimport.Row) error {
|
||||
return insertRows(at, rows, extraLabels)
|
||||
})
|
||||
}
|
||||
|
||||
func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
|
||||
func insertRows(at *auth.Token, rows []csvimport.Row, extraLabels []prompbmarshal.Label) error {
|
||||
ctx := common.GetPushCtx()
|
||||
defer common.PutPushCtx(ctx)
|
||||
|
||||
|
|
|
@ -7,10 +7,10 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -23,7 +23,7 @@ var (
|
|||
|
||||
// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
|
||||
func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -56,7 +56,7 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
|
|||
})
|
||||
}
|
||||
for _, tag := range sketch.Tags {
|
||||
name, value := datadogutils.SplitTag(tag)
|
||||
name, value := datadogutil.SplitTag(tag)
|
||||
if name == "host" {
|
||||
name = "exported_host"
|
||||
}
|
||||
|
|
|
@ -7,10 +7,10 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -23,7 +23,7 @@ var (
|
|||
|
||||
// InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
|
||||
func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -62,7 +62,7 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbm
|
|||
})
|
||||
}
|
||||
for _, tag := range ss.Tags {
|
||||
name, value := datadogutils.SplitTag(tag)
|
||||
name, value := datadogutil.SplitTag(tag)
|
||||
if name == "host" {
|
||||
name = "exported_host"
|
||||
}
|
||||
|
|
|
@ -7,10 +7,10 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -25,7 +25,7 @@ var (
|
|||
//
|
||||
// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
|
||||
func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -65,7 +65,7 @@ func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompbm
|
|||
})
|
||||
}
|
||||
for _, tag := range ss.Tags {
|
||||
name, value := datadogutils.SplitTag(tag)
|
||||
name, value := datadogutil.SplitTag(tag)
|
||||
if name == "host" {
|
||||
name = "exported_host"
|
||||
}
|
||||
|
|
|
@ -21,7 +21,7 @@ var (
|
|||
//
|
||||
// See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
|
||||
func InsertHandler(r io.Reader) error {
|
||||
return stream.Parse(r, false, func(rows []parser.Row) error {
|
||||
return stream.Parse(r, "", func(rows []parser.Row) error {
|
||||
return insertRows(nil, rows)
|
||||
})
|
||||
}
|
||||
|
|
|
@ -12,9 +12,9 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -35,8 +35,8 @@ var (
|
|||
// InsertHandlerForReader processes remote write for influx line protocol.
|
||||
//
|
||||
// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
|
||||
func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
|
||||
return stream.Parse(r, true, isGzipped, "", "", func(db string, rows []parser.Row) error {
|
||||
func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
|
||||
return stream.Parse(r, encoding, true, "", "", func(db string, rows []influx.Row) error {
|
||||
return insertRows(at, db, rows, nil)
|
||||
})
|
||||
}
|
||||
|
@ -45,22 +45,22 @@ func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
|
|||
//
|
||||
// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
|
||||
func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
|
||||
isStreamMode := req.Header.Get("Stream-Mode") == "1"
|
||||
q := req.URL.Query()
|
||||
precision := q.Get("precision")
|
||||
// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
|
||||
db := q.Get("db")
|
||||
return stream.Parse(req.Body, isStreamMode, isGzipped, precision, db, func(db string, rows []parser.Row) error {
|
||||
encoding := req.Header.Get("Content-Encoding")
|
||||
isStreamMode := req.Header.Get("Stream-Mode") == "1"
|
||||
return stream.Parse(req.Body, encoding, isStreamMode, precision, db, func(db string, rows []influx.Row) error {
|
||||
return insertRows(at, db, rows, extraLabels)
|
||||
})
|
||||
}
|
||||
|
||||
func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
|
||||
func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prompbmarshal.Label) error {
|
||||
ctx := getPushCtx()
|
||||
defer putPushCtx(ctx)
|
||||
|
||||
|
|
|
@ -34,7 +34,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutil"
|
||||
graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
|
||||
influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
|
||||
opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
|
||||
|
@ -42,8 +42,8 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
|
||||
|
@ -105,7 +105,7 @@ func main() {
|
|||
// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
|
||||
// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
|
||||
// exceeds 0.05 for extended periods of time.
|
||||
cgroup.SetGOGC(30)
|
||||
cgroup.SetGOGC(50)
|
||||
|
||||
// Write flags and help message to stdout, since it is easier to grep or pipe.
|
||||
flag.CommandLine.SetOutput(os.Stdout)
|
||||
|
@ -145,10 +145,10 @@ func main() {
|
|||
startTime := time.Now()
|
||||
remotewrite.StartIngestionRateLimiter()
|
||||
remotewrite.Init()
|
||||
common.StartUnmarshalWorkers()
|
||||
protoparserutil.StartUnmarshalWorkers()
|
||||
if len(*influxListenAddr) > 0 {
|
||||
influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
|
||||
return influx.InsertHandlerForReader(nil, r, false)
|
||||
return influx.InsertHandlerForReader(nil, r, "")
|
||||
})
|
||||
}
|
||||
if len(*graphiteListenAddr) > 0 {
|
||||
|
@ -195,7 +195,7 @@ func main() {
|
|||
if len(*opentsdbHTTPListenAddr) > 0 {
|
||||
opentsdbhttpServer.MustStop()
|
||||
}
|
||||
common.StopUnmarshalWorkers()
|
||||
protoparserutil.StopUnmarshalWorkers()
|
||||
remotewrite.Stop()
|
||||
|
||||
logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
|
||||
|
@ -242,7 +242,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
|||
}
|
||||
w.Header().Add("Content-Type", "text/html; charset=utf-8")
|
||||
fmt.Fprintf(w, "<h2>vmagent</h2>")
|
||||
fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent/'>https://docs.victoriametrics.com/vmagent/</a></br>")
|
||||
fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victoriametrics/vmagent/'>https://docs.victoriametrics.com/victoriametrics/vmagent/</a></br>")
|
||||
fmt.Fprintf(w, "Useful endpoints:</br>")
|
||||
httpserver.WriteAPIHelp(w, [][2]string{
|
||||
{"targets", "status for discovered active targets"},
|
||||
|
@ -282,7 +282,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
|||
}
|
||||
switch path {
|
||||
case "/prometheus/api/v1/write", "/api/v1/write", "/api/v1/push", "/prometheus/api/v1/push":
|
||||
if common.HandleVMProtoServerHandshake(w, r) {
|
||||
if protoparserutil.HandleVMProtoServerHandshake(w, r) {
|
||||
return true
|
||||
}
|
||||
prometheusWriteRequests.Inc()
|
||||
|
@ -331,11 +331,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
|||
return true
|
||||
case "/influx/query", "/query":
|
||||
influxQueryRequests.Inc()
|
||||
influxutils.WriteDatabaseNames(w)
|
||||
influxutil.WriteDatabaseNames(w)
|
||||
return true
|
||||
case "/influx/health":
|
||||
influxHealthRequests.Inc()
|
||||
influxutils.WriteHealthCheckResponse(w)
|
||||
influxutil.WriteHealthCheckResponse(w)
|
||||
return true
|
||||
case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
|
||||
opentelemetryPushRequests.Inc()
|
||||
|
@ -443,8 +443,10 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
|||
case "/prometheus/api/v1/targets", "/api/v1/targets":
|
||||
promscrapeAPIV1TargetsRequests.Inc()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
// https://prometheus.io/docs/prometheus/latest/querying/api/#targets
|
||||
state := r.FormValue("state")
|
||||
promscrape.WriteAPIV1Targets(w, state)
|
||||
scrapePool := r.FormValue("scrapePool")
|
||||
promscrape.WriteAPIV1Targets(w, state, scrapePool)
|
||||
return true
|
||||
case "/prometheus/target_response", "/target_response":
|
||||
promscrapeTargetResponseRequests.Inc()
|
||||
|
@ -587,11 +589,11 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
|
|||
return true
|
||||
case "influx/query":
|
||||
influxQueryRequests.Inc()
|
||||
influxutils.WriteDatabaseNames(w)
|
||||
influxutil.WriteDatabaseNames(w)
|
||||
return true
|
||||
case "influx/health":
|
||||
influxHealthRequests.Inc()
|
||||
influxutils.WriteHealthCheckResponse(w)
|
||||
influxutil.WriteHealthCheckResponse(w)
|
||||
return true
|
||||
case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
|
||||
opentelemetryPushRequests.Inc()
|
||||
|
@ -750,7 +752,7 @@ func usage() {
|
|||
const s = `
|
||||
vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
|
||||
|
||||
See the docs at https://docs.victoriametrics.com/vmagent/ .
|
||||
See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
|
||||
`
|
||||
flagutil.Usage(s)
|
||||
}
|
||||
|
|
|
@ -9,8 +9,8 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -25,12 +25,12 @@ var (
|
|||
//
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
|
||||
func InsertHandler(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isGzip := req.Header.Get("Content-Encoding") == "gzip"
|
||||
return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
|
||||
encoding := req.Header.Get("Content-Encoding")
|
||||
return stream.Parse(req.Body, encoding, func(block *stream.Block) error {
|
||||
return insertRows(at, block, extraLabels)
|
||||
})
|
||||
}
|
||||
|
|
|
@ -10,9 +10,9 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
)
|
||||
|
||||
|
@ -24,13 +24,12 @@ var (
|
|||
|
||||
// InsertHandlerForHTTP processes remote write for NewRelic POST /infra/v2/metrics/events/bulk request.
|
||||
func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ce := req.Header.Get("Content-Encoding")
|
||||
isGzip := ce == "gzip"
|
||||
return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
|
||||
encoding := req.Header.Get("Content-Encoding")
|
||||
return stream.Parse(req.Body, encoding, func(rows []newrelic.Row) error {
|
||||
return insertRows(at, rows, extraLabels)
|
||||
})
|
||||
}
|
||||
|
|
|
@ -8,9 +8,9 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -23,11 +23,11 @@ var (
|
|||
|
||||
// InsertHandler processes opentelemetry metrics.
|
||||
func InsertHandler(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
|
||||
encoding := req.Header.Get("Content-Encoding")
|
||||
var processBody func([]byte) ([]byte, error)
|
||||
if req.Header.Get("Content-Type") == "application/json" {
|
||||
if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
|
||||
|
@ -36,7 +36,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
|
|||
return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
|
||||
}
|
||||
}
|
||||
return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
|
||||
return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompbmarshal.TimeSeries) error {
|
||||
return insertRows(at, tss, extraLabels)
|
||||
})
|
||||
}
|
||||
|
|
|
@ -7,9 +7,9 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
|
@ -21,16 +21,16 @@ var (
|
|||
// InsertHandler processes HTTP OpenTSDB put requests.
|
||||
// See http://opentsdb.net/docs/build/html/api_http/put.html
|
||||
func InsertHandler(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return stream.Parse(req, func(rows []parser.Row) error {
|
||||
return stream.Parse(req, func(rows []opentsdbhttp.Row) error {
|
||||
return insertRows(at, rows, extraLabels)
|
||||
})
|
||||
}
|
||||
|
||||
func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
|
||||
func insertRows(at *auth.Token, rows []opentsdbhttp.Row, extraLabels []prompbmarshal.Label) error {
|
||||
ctx := common.GetPushCtx()
|
||||
defer common.PutPushCtx(ctx)
|
||||
|
||||
|
|
|
@ -8,9 +8,9 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -23,23 +23,23 @@ var (
|
|||
|
||||
// InsertHandler processes `/api/v1/import/prometheus` request.
|
||||
func InsertHandler(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defaultTimestamp, err := parserCommon.GetTimestamp(req)
|
||||
defaultTimestamp, err := protoparserutil.GetTimestamp(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
|
||||
return stream.Parse(req.Body, defaultTimestamp, isGzipped, true, func(rows []parser.Row) error {
|
||||
encoding := req.Header.Get("Content-Encoding")
|
||||
return stream.Parse(req.Body, defaultTimestamp, encoding, true, func(rows []prometheus.Row) error {
|
||||
return insertRows(at, rows, extraLabels)
|
||||
}, func(s string) {
|
||||
httpserver.LogError(req, s)
|
||||
})
|
||||
}
|
||||
|
||||
func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
|
||||
func insertRows(at *auth.Token, rows []prometheus.Row, extraLabels []prompbmarshal.Label) error {
|
||||
ctx := common.GetPushCtx()
|
||||
defer common.PutPushCtx(ctx)
|
||||
|
||||
|
|
|
@ -12,7 +12,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -26,7 +26,7 @@ func TestInsertHandler(t *testing.T) {
|
|||
req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
|
||||
go_memstats_alloc_bytes_total 1`))
|
||||
if err := InsertHandler(nil, req); err != nil {
|
||||
t.Fatalf("unxepected error %s", err)
|
||||
t.Fatalf("unexpected error %s", err)
|
||||
}
|
||||
expectedMsg := "cannot unmarshal Prometheus line"
|
||||
if !strings.Contains(testOutput.String(), expectedMsg) {
|
||||
|
@ -44,14 +44,14 @@ func setUp() {
|
|||
log.Fatalf("unable to set %q with value %q, err: %v", remoteWriteFlag, srv.URL, err)
|
||||
}
|
||||
logger.Init()
|
||||
common.StartUnmarshalWorkers()
|
||||
protoparserutil.StartUnmarshalWorkers()
|
||||
remotewrite.Init()
|
||||
testOutput = &bytes.Buffer{}
|
||||
logger.SetOutputForTests(testOutput)
|
||||
}
|
||||
|
||||
func tearDown() {
|
||||
common.StopUnmarshalWorkers()
|
||||
protoparserutil.StopUnmarshalWorkers()
|
||||
srv.Close()
|
||||
logger.ResetOutputForTest()
|
||||
tmpDataDir := flag.Lookup("remoteWrite.tmpDataPath").Value.String()
|
||||
|
|
|
@ -8,8 +8,8 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
@ -22,7 +22,7 @@ var (
|
|||
|
||||
// InsertHandler processes remote write for prometheus.
|
||||
func InsertHandler(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -10,26 +10,29 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/golang/snappy"
|
||||
)
|
||||
|
||||
var (
|
||||
forcePromProto = flagutil.NewArrayBool("remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
|
||||
"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
|
||||
"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
|
||||
forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
|
||||
"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
|
||||
"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
|
||||
|
||||
rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
|
||||
"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
|
||||
|
@ -87,7 +90,8 @@ type client struct {
|
|||
remoteWriteURL string
|
||||
|
||||
// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
|
||||
useVMProto bool
|
||||
useVMProto atomic.Bool
|
||||
canDowngradeVMProto atomic.Bool
|
||||
|
||||
fq *persistentqueue.FastQueue
|
||||
hc *http.Client
|
||||
|
@ -124,14 +128,14 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
|
|||
if err != nil {
|
||||
logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
|
||||
}
|
||||
tr := &http.Transport{
|
||||
DialContext: netutil.NewStatDialFunc("vmagent_remotewrite"),
|
||||
TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
|
||||
MaxConnsPerHost: 2 * concurrency,
|
||||
MaxIdleConnsPerHost: 2 * concurrency,
|
||||
IdleConnTimeout: time.Minute,
|
||||
WriteBufferSize: 64 * 1024,
|
||||
}
|
||||
|
||||
tr := httputil.NewTransport(false, "vmagent_remotewrite")
|
||||
tr.TLSHandshakeTimeout = tlsHandshakeTimeout.GetOptionalArg(argIdx)
|
||||
tr.MaxConnsPerHost = 2 * concurrency
|
||||
tr.MaxIdleConnsPerHost = 2 * concurrency
|
||||
tr.IdleConnTimeout = time.Minute
|
||||
tr.WriteBufferSize = 64 * 1024
|
||||
|
||||
pURL := proxyURL.GetOptionalArg(argIdx)
|
||||
if len(pURL) > 0 {
|
||||
if !strings.Contains(pURL, "://") {
|
||||
|
@ -166,17 +170,11 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
|
|||
logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
|
||||
}
|
||||
if !useVMProto && !usePromProto {
|
||||
// Auto-detect whether the remote storage supports VictoriaMetrics remote write protocol.
|
||||
doRequest := func(url string) (*http.Response, error) {
|
||||
return c.doRequest(url, nil)
|
||||
}
|
||||
useVMProto = common.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
|
||||
if !useVMProto {
|
||||
logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
|
||||
"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", sanitizedURL)
|
||||
}
|
||||
// The VM protocol could be downgraded later at runtime if unsupported media type response status is received.
|
||||
useVMProto = true
|
||||
c.canDowngradeVMProto.Store(true)
|
||||
}
|
||||
c.useVMProto = useVMProto
|
||||
c.useVMProto.Store(useVMProto)
|
||||
|
||||
return c
|
||||
}
|
||||
|
@ -384,7 +382,7 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
|
|||
h := req.Header
|
||||
h.Set("User-Agent", "vmagent")
|
||||
h.Set("Content-Type", "application/x-protobuf")
|
||||
if c.useVMProto {
|
||||
if encoding.IsZstd(body) {
|
||||
h.Set("Content-Encoding", "zstd")
|
||||
h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
|
||||
} else {
|
||||
|
@ -420,7 +418,7 @@ again:
|
|||
if retryDuration > maxRetryDuration {
|
||||
retryDuration = maxRetryDuration
|
||||
}
|
||||
logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
|
||||
remoteWriteRetryLogger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
|
||||
len(block), c.sanitizedURL, err, retryDuration.Seconds())
|
||||
t := timerpool.Get(retryDuration)
|
||||
select {
|
||||
|
@ -433,6 +431,7 @@ again:
|
|||
c.retriesCount.Inc()
|
||||
goto again
|
||||
}
|
||||
|
||||
statusCode := resp.StatusCode
|
||||
if statusCode/100 == 2 {
|
||||
_ = resp.Body.Close()
|
||||
|
@ -441,24 +440,46 @@ again:
|
|||
c.blocksSent.Inc()
|
||||
return true
|
||||
}
|
||||
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
|
||||
if statusCode == 409 || statusCode == 400 {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
|
||||
"failed to read response body: %s",
|
||||
len(block), c.sanitizedURL, statusCode, err)
|
||||
} else {
|
||||
remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
|
||||
len(block), c.sanitizedURL, statusCode, string(body))
|
||||
}
|
||||
// Just drop block on 409 and 400 status codes like Prometheus does.
|
||||
if statusCode == 409 {
|
||||
logBlockRejected(block, c.sanitizedURL, resp)
|
||||
|
||||
// Just drop block on 409 status code like Prometheus does.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
|
||||
// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
|
||||
_ = resp.Body.Close()
|
||||
c.packetsDropped.Inc()
|
||||
return true
|
||||
// - Remote Write v1 specification implicitly expects a `400 Bad Request` when the encoding is not supported.
|
||||
// - Remote Write v2 specification explicitly specifies a `415 Unsupported Media Type` for unsupported encodings.
|
||||
// - Real-world implementations of v1 use both 400 and 415 status codes.
|
||||
// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
|
||||
} else if statusCode == 415 || statusCode == 400 {
|
||||
if c.canDowngradeVMProto.Swap(false) {
|
||||
logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
|
||||
c.useVMProto.Store(false)
|
||||
}
|
||||
|
||||
if encoding.IsZstd(block) {
|
||||
logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
|
||||
|
||||
block = mustRepackBlockFromZstdToSnappy(block)
|
||||
|
||||
c.retriesCount.Inc()
|
||||
_ = resp.Body.Close()
|
||||
goto again
|
||||
}
|
||||
|
||||
// Just drop snappy blocks on 400 or 415 status codes like Prometheus does.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
|
||||
// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
|
||||
logBlockRejected(block, c.sanitizedURL, resp)
|
||||
_ = resp.Body.Close()
|
||||
c.packetsDropped.Inc()
|
||||
return true
|
||||
}
|
||||
|
||||
// Unexpected status code returned
|
||||
|
@ -488,6 +509,7 @@ again:
|
|||
}
|
||||
|
||||
var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
|
||||
var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
|
||||
|
||||
// getRetryDuration returns retry duration.
|
||||
// retryAfterDuration has the highest priority.
|
||||
|
@ -510,6 +532,28 @@ func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.D
|
|||
return retryDuration
|
||||
}
|
||||
|
||||
func mustRepackBlockFromZstdToSnappy(zstdBlock []byte) []byte {
|
||||
plainBlock := make([]byte, 0, len(zstdBlock)*2)
|
||||
plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
|
||||
if err != nil {
|
||||
logger.Panicf("FATAL: cannot re-pack block with size %d bytes from Zstd to Snappy: %s", len(zstdBlock), err)
|
||||
}
|
||||
|
||||
return snappy.Encode(nil, plainBlock)
|
||||
}
|
||||
|
||||
func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
|
||||
"failed to read response body: %s",
|
||||
len(block), sanitizedURL, resp.StatusCode, err)
|
||||
} else {
|
||||
remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
|
||||
len(block), sanitizedURL, resp.StatusCode, string(body))
|
||||
}
|
||||
}
|
||||
|
||||
// parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
|
||||
// retryAfterString should be in either HTTP-date or a number of seconds.
|
||||
// It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.
|
||||
|
|
|
@ -5,6 +5,9 @@ import (
|
|||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
||||
"github.com/golang/snappy"
|
||||
)
|
||||
|
||||
func TestCalculateRetryDuration(t *testing.T) {
|
||||
|
@ -97,3 +100,19 @@ func helper(d time.Duration) time.Duration {
|
|||
|
||||
return d + dv
|
||||
}
|
||||
|
||||
func TestRepackBlockFromZstdToSnappy(t *testing.T) {
|
||||
expectedPlainBlock := []byte(`foobar`)
|
||||
|
||||
zstdBlock := encoding.CompressZSTDLevel(nil, expectedPlainBlock, 1)
|
||||
snappyBlock := mustRepackBlockFromZstdToSnappy(zstdBlock)
|
||||
|
||||
actualPlainBlock, err := snappy.Decode(nil, snappyBlock)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
if string(actualPlainBlock) != string(expectedPlainBlock) {
|
||||
t.Fatalf("unexpected plain block; got %q; want %q", actualPlainBlock, expectedPlainBlock)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -16,6 +16,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/golang/snappy"
|
||||
|
@ -28,7 +29,7 @@ var (
|
|||
maxRowsPerBlock = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
|
||||
vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
|
||||
"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
|
||||
"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
|
||||
"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
|
||||
)
|
||||
|
||||
type pendingSeries struct {
|
||||
|
@ -39,7 +40,7 @@ type pendingSeries struct {
|
|||
periodicFlusherWG sync.WaitGroup
|
||||
}
|
||||
|
||||
func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
|
||||
func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Bool, significantFigures, roundDigits int) *pendingSeries {
|
||||
var ps pendingSeries
|
||||
ps.wr.fq = fq
|
||||
ps.wr.isVMRemoteWrite = isVMRemoteWrite
|
||||
|
@ -99,7 +100,7 @@ type writeRequest struct {
|
|||
fq *persistentqueue.FastQueue
|
||||
|
||||
// Whether to encode the write request with VictoriaMetrics remote write protocol.
|
||||
isVMRemoteWrite bool
|
||||
isVMRemoteWrite *atomic.Bool
|
||||
|
||||
// How many significant figures must be left before sending the writeRequest to fq.
|
||||
significantFigures int
|
||||
|
@ -118,7 +119,7 @@ type writeRequest struct {
|
|||
}
|
||||
|
||||
func (wr *writeRequest) reset() {
|
||||
// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
|
||||
// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are reused.
|
||||
|
||||
wr.wr.Timeseries = nil
|
||||
|
||||
|
@ -137,7 +138,7 @@ func (wr *writeRequest) reset() {
|
|||
// This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
|
||||
func (wr *writeRequest) mustFlushOnStop() {
|
||||
wr.wr.Timeseries = wr.tss
|
||||
if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite) {
|
||||
if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite.Load()) {
|
||||
logger.Panicf("BUG: final flush must always return true")
|
||||
}
|
||||
wr.reset()
|
||||
|
@ -151,7 +152,7 @@ func (wr *writeRequest) mustWriteBlock(block []byte) bool {
|
|||
func (wr *writeRequest) tryFlush() bool {
|
||||
wr.wr.Timeseries = wr.tss
|
||||
wr.lastFlushTime.Store(fasttime.UnixTimestamp())
|
||||
if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
|
||||
if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite.Load()) {
|
||||
return false
|
||||
}
|
||||
wr.reset()
|
||||
|
@ -197,28 +198,43 @@ func (wr *writeRequest) tryPush(src []prompbmarshal.TimeSeries) bool {
|
|||
}
|
||||
|
||||
func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
|
||||
labelsDst := wr.labels
|
||||
labelsSrc := src.Labels
|
||||
|
||||
// Pre-allocate memory for labels.
|
||||
labelsLen := len(wr.labels)
|
||||
samplesDst := wr.samples
|
||||
buf := wr.buf
|
||||
for i := range src.Labels {
|
||||
labelsDst = append(labelsDst, prompbmarshal.Label{})
|
||||
dstLabel := &labelsDst[len(labelsDst)-1]
|
||||
srcLabel := &src.Labels[i]
|
||||
wr.labels = slicesutil.SetLength(wr.labels, labelsLen+len(labelsSrc))
|
||||
labelsDst := wr.labels[labelsLen:]
|
||||
|
||||
buf = append(buf, srcLabel.Name...)
|
||||
dstLabel.Name = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Name):])
|
||||
buf = append(buf, srcLabel.Value...)
|
||||
dstLabel.Value = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Value):])
|
||||
// Pre-allocate memory for byte slice needed for storing label names and values.
|
||||
neededBufLen := 0
|
||||
for i := range labelsSrc {
|
||||
label := &labelsSrc[i]
|
||||
neededBufLen += len(label.Name) + len(label.Value)
|
||||
}
|
||||
dst.Labels = labelsDst[labelsLen:]
|
||||
bufLen := len(wr.buf)
|
||||
wr.buf = slicesutil.SetLength(wr.buf, bufLen+neededBufLen)
|
||||
buf := wr.buf[:bufLen]
|
||||
|
||||
samplesDst = append(samplesDst, src.Samples...)
|
||||
dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
|
||||
// Copy labels
|
||||
for i := range labelsSrc {
|
||||
dstLabel := &labelsDst[i]
|
||||
srcLabel := &labelsSrc[i]
|
||||
|
||||
wr.samples = samplesDst
|
||||
wr.labels = labelsDst
|
||||
bufLen := len(buf)
|
||||
buf = append(buf, srcLabel.Name...)
|
||||
dstLabel.Name = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||
|
||||
bufLen = len(buf)
|
||||
buf = append(buf, srcLabel.Value...)
|
||||
dstLabel.Value = bytesutil.ToUnsafeString(buf[bufLen:])
|
||||
}
|
||||
wr.buf = buf
|
||||
dst.Labels = labelsDst
|
||||
|
||||
// Copy samples
|
||||
samplesLen := len(wr.samples)
|
||||
wr.samples = append(wr.samples, src.Samples...)
|
||||
dst.Samples = wr.samples[samplesLen:]
|
||||
}
|
||||
|
||||
// marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.
|
||||
|
|
|
@ -19,10 +19,10 @@ var (
|
|||
relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
|
||||
"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
|
||||
"The path can point either to local file or to http url. "+
|
||||
"See https://docs.victoriametrics.com/vmagent/#relabeling")
|
||||
"See https://docs.victoriametrics.com/victoriametrics/vmagent/#relabeling")
|
||||
relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
|
||||
"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
|
||||
"See https://docs.victoriametrics.com/vmagent/#relabeling")
|
||||
"See https://docs.victoriametrics.com/victoriametrics/vmagent/#relabeling")
|
||||
|
||||
usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
|
||||
"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
|
||||
|
|
|
@ -6,7 +6,7 @@ import (
|
|||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
)
|
||||
|
||||
func TestApplyRelabeling(t *testing.T) {
|
||||
|
@ -63,7 +63,7 @@ func TestAppendExtraLabels(t *testing.T) {
|
|||
func parseSeries(data string) []prompbmarshal.TimeSeries {
|
||||
var tss []prompbmarshal.TimeSeries
|
||||
tss = append(tss, prompbmarshal.TimeSeries{
|
||||
Labels: promutils.MustNewLabelsFromString(data).GetLabels(),
|
||||
Labels: promutil.MustNewLabelsFromString(data).GetLabels(),
|
||||
})
|
||||
return tss
|
||||
}
|
||||
|
|
|
@ -15,6 +15,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
|
@ -25,7 +26,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
|
||||
|
@ -41,20 +42,19 @@ var (
|
|||
enableMultitenantHandlers = flag.Bool("enableMultitenantHandlers", false, "Whether to process incoming data via multitenant insert handlers according to "+
|
||||
"https://docs.victoriametrics.com/cluster-victoriametrics/#url-format . By default incoming data is processed via single-node insert handlers "+
|
||||
"according to https://docs.victoriametrics.com/#how-to-import-time-series-data ."+
|
||||
"See https://docs.victoriametrics.com/vmagent/#multitenancy for details")
|
||||
"See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy for details")
|
||||
|
||||
shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . "+
|
||||
"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages . "+
|
||||
shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url. "+
|
||||
"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages . "+
|
||||
"See also -remoteWrite.shardByURLReplicas")
|
||||
shardByURLReplicas = flag.Int("remoteWrite.shardByURLReplicas", 1, "How many copies of data to make among remote storage systems enumerated via -remoteWrite.url "+
|
||||
"when -remoteWrite.shardByURL is set. See https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages")
|
||||
"when -remoteWrite.shardByURL is set. See https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
|
||||
shardByURLLabels = flagutil.NewArrayString("remoteWrite.shardByURL.labels", "Optional list of labels, which must be used for sharding outgoing samples "+
|
||||
"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
|
||||
"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.ignoreLabels")
|
||||
shardByURLIgnoreLabels = flagutil.NewArrayString("remoteWrite.shardByURL.ignoreLabels", "Optional list of labels, which must be ignored when sharding outgoing samples "+
|
||||
"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
|
||||
"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.labels")
|
||||
|
||||
tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
|
||||
"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
|
||||
keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
|
||||
|
@ -81,28 +81,30 @@ var (
|
|||
`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
|
||||
`Enabled sorting for labels can slow down ingestion performance a bit`)
|
||||
maxHourlySeries = flag.Int("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
|
||||
"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent/#cardinality-limiter")
|
||||
"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
|
||||
maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
|
||||
"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent/#cardinality-limiter")
|
||||
"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
|
||||
maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmagent can receive per second. Data ingestion is paused when the limit is exceeded. "+
|
||||
"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
|
||||
|
||||
disableOnDiskQueue = flagutil.NewArrayBool("remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
|
||||
"when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. "+
|
||||
"See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
|
||||
"See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
|
||||
dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
|
||||
"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
|
||||
"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence")
|
||||
)
|
||||
|
||||
var (
|
||||
// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
|
||||
rwctxsGlobal []*remoteWriteCtx
|
||||
rwctxsGlobal []*remoteWriteCtx
|
||||
rwctxsGlobalIdx []int
|
||||
rwctxConsistentHashGlobal *consistenthash.ConsistentHash
|
||||
|
||||
// ErrQueueFullHTTPRetry must be returned when TryPush() returns false.
|
||||
ErrQueueFullHTTPRetry = &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("remote storage systems cannot keep up with the data ingestion rate; retry the request later " +
|
||||
"or remove -remoteWrite.disableOnDiskQueue from vmagent command-line flags, so it could save pending data to -remoteWrite.tmpDataPath; " +
|
||||
"see https://docs.victoriametrics.com/vmagent/#disabling-on-disk-persistence"),
|
||||
"see https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence"),
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
}
|
||||
|
||||
|
@ -183,7 +185,7 @@ func Init() {
|
|||
|
||||
if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
|
||||
logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
|
||||
"see https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages")
|
||||
"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
|
||||
}
|
||||
shardByURLLabelsMap = newMapFromStrings(*shardByURLLabels)
|
||||
shardByURLIgnoreLabelsMap = newMapFromStrings(*shardByURLIgnoreLabels)
|
||||
|
@ -205,7 +207,7 @@ func Init() {
|
|||
|
||||
initStreamAggrConfigGlobal()
|
||||
|
||||
rwctxsGlobal = newRemoteWriteCtxs(*remoteWriteURLs)
|
||||
initRemoteWriteCtxs(*remoteWriteURLs)
|
||||
|
||||
disableOnDiskQueues := []bool(*disableOnDiskQueue)
|
||||
disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
|
||||
|
@ -290,7 +292,7 @@ var (
|
|||
relabelConfigTimestamp = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
|
||||
)
|
||||
|
||||
func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
|
||||
func initRemoteWriteCtxs(urls []string) {
|
||||
if len(urls) == 0 {
|
||||
logger.Panicf("BUG: urls must be non-empty")
|
||||
}
|
||||
|
@ -306,6 +308,7 @@ func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
|
|||
maxInmemoryBlocks = 2
|
||||
}
|
||||
rwctxs := make([]*remoteWriteCtx, len(urls))
|
||||
rwctxIdx := make([]int, len(urls))
|
||||
for i, remoteWriteURLRaw := range urls {
|
||||
remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
|
||||
if err != nil {
|
||||
|
@ -316,8 +319,19 @@ func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
|
|||
sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
|
||||
}
|
||||
rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
|
||||
rwctxIdx[i] = i
|
||||
}
|
||||
return rwctxs
|
||||
|
||||
if *shardByURL {
|
||||
consistentHashNodes := make([]string, 0, len(urls))
|
||||
for i, url := range urls {
|
||||
consistentHashNodes = append(consistentHashNodes, fmt.Sprintf("%d:%s", i+1, url))
|
||||
}
|
||||
rwctxConsistentHashGlobal = consistenthash.NewConsistentHash(consistentHashNodes, 0)
|
||||
}
|
||||
|
||||
rwctxsGlobal = rwctxs
|
||||
rwctxsGlobalIdx = rwctxIdx
|
||||
}
|
||||
|
||||
var (
|
||||
|
@ -501,6 +515,10 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
|
|||
return true
|
||||
}
|
||||
|
||||
// getEligibleRemoteWriteCtxs checks whether writes to configured remote storage systems are blocked and
|
||||
// returns only the unblocked rwctx.
|
||||
//
|
||||
// calculateHealthyRwctxIdx will rely on the order of rwctx to be in ascending order.
|
||||
func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
|
||||
if !disableOnDiskQueueAny {
|
||||
return rwctxsGlobal, true
|
||||
|
@ -517,6 +535,12 @@ func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamples
|
|||
return nil, false
|
||||
}
|
||||
rowsCount := getRowsCount(tss)
|
||||
if *shardByURL {
|
||||
// Todo: When shardByURL is enabled, the following metrics won't be 100% accurate. Because vmagent don't know
|
||||
// which rwctx should data be pushed to yet. Let's consider the hashing algorithm fair and will distribute
|
||||
// data to all rwctxs evenly.
|
||||
rowsCount = rowsCount / len(rwctxsGlobal)
|
||||
}
|
||||
rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
|
||||
}
|
||||
}
|
||||
|
@ -528,6 +552,7 @@ func pushToRemoteStoragesTrackDropped(tss []prompbmarshal.TimeSeries) {
|
|||
if len(rwctxs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
|
||||
logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
|
||||
}
|
||||
|
@ -578,42 +603,7 @@ func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []pr
|
|||
defer putTSSShards(x)
|
||||
|
||||
shards := x.shards
|
||||
tmpLabels := promutils.GetLabels()
|
||||
for _, ts := range tssBlock {
|
||||
hashLabels := ts.Labels
|
||||
if len(shardByURLLabelsMap) > 0 {
|
||||
hashLabels = tmpLabels.Labels[:0]
|
||||
for _, label := range ts.Labels {
|
||||
if _, ok := shardByURLLabelsMap[label.Name]; ok {
|
||||
hashLabels = append(hashLabels, label)
|
||||
}
|
||||
}
|
||||
tmpLabels.Labels = hashLabels
|
||||
} else if len(shardByURLIgnoreLabelsMap) > 0 {
|
||||
hashLabels = tmpLabels.Labels[:0]
|
||||
for _, label := range ts.Labels {
|
||||
if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
|
||||
hashLabels = append(hashLabels, label)
|
||||
}
|
||||
}
|
||||
tmpLabels.Labels = hashLabels
|
||||
}
|
||||
h := getLabelsHash(hashLabels)
|
||||
idx := h % uint64(len(shards))
|
||||
i := 0
|
||||
for {
|
||||
shards[idx] = append(shards[idx], ts)
|
||||
i++
|
||||
if i >= replicas {
|
||||
break
|
||||
}
|
||||
idx++
|
||||
if idx >= uint64(len(shards)) {
|
||||
idx = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
promutils.PutLabels(tmpLabels)
|
||||
shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
|
||||
|
||||
// Push sharded samples to remote storage systems in parallel in order to reduce
|
||||
// the time needed for sending the data to multiple remote storage systems.
|
||||
|
@ -636,6 +626,86 @@ func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []pr
|
|||
return !anyPushFailed.Load()
|
||||
}
|
||||
|
||||
// calculateHealthyRwctxIdx returns the index of healthyRwctxs in rwctxsGlobal.
|
||||
// It relies on the order of rwctx in healthyRwctxs, which is appended by getEligibleRemoteWriteCtxs.
|
||||
func calculateHealthyRwctxIdx(healthyRwctxs []*remoteWriteCtx) ([]int, []int) {
|
||||
// fast path: all rwctxs are healthy.
|
||||
if len(healthyRwctxs) == len(rwctxsGlobal) {
|
||||
return rwctxsGlobalIdx, nil
|
||||
}
|
||||
|
||||
unhealthyIdx := make([]int, 0, len(rwctxsGlobal))
|
||||
healthyIdx := make([]int, 0, len(rwctxsGlobal))
|
||||
|
||||
var i int
|
||||
for j := range rwctxsGlobal {
|
||||
if i < len(healthyRwctxs) && rwctxsGlobal[j].idx == healthyRwctxs[i].idx {
|
||||
healthyIdx = append(healthyIdx, j)
|
||||
i++
|
||||
} else {
|
||||
unhealthyIdx = append(unhealthyIdx, j)
|
||||
}
|
||||
}
|
||||
|
||||
return healthyIdx, unhealthyIdx
|
||||
}
|
||||
|
||||
// shardAmountRemoteWriteCtx distribute time series to shards by consistent hashing.
|
||||
func shardAmountRemoteWriteCtx(tssBlock []prompbmarshal.TimeSeries, shards [][]prompbmarshal.TimeSeries, rwctxs []*remoteWriteCtx, replicas int) {
|
||||
tmpLabels := promutil.GetLabels()
|
||||
defer promutil.PutLabels(tmpLabels)
|
||||
|
||||
healthyIdx, unhealthyIdx := calculateHealthyRwctxIdx(rwctxs)
|
||||
|
||||
// shardsIdxMap is a map to find which the shard idx by rwctxs idx.
|
||||
// rwctxConsistentHashGlobal will tell which the rwctxs idx a time series should be written to.
|
||||
// And this time series should be appended to the shards by correct shard idx.
|
||||
shardsIdxMap := make(map[int]int, len(healthyIdx))
|
||||
for idx, rwctxsIdx := range healthyIdx {
|
||||
shardsIdxMap[rwctxsIdx] = idx
|
||||
}
|
||||
|
||||
for _, ts := range tssBlock {
|
||||
hashLabels := ts.Labels
|
||||
if len(shardByURLLabelsMap) > 0 {
|
||||
hashLabels = tmpLabels.Labels[:0]
|
||||
for _, label := range ts.Labels {
|
||||
if _, ok := shardByURLLabelsMap[label.Name]; ok {
|
||||
hashLabels = append(hashLabels, label)
|
||||
}
|
||||
}
|
||||
tmpLabels.Labels = hashLabels
|
||||
} else if len(shardByURLIgnoreLabelsMap) > 0 {
|
||||
hashLabels = tmpLabels.Labels[:0]
|
||||
for _, label := range ts.Labels {
|
||||
if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
|
||||
hashLabels = append(hashLabels, label)
|
||||
}
|
||||
}
|
||||
tmpLabels.Labels = hashLabels
|
||||
}
|
||||
h := getLabelsHash(hashLabels)
|
||||
|
||||
// Get the rwctxIdx through consistent hashing and then map it to the index in shards.
|
||||
// The rwctxIdx is not always equal to the shardIdx, for example, when some rwctx are not available.
|
||||
rwctxIdx := rwctxConsistentHashGlobal.GetNodeIdx(h, unhealthyIdx)
|
||||
shardIdx := shardsIdxMap[rwctxIdx]
|
||||
|
||||
replicated := 0
|
||||
for {
|
||||
shards[shardIdx] = append(shards[shardIdx], ts)
|
||||
replicated++
|
||||
if replicated >= replicas {
|
||||
break
|
||||
}
|
||||
shardIdx++
|
||||
if shardIdx >= len(shards) {
|
||||
shardIdx = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type tssShards struct {
|
||||
shards [][]prompbmarshal.TimeSeries
|
||||
}
|
||||
|
@ -807,7 +877,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
|
|||
}
|
||||
pss := make([]*pendingSeries, pssLen)
|
||||
for i := range pss {
|
||||
pss[i] = newPendingSeries(fq, c.useVMProto, sf, rd)
|
||||
pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
|
||||
}
|
||||
|
||||
rwctx := &remoteWriteCtx{
|
||||
|
|
|
@ -4,13 +4,17 @@ import (
|
|||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
|
@ -68,7 +72,9 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
|
|||
allRelabelConfigs.Store(rcs)
|
||||
|
||||
pss := make([]*pendingSeries, 1)
|
||||
pss[0] = newPendingSeries(nil, true, 0, 100)
|
||||
isVMProto := &atomic.Bool{}
|
||||
isVMProto.Store(true)
|
||||
pss[0] = newPendingSeries(nil, isVMProto, 0, 100)
|
||||
rwctx := &remoteWriteCtx{
|
||||
idx: 0,
|
||||
streamAggrKeepInput: keepInput,
|
||||
|
@ -171,3 +177,173 @@ metric{env="dev"} 15
|
|||
metric{env="bar"} 25
|
||||
`)
|
||||
}
|
||||
|
||||
func TestShardAmountRemoteWriteCtx(t *testing.T) {
|
||||
// 1. distribute 100000 series to n nodes.
|
||||
// 2. remove the last node from healthy list.
|
||||
// 3. distribute the same 10000 series to (n-1) node again.
|
||||
// 4. check active time series change rate:
|
||||
// change rate must < (3/total nodes). e.g. +30% if 10 you have 10 nodes.
|
||||
|
||||
f := func(remoteWriteCount int, healthyIdx []int, replicas int) {
|
||||
t.Helper()
|
||||
defer func() {
|
||||
rwctxsGlobal = nil
|
||||
rwctxsGlobalIdx = nil
|
||||
rwctxConsistentHashGlobal = nil
|
||||
}()
|
||||
|
||||
rwctxsGlobal = make([]*remoteWriteCtx, remoteWriteCount)
|
||||
rwctxsGlobalIdx = make([]int, remoteWriteCount)
|
||||
rwctxs := make([]*remoteWriteCtx, 0, len(healthyIdx))
|
||||
|
||||
for i := range remoteWriteCount {
|
||||
rwCtx := &remoteWriteCtx{
|
||||
idx: i,
|
||||
}
|
||||
rwctxsGlobalIdx[i] = i
|
||||
|
||||
if i >= len(healthyIdx) {
|
||||
rwctxsGlobal[i] = rwCtx
|
||||
continue
|
||||
}
|
||||
hIdx := healthyIdx[i]
|
||||
if hIdx != i {
|
||||
rwctxs = append(rwctxs, &remoteWriteCtx{
|
||||
idx: hIdx,
|
||||
})
|
||||
} else {
|
||||
rwctxs = append(rwctxs, rwCtx)
|
||||
}
|
||||
rwctxsGlobal[i] = rwCtx
|
||||
}
|
||||
|
||||
seriesCount := 100000
|
||||
// build 1000000 series
|
||||
tssBlock := make([]prompbmarshal.TimeSeries, 0, seriesCount)
|
||||
for i := 0; i < seriesCount; i++ {
|
||||
tssBlock = append(tssBlock, prompbmarshal.TimeSeries{
|
||||
Labels: []prompbmarshal.Label{
|
||||
{
|
||||
Name: "label",
|
||||
Value: strconv.Itoa(i),
|
||||
},
|
||||
},
|
||||
Samples: []prompbmarshal.Sample{
|
||||
{
|
||||
Timestamp: 0,
|
||||
Value: 0,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// build consistent hash for x remote write context
|
||||
// build active time series set
|
||||
nodes := make([]string, 0, remoteWriteCount)
|
||||
activeTimeSeriesByNodes := make([]map[string]struct{}, remoteWriteCount)
|
||||
for i := 0; i < remoteWriteCount; i++ {
|
||||
nodes = append(nodes, fmt.Sprintf("node%d", i))
|
||||
activeTimeSeriesByNodes[i] = make(map[string]struct{})
|
||||
}
|
||||
rwctxConsistentHashGlobal = consistenthash.NewConsistentHash(nodes, 0)
|
||||
|
||||
// create shards
|
||||
x := getTSSShards(len(rwctxs))
|
||||
shards := x.shards
|
||||
|
||||
// execute
|
||||
shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
|
||||
|
||||
for i, nodeIdx := range healthyIdx {
|
||||
for _, ts := range shards[i] {
|
||||
// add it to node[nodeIdx]'s active time series
|
||||
activeTimeSeriesByNodes[nodeIdx][prompbmarshal.LabelsToString(ts.Labels)] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
totalActiveTimeSeries := 0
|
||||
for _, activeTimeSeries := range activeTimeSeriesByNodes {
|
||||
totalActiveTimeSeries += len(activeTimeSeries)
|
||||
}
|
||||
avgActiveTimeSeries1 := totalActiveTimeSeries / remoteWriteCount
|
||||
putTSSShards(x)
|
||||
|
||||
// removed last node
|
||||
rwctxs = rwctxs[:len(rwctxs)-1]
|
||||
healthyIdx = healthyIdx[:len(healthyIdx)-1]
|
||||
|
||||
x = getTSSShards(len(rwctxs))
|
||||
shards = x.shards
|
||||
|
||||
// execute
|
||||
shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
|
||||
for i, nodeIdx := range healthyIdx {
|
||||
for _, ts := range shards[i] {
|
||||
// add it to node[nodeIdx]'s active time series
|
||||
activeTimeSeriesByNodes[nodeIdx][prompbmarshal.LabelsToString(ts.Labels)] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
totalActiveTimeSeries = 0
|
||||
for _, activeTimeSeries := range activeTimeSeriesByNodes {
|
||||
totalActiveTimeSeries += len(activeTimeSeries)
|
||||
}
|
||||
avgActiveTimeSeries2 := totalActiveTimeSeries / remoteWriteCount
|
||||
|
||||
changed := math.Abs(float64(avgActiveTimeSeries2-avgActiveTimeSeries1) / float64(avgActiveTimeSeries1))
|
||||
threshold := 3 / float64(remoteWriteCount)
|
||||
|
||||
if changed >= threshold {
|
||||
t.Fatalf("average active time series before: %d, after: %d, changed: %.2f. threshold: %.2f", avgActiveTimeSeries1, avgActiveTimeSeries2, changed, threshold)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
f(5, []int{0, 1, 2, 3, 4}, 1)
|
||||
|
||||
f(5, []int{0, 1, 2, 3, 4}, 2)
|
||||
|
||||
f(10, []int{0, 1, 2, 3, 4, 5, 6, 7, 9}, 1)
|
||||
|
||||
f(10, []int{0, 1, 2, 3, 4, 5, 6, 7, 9}, 3)
|
||||
}
|
||||
|
||||
func TestCalculateHealthyRwctxIdx(t *testing.T) {
|
||||
f := func(total int, healthyIdx []int, unhealthyIdx []int) {
|
||||
t.Helper()
|
||||
|
||||
healthyMap := make(map[int]bool)
|
||||
for _, idx := range healthyIdx {
|
||||
healthyMap[idx] = true
|
||||
}
|
||||
rwctxsGlobal = make([]*remoteWriteCtx, total)
|
||||
rwctxsGlobalIdx = make([]int, total)
|
||||
rwctxs := make([]*remoteWriteCtx, 0, len(healthyIdx))
|
||||
for i := range rwctxsGlobal {
|
||||
rwctx := &remoteWriteCtx{idx: i}
|
||||
rwctxsGlobal[i] = rwctx
|
||||
if healthyMap[i] {
|
||||
rwctxs = append(rwctxs, rwctx)
|
||||
}
|
||||
rwctxsGlobalIdx[i] = i
|
||||
}
|
||||
|
||||
gotHealthyIdx, gotUnhealthyIdx := calculateHealthyRwctxIdx(rwctxs)
|
||||
if !reflect.DeepEqual(healthyIdx, gotHealthyIdx) {
|
||||
t.Errorf("calculateHealthyRwctxIdx want healthyIdx = %v, got %v", healthyIdx, gotHealthyIdx)
|
||||
}
|
||||
if !reflect.DeepEqual(unhealthyIdx, gotUnhealthyIdx) {
|
||||
t.Errorf("calculateHealthyRwctxIdx want unhealthyIdx = %v, got %v", unhealthyIdx, gotUnhealthyIdx)
|
||||
}
|
||||
}
|
||||
|
||||
f(5, []int{0, 1, 2, 3, 4}, nil)
|
||||
f(5, []int{0, 1, 2, 4}, []int{3})
|
||||
f(5, []int{2, 4}, []int{0, 1, 3})
|
||||
f(5, []int{0, 2, 4}, []int{1, 3})
|
||||
f(5, []int{}, []int{0, 1, 2, 3, 4})
|
||||
f(5, []int{4}, []int{0, 1, 2, 3})
|
||||
f(1, []int{0}, nil)
|
||||
f(1, []int{}, []int{0})
|
||||
}
|
||||
|
|
|
@ -56,7 +56,7 @@ var (
|
|||
"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
|
||||
streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
|
||||
"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
|
||||
"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
|
||||
"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving buffered delayed data from clients pushing data into the vmagent. "+
|
||||
"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
|
||||
streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
|
||||
"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
|
||||
|
@ -131,7 +131,7 @@ func reloadStreamAggrConfigGlobal() {
|
|||
func initStreamAggrConfigGlobal() {
|
||||
sas, err := newStreamAggrConfigGlobal()
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot initialize gloabl stream aggregators: %s", err)
|
||||
logger.Fatalf("cannot initialize global stream aggregators: %s", err)
|
||||
}
|
||||
if sas != nil {
|
||||
filePath := sas.FilePath()
|
||||
|
|
|
@ -9,8 +9,8 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
@ -26,17 +26,17 @@ var (
|
|||
//
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
|
||||
func InsertHandler(at *auth.Token, req *http.Request) error {
|
||||
extraLabels, err := parserCommon.GetExtraLabels(req)
|
||||
extraLabels, err := protoparserutil.GetExtraLabels(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
|
||||
return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
|
||||
encoding := req.Header.Get("Content-Encoding")
|
||||
return stream.Parse(req.Body, encoding, func(rows []vmimport.Row) error {
|
||||
return insertRows(at, rows, extraLabels)
|
||||
})
|
||||
}
|
||||
|
||||
func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
|
||||
func insertRows(at *auth.Token, rows []vmimport.Row, extraLabels []prompbmarshal.Label) error {
|
||||
ctx := common.GetPushCtx()
|
||||
defer common.PutPushCtx(ctx)
|
||||
|
||||
|
|
|
@ -1,3 +1,3 @@
|
|||
See vmalert-tool docs [here](https://docs.victoriametrics.com/vmalert-tool.html).
|
||||
See vmalert-tool docs [here](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/).
|
||||
|
||||
vmalert-tool docs can be edited at [docs/vmalert-tool.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmalert-tool.md).
|
||||
vmalert-tool docs can be edited at [docs/vmalert-tool.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmalert-tool.md).
|
||||
|
|
8
app/vmalert-tool/deployment/Dockerfile
Normal file
8
app/vmalert-tool/deployment/Dockerfile
Normal file
|
@ -0,0 +1,8 @@
|
|||
ARG base_image=non-existing
|
||||
FROM $base_image
|
||||
|
||||
EXPOSE 8880
|
||||
|
||||
ENTRYPOINT ["/vmalert-tool-prod"]
|
||||
ARG src_binary=non-existing
|
||||
COPY $src_binary ./vmalert-tool-prod
|
|
@ -17,13 +17,13 @@ func main() {
|
|||
app := &cli.App{
|
||||
Name: "vmalert-tool",
|
||||
Usage: "VMAlert command-line tool",
|
||||
UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html",
|
||||
UsageText: "More info in https://docs.victoriametrics.com/victoriametrics/vmalert-tool/",
|
||||
Version: buildinfo.Version,
|
||||
Commands: []*cli.Command{
|
||||
{
|
||||
Name: "unittest",
|
||||
Usage: "Run unittest for alerting and recording rules.",
|
||||
UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html#Unit-testing-for-rules",
|
||||
UsageText: "More info in https://docs.victoriametrics.com/victoriametrics/vmalert-tool/#unit-testing-for-rules",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringSliceFlag{
|
||||
Name: "files",
|
||||
|
|
|
@ -1,15 +1,15 @@
|
|||
package unittest
|
||||
|
||||
import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
)
|
||||
|
||||
// alertTestCase holds alert_rule_test cases defined in test file
|
||||
type alertTestCase struct {
|
||||
EvalTime *promutils.Duration `yaml:"eval_time"`
|
||||
GroupName string `yaml:"groupname"`
|
||||
Alertname string `yaml:"alertname"`
|
||||
ExpAlerts []expAlert `yaml:"exp_alerts"`
|
||||
EvalTime *promutil.Duration `yaml:"eval_time"`
|
||||
GroupName string `yaml:"groupname"`
|
||||
Alertname string `yaml:"alertname"`
|
||||
ExpAlerts []expAlert `yaml:"exp_alerts"`
|
||||
}
|
||||
|
||||
// expAlert holds exp_alerts defined in test file
|
||||
|
|
|
@ -14,7 +14,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
"github.com/VictoriaMetrics/metricsql"
|
||||
)
|
||||
|
||||
|
@ -41,7 +41,7 @@ func httpWrite(address string, r io.Reader) {
|
|||
}
|
||||
|
||||
// writeInputSeries send input series to vmstorage and flush them
|
||||
func writeInputSeries(input []series, interval *promutils.Duration, startStamp time.Time, dst string) error {
|
||||
func writeInputSeries(input []series, interval *promutil.Duration, startStamp time.Time, dst string) error {
|
||||
r := testutil.WriteRequest{}
|
||||
var err error
|
||||
r.Timeseries, err = parseInputSeries(input, interval, startStamp)
|
||||
|
@ -56,7 +56,7 @@ func writeInputSeries(input []series, interval *promutils.Duration, startStamp t
|
|||
return nil
|
||||
}
|
||||
|
||||
func parseInputSeries(input []series, interval *promutils.Duration, startStamp time.Time) ([]testutil.TimeSeries, error) {
|
||||
func parseInputSeries(input []series, interval *promutil.Duration, startStamp time.Time) ([]testutil.TimeSeries, error) {
|
||||
var res []testutil.TimeSeries
|
||||
for _, data := range input {
|
||||
expr, err := metricsql.Parse(data.Series)
|
||||
|
|
|
@ -5,7 +5,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
)
|
||||
|
||||
func TestParseInputValue_Failure(t *testing.T) {
|
||||
|
@ -70,7 +70,7 @@ func TestParseInputValue_Success(t *testing.T) {
|
|||
func TestParseInputSeries_Success(t *testing.T) {
|
||||
f := func(input []series) {
|
||||
t.Helper()
|
||||
var interval promutils.Duration
|
||||
var interval promutil.Duration
|
||||
_, err := parseInputSeries(input, &interval, time.Now())
|
||||
if err != nil {
|
||||
t.Fatalf("expect to see no error: %v", err)
|
||||
|
@ -86,7 +86,7 @@ func TestParseInputSeries_Success(t *testing.T) {
|
|||
func TestParseInputSeries_Fail(t *testing.T) {
|
||||
f := func(input []series) {
|
||||
t.Helper()
|
||||
var interval promutils.Duration
|
||||
var interval promutil.Duration
|
||||
_, err := parseInputSeries(input, &interval, time.Now())
|
||||
if err == nil {
|
||||
t.Fatalf("expect to see error: %v", err)
|
||||
|
|
|
@ -10,15 +10,15 @@ import (
|
|||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
"github.com/VictoriaMetrics/metricsql"
|
||||
)
|
||||
|
||||
// metricsqlTestCase holds metricsql_expr_test cases defined in test file
|
||||
type metricsqlTestCase struct {
|
||||
Expr string `yaml:"expr"`
|
||||
EvalTime *promutils.Duration `yaml:"eval_time"`
|
||||
ExpSamples []expSample `yaml:"exp_samples"`
|
||||
Expr string `yaml:"expr"`
|
||||
EvalTime *promutil.Duration `yaml:"eval_time"`
|
||||
ExpSamples []expSample `yaml:"exp_samples"`
|
||||
}
|
||||
|
||||
type expSample struct {
|
||||
|
@ -95,7 +95,7 @@ Outer:
|
|||
return
|
||||
}
|
||||
|
||||
func durationToTime(pd *promutils.Duration) time.Time {
|
||||
func durationToTime(pd *promutil.Duration) time.Time {
|
||||
if pd == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
|
|
|
@ -36,7 +36,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
|
@ -182,7 +182,7 @@ func ruleUnitTest(filename string, content []byte, externalLabels map[string]str
|
|||
|
||||
if unitTestInp.EvaluationInterval.Duration() == 0 {
|
||||
fmt.Println("evaluation_interval set to 1m by default")
|
||||
unitTestInp.EvaluationInterval = &promutils.Duration{D: 1 * time.Minute}
|
||||
unitTestInp.EvaluationInterval = &promutil.Duration{D: 1 * time.Minute}
|
||||
}
|
||||
|
||||
groupOrderMap := make(map[string]int)
|
||||
|
@ -312,7 +312,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
|
|||
defer tearDown()
|
||||
|
||||
if tg.Interval == nil {
|
||||
tg.Interval = promutils.NewDuration(evalInterval)
|
||||
tg.Interval = promutil.NewDuration(evalInterval)
|
||||
}
|
||||
err := writeInputSeries(tg.InputSeries, tg.Interval, testStartTime, fmt.Sprintf("http://127.0.0.1:%s/api/v1/write", httpListenAddr))
|
||||
if err != nil {
|
||||
|
@ -472,15 +472,15 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
|
|||
|
||||
// unitTestFile holds the contents of a single unit test file
|
||||
type unitTestFile struct {
|
||||
RuleFiles []string `yaml:"rule_files"`
|
||||
EvaluationInterval *promutils.Duration `yaml:"evaluation_interval"`
|
||||
GroupEvalOrder []string `yaml:"group_eval_order"`
|
||||
Tests []testGroup `yaml:"tests"`
|
||||
RuleFiles []string `yaml:"rule_files"`
|
||||
EvaluationInterval *promutil.Duration `yaml:"evaluation_interval"`
|
||||
GroupEvalOrder []string `yaml:"group_eval_order"`
|
||||
Tests []testGroup `yaml:"tests"`
|
||||
}
|
||||
|
||||
// testGroup is a group of input series and test cases associated with it
|
||||
type testGroup struct {
|
||||
Interval *promutils.Duration `yaml:"interval"`
|
||||
Interval *promutil.Duration `yaml:"interval"`
|
||||
InputSeries []series `yaml:"input_series"`
|
||||
AlertRuleTests []alertTestCase `yaml:"alert_rule_test"`
|
||||
MetricsqlExprTests []metricsqlTestCase `yaml:"metricsql_expr_test"`
|
||||
|
|
|
@ -74,7 +74,7 @@ test-vmalert:
|
|||
go test -v -race -cover ./app/vmalert/notifier
|
||||
go test -v -race -cover ./app/vmalert/config
|
||||
go test -v -race -cover ./app/vmalert/remotewrite
|
||||
go test -v -race -cover ./app/vmalert/utils
|
||||
go test -v -race -cover ./app/vmalert/vmalertutil
|
||||
|
||||
run-vmalert: vmalert
|
||||
./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules2-good.rules \
|
||||
|
|
|
@ -1,3 +1,3 @@
|
|||
See vmalert docs [here](https://docs.victoriametrics.com/vmalert/).
|
||||
See vmalert docs [here](https://docs.victoriametrics.com/victoriametrics/vmalert/).
|
||||
|
||||
vmalert docs can be edited at [docs/vmalert.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmalert.md).
|
||||
vmalert docs can be edited at [docs/vmalert.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmalert.md).
|
||||
|
|
|
@ -2,7 +2,6 @@ package config
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"flag"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
|
@ -11,15 +10,16 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/log"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/log"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultRuleType = flag.String("rule.defaultRuleType", "prometheus", `Default type for rule expressions, can be overridden via "type" parameter on the group level, see https://docs.victoriametrics.com/vmalert/#groups. Supported values: "graphite", "prometheus" and "vlogs".`)
|
||||
defaultRuleType = flag.String("rule.defaultRuleType", "prometheus", `Default type for rule expressions, can be overridden via "type" parameter on the group level, see https://docs.victoriametrics.com/victoriametrics/vmalert/#groups. Supported values: "graphite", "prometheus" and "vlogs".`)
|
||||
)
|
||||
|
||||
// Group contains list of Rules grouped into
|
||||
|
@ -27,15 +27,15 @@ var (
|
|||
type Group struct {
|
||||
Type Type `yaml:"type,omitempty"`
|
||||
File string
|
||||
Name string `yaml:"name"`
|
||||
Interval *promutils.Duration `yaml:"interval,omitempty"`
|
||||
EvalOffset *promutils.Duration `yaml:"eval_offset,omitempty"`
|
||||
Name string `yaml:"name"`
|
||||
Interval *promutil.Duration `yaml:"interval,omitempty"`
|
||||
EvalOffset *promutil.Duration `yaml:"eval_offset,omitempty"`
|
||||
// EvalDelay will adjust the `time` parameter of rule evaluation requests to compensate intentional query delay from datasource.
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155
|
||||
EvalDelay *promutils.Duration `yaml:"eval_delay,omitempty"`
|
||||
Limit int `yaml:"limit,omitempty"`
|
||||
Rules []Rule `yaml:"rules"`
|
||||
Concurrency int `yaml:"concurrency"`
|
||||
EvalDelay *promutil.Duration `yaml:"eval_delay,omitempty"`
|
||||
Limit int `yaml:"limit,omitempty"`
|
||||
Rules []Rule `yaml:"rules"`
|
||||
Concurrency int `yaml:"concurrency"`
|
||||
// Labels is a set of label value pairs, that will be added to every rule.
|
||||
// It has priority over the external labels.
|
||||
Labels map[string]string `yaml:"labels"`
|
||||
|
@ -67,7 +67,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
|
|||
if g.Type.Get() == "" {
|
||||
g.Type = NewRawType(*defaultRuleType)
|
||||
}
|
||||
h := md5.New()
|
||||
h := fnv.New64a()
|
||||
h.Write(b)
|
||||
g.Checksum = fmt.Sprintf("%x", h.Sum(nil))
|
||||
return nil
|
||||
|
@ -135,15 +135,15 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
|
|||
// recording rule or alerting rule.
|
||||
type Rule struct {
|
||||
ID uint64
|
||||
Record string `yaml:"record,omitempty"`
|
||||
Alert string `yaml:"alert,omitempty"`
|
||||
Expr string `yaml:"expr"`
|
||||
For *promutils.Duration `yaml:"for,omitempty"`
|
||||
Record string `yaml:"record,omitempty"`
|
||||
Alert string `yaml:"alert,omitempty"`
|
||||
Expr string `yaml:"expr"`
|
||||
For *promutil.Duration `yaml:"for,omitempty"`
|
||||
// Alert will continue firing for this long even when the alerting expression no longer has results.
|
||||
KeepFiringFor *promutils.Duration `yaml:"keep_firing_for,omitempty"`
|
||||
Labels map[string]string `yaml:"labels,omitempty"`
|
||||
Annotations map[string]string `yaml:"annotations,omitempty"`
|
||||
Debug bool `yaml:"debug,omitempty"`
|
||||
KeepFiringFor *promutil.Duration `yaml:"keep_firing_for,omitempty"`
|
||||
Labels map[string]string `yaml:"labels,omitempty"`
|
||||
Annotations map[string]string `yaml:"annotations,omitempty"`
|
||||
Debug bool `yaml:"debug,omitempty"`
|
||||
// UpdateEntriesLimit defines max number of rule's state updates stored in memory.
|
||||
// Overrides `-rule.updateEntriesLimit`.
|
||||
UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`
|
||||
|
@ -265,7 +265,7 @@ func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressio
|
|||
}
|
||||
|
||||
func parse(files map[string][]byte, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
|
||||
errGroup := new(utils.ErrGroup)
|
||||
errGroup := new(vmalertutil.ErrGroup)
|
||||
var groups []Group
|
||||
for file, data := range files {
|
||||
uniqueGroups := map[string]struct{}{}
|
||||
|
|
|
@ -11,7 +11,7 @@ import (
|
|||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
|
@ -162,7 +162,7 @@ func TestGroupValidate_Failure(t *testing.T) {
|
|||
Rules: []Rule{
|
||||
{
|
||||
Expr: "sum(up == 0 ) by (host)",
|
||||
For: promutils.NewDuration(10 * time.Millisecond),
|
||||
For: promutil.NewDuration(10 * time.Millisecond),
|
||||
},
|
||||
{
|
||||
Expr: "sumSeries(time('foo.bar',10))",
|
||||
|
@ -172,13 +172,13 @@ func TestGroupValidate_Failure(t *testing.T) {
|
|||
|
||||
f(&Group{
|
||||
Name: "negative interval",
|
||||
Interval: promutils.NewDuration(-1),
|
||||
Interval: promutil.NewDuration(-1),
|
||||
}, false, "interval shouldn't be lower than 0")
|
||||
|
||||
f(&Group{
|
||||
Name: "wrong eval_offset",
|
||||
Interval: promutils.NewDuration(time.Minute),
|
||||
EvalOffset: promutils.NewDuration(2 * time.Minute),
|
||||
Interval: promutil.NewDuration(time.Minute),
|
||||
EvalOffset: promutil.NewDuration(2 * time.Minute),
|
||||
}, false, "eval_offset should be smaller than interval")
|
||||
|
||||
f(&Group{
|
||||
|
@ -309,7 +309,7 @@ func TestGroupValidate_Failure(t *testing.T) {
|
|||
Record: "r1",
|
||||
ID: 1,
|
||||
Expr: "sumSeries(time('foo.bar',10))",
|
||||
For: promutils.NewDuration(10 * time.Millisecond),
|
||||
For: promutil.NewDuration(10 * time.Millisecond),
|
||||
},
|
||||
{
|
||||
Record: "r2",
|
||||
|
@ -326,7 +326,7 @@ func TestGroupValidate_Failure(t *testing.T) {
|
|||
{
|
||||
Record: "r1",
|
||||
Expr: "sum(up == 0 ) by (host)",
|
||||
For: promutils.NewDuration(10 * time.Millisecond),
|
||||
For: promutil.NewDuration(10 * time.Millisecond),
|
||||
},
|
||||
},
|
||||
}, true, "bad LogsQL expr")
|
||||
|
@ -338,7 +338,7 @@ func TestGroupValidate_Failure(t *testing.T) {
|
|||
{
|
||||
Record: "r1",
|
||||
Expr: "* | stats by (path) count()",
|
||||
For: promutils.NewDuration(10 * time.Millisecond),
|
||||
For: promutil.NewDuration(10 * time.Millisecond),
|
||||
},
|
||||
},
|
||||
}, true, "bad prometheus expr")
|
||||
|
@ -488,7 +488,7 @@ func TestHashRule_Equal(t *testing.T) {
|
|||
f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Alert: "record", Expr: "up == 1"})
|
||||
|
||||
f(Rule{
|
||||
Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute),
|
||||
Alert: "alert", Expr: "up == 1", For: promutil.NewDuration(time.Minute), KeepFiringFor: promutil.NewDuration(time.Minute),
|
||||
}, Rule{Alert: "alert", Expr: "up == 1"})
|
||||
}
|
||||
|
||||
|
|
|
@ -9,7 +9,7 @@ groups:
|
|||
annotations:
|
||||
summary: "{{ }}"
|
||||
description: "{{$labels}}"
|
||||
- alert: UnkownAnnotationsFunction
|
||||
- alert: UnknownAnnotationsFunction
|
||||
for: 5m
|
||||
expr: vm_rows > 0
|
||||
labels:
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
groups:
|
||||
- name: group
|
||||
rules:
|
||||
- alert: UnkownLabelFunction
|
||||
- alert: UnknownLabelFunction
|
||||
for: 5m
|
||||
expr: vm_rows > 0
|
||||
labels:
|
||||
|
|
|
@ -35,6 +35,8 @@ type promResponse struct {
|
|||
Stats struct {
|
||||
SeriesFetched *string `json:"seriesFetched,omitempty"`
|
||||
} `json:"stats,omitempty"`
|
||||
// IsPartial supported by VictoriaMetrics
|
||||
IsPartial *bool `json:"isPartial,omitempty"`
|
||||
}
|
||||
|
||||
// see https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
|
||||
|
@ -209,7 +211,7 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) (res Result
|
|||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res = Result{Data: ms}
|
||||
res = Result{Data: ms, IsPartial: r.IsPartial}
|
||||
if r.Stats.SeriesFetched != nil {
|
||||
intV, err := strconv.Atoi(*r.Stats.SeriesFetched)
|
||||
if err != nil {
|
||||
|
|
|
@ -12,7 +12,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
)
|
||||
|
@ -72,12 +72,14 @@ func TestVMInstantQuery(t *testing.T) {
|
|||
w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
|
||||
case 7:
|
||||
w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]},"stats":{"seriesFetched": "42"}}`))
|
||||
case 8:
|
||||
w.Write([]byte(`{"status":"success", "isPartial":true, "data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/render", func(w http.ResponseWriter, _ *http.Request) {
|
||||
c++
|
||||
switch c {
|
||||
case 8:
|
||||
case 9:
|
||||
w.Write([]byte(`[{"target":"constantLine(10)","tags":{"name":"constantLine(10)"},"datapoints":[[10,1611758343],[10,1611758373],[10,1611758403]]}]`))
|
||||
}
|
||||
})
|
||||
|
@ -100,9 +102,9 @@ func TestVMInstantQuery(t *testing.T) {
|
|||
t.Fatalf("failed to parse 'time' query param %q: %s", timeParam, err)
|
||||
}
|
||||
switch c {
|
||||
case 9:
|
||||
w.Write([]byte("[]"))
|
||||
case 10:
|
||||
w.Write([]byte("[]"))
|
||||
case 11:
|
||||
w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"total","foo":"bar"},"value":[1583786142,"13763"]},{"metric":{"__name__":"total","foo":"baz"},"value":[1583786140,"2000"]}]}}`))
|
||||
}
|
||||
})
|
||||
|
@ -203,10 +205,18 @@ func TestVMInstantQuery(t *testing.T) {
|
|||
*res.SeriesFetched)
|
||||
}
|
||||
|
||||
res, _, err = pq.Query(ctx, vmQuery, ts) // 8
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected %s", err)
|
||||
}
|
||||
if res.IsPartial != nil && !*res.IsPartial {
|
||||
t.Fatalf("unexpected metric isPartial want %+v", true)
|
||||
}
|
||||
|
||||
// test graphite
|
||||
gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
|
||||
|
||||
res, _, err = gq.Query(ctx, queryRender, ts) // 8 - graphite
|
||||
res, _, err = gq.Query(ctx, queryRender, ts) // 9 - graphite
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected %s", err)
|
||||
}
|
||||
|
@ -226,9 +236,9 @@ func TestVMInstantQuery(t *testing.T) {
|
|||
vlogs := datasourceVLogs
|
||||
pq = s.BuildWithParams(QuerierParams{DataSourceType: string(vlogs), EvaluationInterval: 15 * time.Second})
|
||||
|
||||
expErr(vlogsQuery, "error parsing response") // 9
|
||||
expErr(vlogsQuery, "error parsing response") // 10
|
||||
|
||||
res, _, err = pq.Query(ctx, vlogsQuery, ts) // 10
|
||||
res, _, err = pq.Query(ctx, vlogsQuery, ts) // 11
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected %s", err)
|
||||
}
|
||||
|
@ -753,7 +763,7 @@ func TestHeaders(t *testing.T) {
|
|||
|
||||
// basic auth
|
||||
f(func() *Client {
|
||||
cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
|
||||
cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "bar", ""))
|
||||
if err != nil {
|
||||
t.Fatalf("Error get auth config: %s", err)
|
||||
}
|
||||
|
@ -766,7 +776,7 @@ func TestHeaders(t *testing.T) {
|
|||
|
||||
// bearer auth
|
||||
f(func() *Client {
|
||||
cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
|
||||
cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBearer("foo", ""))
|
||||
if err != nil {
|
||||
t.Fatalf("Error get auth config: %s", err)
|
||||
}
|
||||
|
@ -798,7 +808,7 @@ func TestHeaders(t *testing.T) {
|
|||
|
||||
// custom header overrides basic auth
|
||||
f(func() *Client {
|
||||
cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
|
||||
cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "bar", ""))
|
||||
if err != nil {
|
||||
t.Fatalf("Error get auth config: %s", err)
|
||||
}
|
||||
|
|
|
@ -34,6 +34,9 @@ type Result struct {
|
|||
// If nil, then this feature is not supported by the datasource.
|
||||
// SeriesFetched is supported by VictoriaMetrics since v1.90.
|
||||
SeriesFetched *int
|
||||
// IsPartial is used by VictoriaMetrics to indicate
|
||||
// whether response data is partial.
|
||||
IsPartial *bool
|
||||
}
|
||||
|
||||
// QuerierBuilder builds Querier with given params.
|
||||
|
|
|
@ -10,8 +10,9 @@ import (
|
|||
// FakeQuerier is a mock querier that return predefined results and error message
|
||||
type FakeQuerier struct {
|
||||
sync.Mutex
|
||||
metrics []Metric
|
||||
err error
|
||||
metrics []Metric
|
||||
err error
|
||||
isPartial *bool
|
||||
}
|
||||
|
||||
// SetErr sets query error message
|
||||
|
@ -21,11 +22,19 @@ func (fq *FakeQuerier) SetErr(err error) {
|
|||
fq.Unlock()
|
||||
}
|
||||
|
||||
// SetPartialResponse marks query response as partial
|
||||
func (fq *FakeQuerier) SetPartialResponse(partial bool) {
|
||||
fq.Lock()
|
||||
fq.isPartial = &partial
|
||||
fq.Unlock()
|
||||
}
|
||||
|
||||
// Reset reset querier's error message and results
|
||||
func (fq *FakeQuerier) Reset() {
|
||||
fq.Lock()
|
||||
fq.err = nil
|
||||
fq.metrics = fq.metrics[:0]
|
||||
fq.isPartial = nil
|
||||
fq.Unlock()
|
||||
}
|
||||
|
||||
|
@ -57,7 +66,7 @@ func (fq *FakeQuerier) Query(_ context.Context, _ string, _ time.Time) (Result,
|
|||
cp := make([]Metric, len(fq.metrics))
|
||||
copy(cp, fq.metrics)
|
||||
req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
|
||||
return Result{Data: cp}, req, nil
|
||||
return Result{Data: cp, IsPartial: fq.isPartial}, req, nil
|
||||
}
|
||||
|
||||
// FakeQuerierWithRegistry can store different results for different query expr
|
||||
|
|
|
@ -8,10 +8,10 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -79,15 +79,13 @@ type Param struct {
|
|||
// Provided extraParams will be added as GET params for
|
||||
// each request.
|
||||
func Init(extraParams url.Values) (QuerierBuilder, error) {
|
||||
if *addr == "" {
|
||||
return nil, fmt.Errorf("datasource.url is empty")
|
||||
if err := httputil.CheckURL(*addr); err != nil {
|
||||
return nil, fmt.Errorf("invalid -datasource.url: %w", err)
|
||||
}
|
||||
|
||||
tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
|
||||
tr, err := promauth.NewTLSTransport(*tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify, "vmalert_datasource")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
|
||||
}
|
||||
tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
|
||||
tr.DisableKeepAlives = *disableKeepAlive
|
||||
tr.MaxIdleConnsPerHost = *maxIdleConnections
|
||||
if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
|
||||
|
@ -106,11 +104,11 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse JSON for -datasource.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
|
||||
}
|
||||
authCfg, err := utils.AuthConfig(
|
||||
utils.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
|
||||
utils.WithBearer(*bearerToken, *bearerTokenFile),
|
||||
utils.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
|
||||
utils.WithHeaders(*headers))
|
||||
authCfg, err := vmalertutil.AuthConfig(
|
||||
vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
|
||||
vmalertutil.WithBearer(*bearerToken, *bearerTokenFile),
|
||||
vmalertutil.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
|
||||
vmalertutil.WithHeaders(*headers))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to configure auth: %w", err)
|
||||
}
|
||||
|
|
|
@ -44,7 +44,7 @@ Enterprise version of vmalert supports S3 and GCS paths to rules.
|
|||
For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules
|
||||
S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches
|
||||
all files with prefix rule_ in folder dir.
|
||||
See https://docs.victoriametrics.com/vmalert/#reading-rules-from-object-storage
|
||||
See https://docs.victoriametrics.com/victoriametrics/vmalert/#reading-rules-from-object-storage
|
||||
`)
|
||||
|
||||
ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions `+
|
||||
|
@ -71,7 +71,7 @@ absolute path to all .tpl files in root.
|
|||
externalURL = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier. By default, hostname is used as address.")
|
||||
externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
|
||||
`for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+
|
||||
`Supports templating - see https://docs.victoriametrics.com/vmalert/#templating . `+
|
||||
`Supports templating - see https://docs.victoriametrics.com/victoriametrics/vmalert/#templating . `+
|
||||
`For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{.Expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. `+
|
||||
`Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. `+
|
||||
`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
|
||||
|
@ -318,7 +318,7 @@ func usage() {
|
|||
const s = `
|
||||
vmalert processes alerts and recording rules.
|
||||
|
||||
See the docs at https://docs.victoriametrics.com/vmalert/ .
|
||||
See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
|
||||
`
|
||||
flagutil.Usage(s)
|
||||
}
|
||||
|
|
|
@ -169,7 +169,6 @@ groups:
|
|||
checkCfg(nil)
|
||||
groupsLen = lenLocked(m)
|
||||
if groupsLen != 2 {
|
||||
fmt.Println(m.groups)
|
||||
t.Fatalf("expected to have exactly 2 groups loaded; got %d", groupsLen)
|
||||
}
|
||||
|
||||
|
|
|
@ -83,7 +83,8 @@ func (m *manager) close() {
|
|||
|
||||
func (m *manager) startGroup(ctx context.Context, g *rule.Group, restore bool) error {
|
||||
m.wg.Add(1)
|
||||
id := g.ID()
|
||||
id := g.GetID()
|
||||
g.Init()
|
||||
go func() {
|
||||
defer m.wg.Done()
|
||||
if restore {
|
||||
|
@ -112,7 +113,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
|
|||
}
|
||||
}
|
||||
ng := rule.NewGroup(cfg, m.querierBuilder, *evaluationInterval, m.labels)
|
||||
groupsRegistry[ng.ID()] = ng
|
||||
groupsRegistry[ng.GetID()] = ng
|
||||
}
|
||||
|
||||
if rrPresent && m.rw == nil {
|
||||
|
@ -130,17 +131,17 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
|
|||
|
||||
m.groupsMu.Lock()
|
||||
for _, og := range m.groups {
|
||||
ng, ok := groupsRegistry[og.ID()]
|
||||
ng, ok := groupsRegistry[og.GetID()]
|
||||
if !ok {
|
||||
// old group is not present in new list,
|
||||
// so must be stopped and deleted
|
||||
og.Close()
|
||||
delete(m.groups, og.ID())
|
||||
delete(m.groups, og.GetID())
|
||||
og = nil
|
||||
continue
|
||||
}
|
||||
delete(groupsRegistry, ng.ID())
|
||||
if og.Checksum != ng.Checksum {
|
||||
delete(groupsRegistry, ng.GetID())
|
||||
if og.GetCheckSum() != ng.GetCheckSum() {
|
||||
toUpdate = append(toUpdate, updateItem{old: og, new: ng})
|
||||
}
|
||||
}
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue