From dcc525b388c622faba0edd22e5033a0328221402 Mon Sep 17 00:00:00 2001
From: f41gh7 <nik@victoriametrics.com>
Date: Tue, 3 Sep 2024 19:16:10 +0200
Subject: [PATCH] follow-up after 1731c0eabfe0c69fb0fa9a3dec8b39d9546db30c
* updates change log
* adds VL-Debug http header
* updates doc
* extracts only the first value of http headers for VL-Stream-Fields and VL-Ignore-Fields.
It makes behaviour the same as Query string args. And allows to easily configure client applications.
Since most of the client collectors don't support multi value headers.
Signed-off-by: f41gh7 <nik@victoriametrics.com>
---
app/vlinsert/insertutils/common_params.go | 31 +++++++++++++++-------
docs/VictoriaLogs/CHANGELOG.md | 14 +++++-----
docs/VictoriaLogs/data-ingestion/README.md | 21 ++++++++++++---
3 files changed, 45 insertions(+), 21 deletions(-)
diff --git a/app/vlinsert/insertutils/common_params.go b/app/vlinsert/insertutils/common_params.go
index 91c4dbcf9a..12a56ee99d 100644
--- a/app/vlinsert/insertutils/common_params.go
+++ b/app/vlinsert/insertutils/common_params.go
@@ -2,6 +2,7 @@ package insertutils
import (
"net/http"
+ "strings"
"sync"
"time"
@@ -39,7 +40,7 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
}
// Extract time field name from _time_field query arg or header
- var timeField = "_time"
+ timeField := "_time"
if tf := r.FormValue("_time_field"); tf != "" {
timeField = tf
} else if tf = r.Header.Get("VL-Time-Field"); tf != "" {
@@ -47,7 +48,7 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
}
// Extract message field name from _msg_field query arg or header
- var msgField = ""
+ msgField := ""
if msgf := r.FormValue("_msg_field"); msgf != "" {
msgField = msgf
} else if msgf = r.Header.Get("VL-Msg-Field"); msgf != "" {
@@ -56,18 +57,28 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
streamFields := httputils.GetArray(r, "_stream_fields")
if len(streamFields) == 0 {
- if sf := r.Header.Values("VL-Stream-Fields"); len(sf) > 0 {
- streamFields = sf
+ if sf := r.Header.Get("VL-Stream-Fields"); len(sf) > 0 {
+ streamFields = strings.Split(sf, ",")
}
}
ignoreFields := httputils.GetArray(r, "ignore_fields")
if len(ignoreFields) == 0 {
- if f := r.Header.Values("VL-Ignore-Fields"); len(f) > 0 {
- ignoreFields = f
+ if f := r.Header.Get("VL-Ignore-Fields"); len(f) > 0 {
+ ignoreFields = strings.Split(f, ",")
}
}
debug := httputils.GetBool(r, "debug")
+ if !debug {
+ if dh := r.Header.Get("VL-Debug"); len(dh) > 0 {
+ hv := strings.ToLower(dh)
+ switch hv {
+ case "", "0", "f", "false", "no":
+ default:
+ debug = true
+ }
+ }
+ }
debugRequestURI := ""
debugRemoteAddr := ""
if debug {
@@ -169,7 +180,7 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Fiel
if lmp.cp.Debug {
s := lmp.lr.GetRowString(0)
lmp.lr.ResetKeepSettings()
- logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
+ logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
rowsDroppedTotalDebug.Inc()
return
}
@@ -211,5 +222,7 @@ func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
return lmp
}
-var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
+var (
+ rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
+ rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
+)
diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md
index bdd02ecf22..f1d6a77d7c 100644
--- a/docs/VictoriaLogs/CHANGELOG.md
+++ b/docs/VictoriaLogs/CHANGELOG.md
@@ -24,10 +24,11 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): move the Markdown toggle to the general settings panel in the upper left corner.
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add search functionality to the column display settings in the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668).
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add the ability to select all columns in the column display settings of the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6680).
+* FEATURE: Allow to define ingestion parameters via headers. Supported headers - `VL-Msg-Field`,`VL-Stream-Fields`,`VL-Ignore-Fields`,`VL-Time-Field`, `VL-Debug`. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6443) for details.
* BUGFIX: properly handle Logstash requests for Elasticsearch configuration when using `outputs.elasticsearch` in Logstash pipelines. Previously, the requests could be rejected with `400 Bad Request` response. Updates [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4750).
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix `not found index.js` error when loading vmui in VictoriaLogs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6764). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6770).
-* BUGFIX: fix filtering when logical operators `AND` and `OR` are used in query expression. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6556).
+* BUGFIX: fix filtering when logical operators `AND` and `OR` are used in query expression. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6556).
## [v0.28.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.28.0-victorialogs)
@@ -242,10 +243,10 @@ Released at 2024-05-24
* FEATURE: return the number of matching log entries per returned value in [HTTP API](https://docs.victoriametrics.com/victorialogs/querying/#http-api) results. This simplifies detecting [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) / [stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) values with the biggest number of logs for the given [LogsQL query](https://docs.victoriametrics.com/victorialogs/logsql/).
* FEATURE: improve performance for [regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter) in the following cases:
- - If the regexp contains just a phrase without special regular expression chars. For example, `~"foo"`.
- - If the regexp starts with `.*` or ends with `.*`. For example, `~".*foo.*"`.
- - If the regexp contains multiple strings delimited by `|`. For example, `~"foo|bar|baz"`.
- - If the regexp contains multiple [words](https://docs.victoriametrics.com/victorialogs/logsql/#word). For example, `~"foo bar baz"`.
+ * If the regexp contains just a phrase without special regular expression chars. For example, `~"foo"`.
+ * If the regexp starts with `.*` or ends with `.*`. For example, `~".*foo.*"`.
+ * If the regexp contains multiple strings delimited by `|`. For example, `~"foo|bar|baz"`.
+ * If the regexp contains multiple [words](https://docs.victoriametrics.com/victorialogs/logsql/#word). For example, `~"foo bar baz"`.
* FEATURE: allow disabling automatic unquoting of the matched placeholders in [`extract` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#extract-pipe). See [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#format-for-extract-pipe-pattern).
* BUGFIX: properly parse `!` in front of [exact filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-filter), [exact-prefix filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-prefix-filter) and [regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter). For example, `!~"some regexp"` is properly parsed as `not ="some regexp"`. Previously it was incorrectly parsed as `'~="some regexp"'` [phrase filter](https://docs.victoriametrics.com/victorialogs/logsql/#phrase-filter).
@@ -305,7 +306,6 @@ Released at 2024-05-15
* FEATURE: add ability to return the first `N` results from [`sort` pipe](#https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe). This is useful when `N` biggest or `N` smallest values must be returned from large amounts of logs.
* FEATURE: add [`quantile`](https://docs.victoriametrics.com/victorialogs/logsql/#quantile-stats) and [`median`](https://docs.victoriametrics.com/victorialogs/logsql/#median-stats) [stats functions](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe).
-
## [v0.6.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.1-victorialogs)
Released at 2024-05-14
@@ -315,7 +315,6 @@ Released at 2024-05-14
* BUGFIX: properly return matching logs in [streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) with small number of entries. Previously they could be skipped. The issue has been introduced in [the release v0.6.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.0-victorialogs).
* BUGFIX: fix `runtime error: index out of range` panic when using [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe) like `_time:1h | sort by (_time)`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6258).
-
## [v0.6.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.0-victorialogs)
Released at 2024-05-12
@@ -343,7 +342,6 @@ Released at 2024-04-11
* BUGFIX: properly register new [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) under high data ingestion rate. The issue has been introduced in [v0.5.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.5.0-victorialogs).
-
## [v0.5.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.5.1-victorialogs)
Released at 2024-04-04
diff --git a/docs/VictoriaLogs/data-ingestion/README.md b/docs/VictoriaLogs/data-ingestion/README.md
index 08e0cf24ce..b2f4434163 100644
--- a/docs/VictoriaLogs/data-ingestion/README.md
+++ b/docs/VictoriaLogs/data-ingestion/README.md
@@ -15,7 +15,6 @@ See also:
- [Log collectors and data ingestion formats](#log-collectors-and-data-ingestion-formats).
- [Data ingestion troubleshooting](#troubleshooting).
-
## HTTP APIs
VictoriaLogs supports the following data ingestion HTTP APIs:
@@ -169,7 +168,12 @@ See also:
### HTTP parameters
-VictoriaLogs accepts the following parameters at [data ingestion HTTP APIs](#http-apis):
+ VictoriaLogs accepts the following configuration parameters via [HTTP Headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields) or URL [Query string](https://en.wikipedia.org/wiki/Query_string) at [data ingestion HTTP APIs](#http-apis).
+First defined parameter is used. [Query string](https://en.wikipedia.org/wiki/Query_string) parameters have priority over HTTP Headers.
+
+#### HTTP Query string parameters
+
+ List of supported [Query string](https://en.wikipedia.org/wiki/Query_string) parameters:
- `_msg_field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
with the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) generated by the log shipper.
@@ -194,10 +198,14 @@ VictoriaLogs accepts the following parameters at [data ingestion HTTP APIs](#htt
See also [HTTP headers](#http-headers).
-### HTTP headers
+#### HTTP headers
+ List of supported [HTTP Headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields) parameters:
+
+- `AccountID` - may contain the needed accountID of tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details.
+
+- `ProjectID`- may contain the projectID needed of tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details.
VictoriaLogs accepts optional `AccountID` and `ProjectID` headers at [data ingestion HTTP APIs](#http-apis).
-These headers may contain the needed tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details.
- `VL-Msg-Field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
with the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) generated by the log shipper.
@@ -217,6 +225,11 @@ These headers may contain the needed tenant to ingest data to. See [multitenancy
- `VL-Ignore-Fields` - this parameter may contain the list of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names,
which must be ignored during data ingestion.
+- `VL-Debug` - if this parameter is set to `1`, then the ingested logs aren't stored in VictoriaLogs. Instead,
+ the ingested data is logged by VictoriaLogs, so it can be investigated later.
+
+See also [HTTP Query string parameters](#http-query-string-parameters).
+
## Troubleshooting
The following command can be used for verifying whether the data is successfully ingested into VictoriaLogs: