From 8b36529b32939c6095b8e44cf9602996a15d080a Mon Sep 17 00:00:00 2001 From: f41gh7 Date: Tue, 3 Sep 2024 19:16:10 +0200 Subject: [PATCH] follow-up after 1731c0eabfe0c69fb0fa9a3dec8b39d9546db30c * updates change log * adds VL-Debug http header * updates doc * extracts only the first value of http headers for VL-Stream-Fields and VL-Ignore-Fields. It makes behaviour the same as Query string args. And allows to easily configure client applications. Since most of the client collectors don't support multi value headers. Signed-off-by: f41gh7 --- app/vlinsert/insertutils/common_params.go | 31 +++++++++++++++------- docs/VictoriaLogs/CHANGELOG.md | 14 +++++----- docs/VictoriaLogs/data-ingestion/README.md | 21 ++++++++++++--- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/app/vlinsert/insertutils/common_params.go b/app/vlinsert/insertutils/common_params.go index 91c4dbcf9..12a56ee99 100644 --- a/app/vlinsert/insertutils/common_params.go +++ b/app/vlinsert/insertutils/common_params.go @@ -2,6 +2,7 @@ package insertutils import ( "net/http" + "strings" "sync" "time" @@ -39,7 +40,7 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) { } // Extract time field name from _time_field query arg or header - var timeField = "_time" + timeField := "_time" if tf := r.FormValue("_time_field"); tf != "" { timeField = tf } else if tf = r.Header.Get("VL-Time-Field"); tf != "" { @@ -47,7 +48,7 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) { } // Extract message field name from _msg_field query arg or header - var msgField = "" + msgField := "" if msgf := r.FormValue("_msg_field"); msgf != "" { msgField = msgf } else if msgf = r.Header.Get("VL-Msg-Field"); msgf != "" { @@ -56,18 +57,28 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) { streamFields := httputils.GetArray(r, "_stream_fields") if len(streamFields) == 0 { - if sf := r.Header.Values("VL-Stream-Fields"); len(sf) > 0 { - streamFields = sf + if sf := r.Header.Get("VL-Stream-Fields"); len(sf) > 0 { + streamFields = strings.Split(sf, ",") } } ignoreFields := httputils.GetArray(r, "ignore_fields") if len(ignoreFields) == 0 { - if f := r.Header.Values("VL-Ignore-Fields"); len(f) > 0 { - ignoreFields = f + if f := r.Header.Get("VL-Ignore-Fields"); len(f) > 0 { + ignoreFields = strings.Split(f, ",") } } debug := httputils.GetBool(r, "debug") + if !debug { + if dh := r.Header.Get("VL-Debug"); len(dh) > 0 { + hv := strings.ToLower(dh) + switch hv { + case "", "0", "f", "false", "no": + default: + debug = true + } + } + } debugRequestURI := "" debugRemoteAddr := "" if debug { @@ -169,7 +180,7 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Fiel if lmp.cp.Debug { s := lmp.lr.GetRowString(0) lmp.lr.ResetKeepSettings() - logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s) + logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s) rowsDroppedTotalDebug.Inc() return } @@ -211,5 +222,7 @@ func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor { return lmp } -var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`) -var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`) +var ( + rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`) + rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`) +) diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index bdd02ecf2..f1d6a77d7 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -24,10 +24,11 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): move the Markdown toggle to the general settings panel in the upper left corner. * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add search functionality to the column display settings in the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668). * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add the ability to select all columns in the column display settings of the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6680). +* FEATURE: Allow to define ingestion parameters via headers. Supported headers - `VL-Msg-Field`,`VL-Stream-Fields`,`VL-Ignore-Fields`,`VL-Time-Field`, `VL-Debug`. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6443) for details. * BUGFIX: properly handle Logstash requests for Elasticsearch configuration when using `outputs.elasticsearch` in Logstash pipelines. Previously, the requests could be rejected with `400 Bad Request` response. Updates [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4750). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix `not found index.js` error when loading vmui in VictoriaLogs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6764). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6770). -* BUGFIX: fix filtering when logical operators `AND` and `OR` are used in query expression. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6556). +* BUGFIX: fix filtering when logical operators `AND` and `OR` are used in query expression. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6556). ## [v0.28.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.28.0-victorialogs) @@ -242,10 +243,10 @@ Released at 2024-05-24 * FEATURE: return the number of matching log entries per returned value in [HTTP API](https://docs.victoriametrics.com/victorialogs/querying/#http-api) results. This simplifies detecting [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) / [stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) values with the biggest number of logs for the given [LogsQL query](https://docs.victoriametrics.com/victorialogs/logsql/). * FEATURE: improve performance for [regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter) in the following cases: - - If the regexp contains just a phrase without special regular expression chars. For example, `~"foo"`. - - If the regexp starts with `.*` or ends with `.*`. For example, `~".*foo.*"`. - - If the regexp contains multiple strings delimited by `|`. For example, `~"foo|bar|baz"`. - - If the regexp contains multiple [words](https://docs.victoriametrics.com/victorialogs/logsql/#word). For example, `~"foo bar baz"`. + * If the regexp contains just a phrase without special regular expression chars. For example, `~"foo"`. + * If the regexp starts with `.*` or ends with `.*`. For example, `~".*foo.*"`. + * If the regexp contains multiple strings delimited by `|`. For example, `~"foo|bar|baz"`. + * If the regexp contains multiple [words](https://docs.victoriametrics.com/victorialogs/logsql/#word). For example, `~"foo bar baz"`. * FEATURE: allow disabling automatic unquoting of the matched placeholders in [`extract` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#extract-pipe). See [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#format-for-extract-pipe-pattern). * BUGFIX: properly parse `!` in front of [exact filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-filter), [exact-prefix filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-prefix-filter) and [regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter). For example, `!~"some regexp"` is properly parsed as `not ="some regexp"`. Previously it was incorrectly parsed as `'~="some regexp"'` [phrase filter](https://docs.victoriametrics.com/victorialogs/logsql/#phrase-filter). @@ -305,7 +306,6 @@ Released at 2024-05-15 * FEATURE: add ability to return the first `N` results from [`sort` pipe](#https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe). This is useful when `N` biggest or `N` smallest values must be returned from large amounts of logs. * FEATURE: add [`quantile`](https://docs.victoriametrics.com/victorialogs/logsql/#quantile-stats) and [`median`](https://docs.victoriametrics.com/victorialogs/logsql/#median-stats) [stats functions](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe). - ## [v0.6.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.1-victorialogs) Released at 2024-05-14 @@ -315,7 +315,6 @@ Released at 2024-05-14 * BUGFIX: properly return matching logs in [streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) with small number of entries. Previously they could be skipped. The issue has been introduced in [the release v0.6.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.0-victorialogs). * BUGFIX: fix `runtime error: index out of range` panic when using [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe) like `_time:1h | sort by (_time)`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6258). - ## [v0.6.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.0-victorialogs) Released at 2024-05-12 @@ -343,7 +342,6 @@ Released at 2024-04-11 * BUGFIX: properly register new [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) under high data ingestion rate. The issue has been introduced in [v0.5.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.5.0-victorialogs). - ## [v0.5.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.5.1-victorialogs) Released at 2024-04-04 diff --git a/docs/VictoriaLogs/data-ingestion/README.md b/docs/VictoriaLogs/data-ingestion/README.md index 08e0cf24c..b2f443416 100644 --- a/docs/VictoriaLogs/data-ingestion/README.md +++ b/docs/VictoriaLogs/data-ingestion/README.md @@ -15,7 +15,6 @@ See also: - [Log collectors and data ingestion formats](#log-collectors-and-data-ingestion-formats). - [Data ingestion troubleshooting](#troubleshooting). - ## HTTP APIs VictoriaLogs supports the following data ingestion HTTP APIs: @@ -169,7 +168,12 @@ See also: ### HTTP parameters -VictoriaLogs accepts the following parameters at [data ingestion HTTP APIs](#http-apis): + VictoriaLogs accepts the following configuration parameters via [HTTP Headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields) or URL [Query string](https://en.wikipedia.org/wiki/Query_string) at [data ingestion HTTP APIs](#http-apis). +First defined parameter is used. [Query string](https://en.wikipedia.org/wiki/Query_string) parameters have priority over HTTP Headers. + +#### HTTP Query string parameters + + List of supported [Query string](https://en.wikipedia.org/wiki/Query_string) parameters: - `_msg_field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) with the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) generated by the log shipper. @@ -194,10 +198,14 @@ VictoriaLogs accepts the following parameters at [data ingestion HTTP APIs](#htt See also [HTTP headers](#http-headers). -### HTTP headers +#### HTTP headers + List of supported [HTTP Headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields) parameters: + +- `AccountID` - may contain the needed accountID of tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details. + +- `ProjectID`- may contain the projectID needed of tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details. VictoriaLogs accepts optional `AccountID` and `ProjectID` headers at [data ingestion HTTP APIs](#http-apis). -These headers may contain the needed tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details. - `VL-Msg-Field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) with the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) generated by the log shipper. @@ -217,6 +225,11 @@ These headers may contain the needed tenant to ingest data to. See [multitenancy - `VL-Ignore-Fields` - this parameter may contain the list of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names, which must be ignored during data ingestion. +- `VL-Debug` - if this parameter is set to `1`, then the ingested logs aren't stored in VictoriaLogs. Instead, + the ingested data is logged by VictoriaLogs, so it can be investigated later. + +See also [HTTP Query string parameters](#http-query-string-parameters). + ## Troubleshooting The following command can be used for verifying whether the data is successfully ingested into VictoriaLogs: