mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
follow-up after 1731c0eabf
* updates change log * adds VL-Debug http header * updates doc * extracts only the first value of http headers for VL-Stream-Fields and VL-Ignore-Fields. It makes behaviour the same as Query string args. And allows to easily configure client applications. Since most of the client collectors don't support multi value headers. Signed-off-by: f41gh7 <nik@victoriametrics.com>
This commit is contained in:
f41gh7
parent
1731c0eabf
commit
8b36529b32
3 changed files with 45 additions and 21 deletions
|
|
@ -2,6 +2,7 @@ package insertutils
|
|||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
|
@ -39,7 +40,7 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
|||
}
|
||||
|
||||
// Extract time field name from _time_field query arg or header
|
||||
var timeField = "_time"
|
||||
timeField := "_time"
|
||||
if tf := r.FormValue("_time_field"); tf != "" {
|
||||
timeField = tf
|
||||
} else if tf = r.Header.Get("VL-Time-Field"); tf != "" {
|
||||
|
|
@ -47,7 +48,7 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
|||
}
|
||||
|
||||
// Extract message field name from _msg_field query arg or header
|
||||
var msgField = ""
|
||||
msgField := ""
|
||||
if msgf := r.FormValue("_msg_field"); msgf != "" {
|
||||
msgField = msgf
|
||||
} else if msgf = r.Header.Get("VL-Msg-Field"); msgf != "" {
|
||||
|
|
@ -56,18 +57,28 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
|||
|
||||
streamFields := httputils.GetArray(r, "_stream_fields")
|
||||
if len(streamFields) == 0 {
|
||||
if sf := r.Header.Values("VL-Stream-Fields"); len(sf) > 0 {
|
||||
streamFields = sf
|
||||
if sf := r.Header.Get("VL-Stream-Fields"); len(sf) > 0 {
|
||||
streamFields = strings.Split(sf, ",")
|
||||
}
|
||||
}
|
||||
ignoreFields := httputils.GetArray(r, "ignore_fields")
|
||||
if len(ignoreFields) == 0 {
|
||||
if f := r.Header.Values("VL-Ignore-Fields"); len(f) > 0 {
|
||||
ignoreFields = f
|
||||
if f := r.Header.Get("VL-Ignore-Fields"); len(f) > 0 {
|
||||
ignoreFields = strings.Split(f, ",")
|
||||
}
|
||||
}
|
||||
|
||||
debug := httputils.GetBool(r, "debug")
|
||||
if !debug {
|
||||
if dh := r.Header.Get("VL-Debug"); len(dh) > 0 {
|
||||
hv := strings.ToLower(dh)
|
||||
switch hv {
|
||||
case "", "0", "f", "false", "no":
|
||||
default:
|
||||
debug = true
|
||||
}
|
||||
}
|
||||
}
|
||||
debugRequestURI := ""
|
||||
debugRemoteAddr := ""
|
||||
if debug {
|
||||
|
|
@ -169,7 +180,7 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Fiel
|
|||
if lmp.cp.Debug {
|
||||
s := lmp.lr.GetRowString(0)
|
||||
lmp.lr.ResetKeepSettings()
|
||||
logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
|
||||
logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
|
||||
rowsDroppedTotalDebug.Inc()
|
||||
return
|
||||
}
|
||||
|
|
@ -211,5 +222,7 @@ func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
|
|||
return lmp
|
||||
}
|
||||
|
||||
var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
|
||||
var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
|
||||
var (
|
||||
rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
|
||||
rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -24,10 +24,11 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
|
|||
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): move the Markdown toggle to the general settings panel in the upper left corner.
|
||||
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add search functionality to the column display settings in the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668).
|
||||
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add the ability to select all columns in the column display settings of the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6680).
|
||||
* FEATURE: Allow to define ingestion parameters via headers. Supported headers - `VL-Msg-Field`,`VL-Stream-Fields`,`VL-Ignore-Fields`,`VL-Time-Field`, `VL-Debug`. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6443) for details.
|
||||
|
||||
* BUGFIX: properly handle Logstash requests for Elasticsearch configuration when using `outputs.elasticsearch` in Logstash pipelines. Previously, the requests could be rejected with `400 Bad Request` response. Updates [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4750).
|
||||
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix `not found index.js` error when loading vmui in VictoriaLogs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6764). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6770).
|
||||
* BUGFIX: fix filtering when logical operators `AND` and `OR` are used in query expression. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6556).
|
||||
* BUGFIX: fix filtering when logical operators `AND` and `OR` are used in query expression. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6556).
|
||||
|
||||
## [v0.28.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.28.0-victorialogs)
|
||||
|
||||
|
|
@ -242,10 +243,10 @@ Released at 2024-05-24
|
|||
|
||||
* FEATURE: return the number of matching log entries per returned value in [HTTP API](https://docs.victoriametrics.com/victorialogs/querying/#http-api) results. This simplifies detecting [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) / [stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) values with the biggest number of logs for the given [LogsQL query](https://docs.victoriametrics.com/victorialogs/logsql/).
|
||||
* FEATURE: improve performance for [regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter) in the following cases:
|
||||
- If the regexp contains just a phrase without special regular expression chars. For example, `~"foo"`.
|
||||
- If the regexp starts with `.*` or ends with `.*`. For example, `~".*foo.*"`.
|
||||
- If the regexp contains multiple strings delimited by `|`. For example, `~"foo|bar|baz"`.
|
||||
- If the regexp contains multiple [words](https://docs.victoriametrics.com/victorialogs/logsql/#word). For example, `~"foo bar baz"`.
|
||||
* If the regexp contains just a phrase without special regular expression chars. For example, `~"foo"`.
|
||||
* If the regexp starts with `.*` or ends with `.*`. For example, `~".*foo.*"`.
|
||||
* If the regexp contains multiple strings delimited by `|`. For example, `~"foo|bar|baz"`.
|
||||
* If the regexp contains multiple [words](https://docs.victoriametrics.com/victorialogs/logsql/#word). For example, `~"foo bar baz"`.
|
||||
* FEATURE: allow disabling automatic unquoting of the matched placeholders in [`extract` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#extract-pipe). See [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#format-for-extract-pipe-pattern).
|
||||
|
||||
* BUGFIX: properly parse `!` in front of [exact filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-filter), [exact-prefix filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-prefix-filter) and [regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter). For example, `!~"some regexp"` is properly parsed as `not ="some regexp"`. Previously it was incorrectly parsed as `'~="some regexp"'` [phrase filter](https://docs.victoriametrics.com/victorialogs/logsql/#phrase-filter).
|
||||
|
|
@ -305,7 +306,6 @@ Released at 2024-05-15
|
|||
* FEATURE: add ability to return the first `N` results from [`sort` pipe](#https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe). This is useful when `N` biggest or `N` smallest values must be returned from large amounts of logs.
|
||||
* FEATURE: add [`quantile`](https://docs.victoriametrics.com/victorialogs/logsql/#quantile-stats) and [`median`](https://docs.victoriametrics.com/victorialogs/logsql/#median-stats) [stats functions](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe).
|
||||
|
||||
|
||||
## [v0.6.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.1-victorialogs)
|
||||
|
||||
Released at 2024-05-14
|
||||
|
|
@ -315,7 +315,6 @@ Released at 2024-05-14
|
|||
* BUGFIX: properly return matching logs in [streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) with small number of entries. Previously they could be skipped. The issue has been introduced in [the release v0.6.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.0-victorialogs).
|
||||
* BUGFIX: fix `runtime error: index out of range` panic when using [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe) like `_time:1h | sort by (_time)`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6258).
|
||||
|
||||
|
||||
## [v0.6.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.0-victorialogs)
|
||||
|
||||
Released at 2024-05-12
|
||||
|
|
@ -343,7 +342,6 @@ Released at 2024-04-11
|
|||
|
||||
* BUGFIX: properly register new [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) under high data ingestion rate. The issue has been introduced in [v0.5.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.5.0-victorialogs).
|
||||
|
||||
|
||||
## [v0.5.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.5.1-victorialogs)
|
||||
|
||||
Released at 2024-04-04
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ See also:
|
|||
- [Log collectors and data ingestion formats](#log-collectors-and-data-ingestion-formats).
|
||||
- [Data ingestion troubleshooting](#troubleshooting).
|
||||
|
||||
|
||||
## HTTP APIs
|
||||
|
||||
VictoriaLogs supports the following data ingestion HTTP APIs:
|
||||
|
|
@ -169,7 +168,12 @@ See also:
|
|||
|
||||
### HTTP parameters
|
||||
|
||||
VictoriaLogs accepts the following parameters at [data ingestion HTTP APIs](#http-apis):
|
||||
VictoriaLogs accepts the following configuration parameters via [HTTP Headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields) or URL [Query string](https://en.wikipedia.org/wiki/Query_string) at [data ingestion HTTP APIs](#http-apis).
|
||||
First defined parameter is used. [Query string](https://en.wikipedia.org/wiki/Query_string) parameters have priority over HTTP Headers.
|
||||
|
||||
#### HTTP Query string parameters
|
||||
|
||||
List of supported [Query string](https://en.wikipedia.org/wiki/Query_string) parameters:
|
||||
|
||||
- `_msg_field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||
with the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) generated by the log shipper.
|
||||
|
|
@ -194,10 +198,14 @@ VictoriaLogs accepts the following parameters at [data ingestion HTTP APIs](#htt
|
|||
|
||||
See also [HTTP headers](#http-headers).
|
||||
|
||||
### HTTP headers
|
||||
#### HTTP headers
|
||||
|
||||
List of supported [HTTP Headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields) parameters:
|
||||
|
||||
- `AccountID` - may contain the needed accountID of tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details.
|
||||
|
||||
- `ProjectID`- may contain the projectID needed of tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details.
|
||||
VictoriaLogs accepts optional `AccountID` and `ProjectID` headers at [data ingestion HTTP APIs](#http-apis).
|
||||
These headers may contain the needed tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details.
|
||||
|
||||
- `VL-Msg-Field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||
with the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) generated by the log shipper.
|
||||
|
|
@ -217,6 +225,11 @@ These headers may contain the needed tenant to ingest data to. See [multitenancy
|
|||
- `VL-Ignore-Fields` - this parameter may contain the list of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names,
|
||||
which must be ignored during data ingestion.
|
||||
|
||||
- `VL-Debug` - if this parameter is set to `1`, then the ingested logs aren't stored in VictoriaLogs. Instead,
|
||||
the ingested data is logged by VictoriaLogs, so it can be investigated later.
|
||||
|
||||
See also [HTTP Query string parameters](#http-query-string-parameters).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
The following command can be used for verifying whether the data is successfully ingested into VictoriaLogs:
|
||||
|
|
|
|||
Loading…
Reference in a new issue