mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-31 15:06:26 +00:00
wip
This commit is contained in:
parent
def99558c2
commit
a15a56def0
22 changed files with 189 additions and 189 deletions
|
@ -94,13 +94,13 @@ These flags can be inspected by running `vlogsgenerator -help`. Below are the mo
|
||||||
|
|
||||||
* `-start` - starting timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
|
* `-start` - starting timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
|
||||||
* `-end` - ending timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
|
* `-end` - ending timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
|
||||||
* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to generate.
|
* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to generate.
|
||||||
* `-logsPerStream` - the number of log entries to generate per each log stream. Log entries are evenly distributed on the [`-start` ... `-end`] interval.
|
* `-logsPerStream` - the number of log entries to generate per each log stream. Log entries are evenly distributed on the [`-start` ... `-end`] interval.
|
||||||
|
|
||||||
The total number of generated logs can be calculated as `-activeStreams` * `-logsPerStream`.
|
The total number of generated logs can be calculated as `-activeStreams` * `-logsPerStream`.
|
||||||
|
|
||||||
For example, the following command generates `1_000_000` log entries on the time range `[2024-01-01 - 2024-02-01]` across `100`
|
For example, the following command generates `1_000_000` log entries on the time range `[2024-01-01 - 2024-02-01]` across `100`
|
||||||
[log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields), where every logs stream contains `10_000` log entries,
|
[log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields), where every logs stream contains `10_000` log entries,
|
||||||
and writes them to `http://localhost:9428/insert/jsonline`:
|
and writes them to `http://localhost:9428/insert/jsonline`:
|
||||||
|
|
||||||
```
|
```
|
||||||
|
@ -113,7 +113,7 @@ bin/vlogsgenerator \
|
||||||
|
|
||||||
### Churn rate
|
### Churn rate
|
||||||
|
|
||||||
It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields)
|
It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
|
||||||
by specifying `-totalStreams` command-line flag bigger than `-activeStreams`. For example, the following command generates
|
by specifying `-totalStreams` command-line flag bigger than `-activeStreams`. For example, the following command generates
|
||||||
logs for `1000` total streams, while the number of active streams equals to `100`. This means that at every time there are logs for `100` streams,
|
logs for `1000` total streams, while the number of active streams equals to `100`. This means that at every time there are logs for `100` streams,
|
||||||
but these streams change over the given [`-start` ... `-end`] time range, so the total number of streams on the given time range becomes `1000`:
|
but these streams change over the given [`-start` ... `-end`] time range, so the total number of streams on the given time range becomes `1000`:
|
||||||
|
|
|
@ -27,32 +27,32 @@ var (
|
||||||
|
|
||||||
start = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/#timestamp-formats")
|
start = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/#timestamp-formats")
|
||||||
end = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/#timestamp-formats")
|
end = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/#timestamp-formats")
|
||||||
activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields")
|
activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields")
|
||||||
totalStreams = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
|
totalStreams = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
|
||||||
"during data generation")
|
"during data generation")
|
||||||
logsPerStream = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
|
logsPerStream = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
|
||||||
constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
|
constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
|
varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
dictFieldsPerLog = flag.Int("dictFieldsPerLog", 2, "The number of fields with up to 8 different values to generate per each log entry; "+
|
dictFieldsPerLog = flag.Int("dictFieldsPerLog", 2, "The number of fields with up to 8 different values to generate per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
u8FieldsPerLog = flag.Int("u8FieldsPerLog", 1, "The number of fields with uint8 values to generate per each log entry; "+
|
u8FieldsPerLog = flag.Int("u8FieldsPerLog", 1, "The number of fields with uint8 values to generate per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
u16FieldsPerLog = flag.Int("u16FieldsPerLog", 1, "The number of fields with uint16 values to generate per each log entry; "+
|
u16FieldsPerLog = flag.Int("u16FieldsPerLog", 1, "The number of fields with uint16 values to generate per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
u32FieldsPerLog = flag.Int("u32FieldsPerLog", 1, "The number of fields with uint32 values to generate per each log entry; "+
|
u32FieldsPerLog = flag.Int("u32FieldsPerLog", 1, "The number of fields with uint32 values to generate per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
|
u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
|
floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
|
ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
timestampFieldsPerLog = flag.Int("timestampFieldsPerLog", 1, "The number of fields with ISO8601 timestamps per each log entry; "+
|
timestampFieldsPerLog = flag.Int("timestampFieldsPerLog", 1, "The number of fields with ISO8601 timestamps per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
jsonFieldsPerLog = flag.Int("jsonFieldsPerLog", 1, "The number of JSON fields to generate per each log entry; "+
|
jsonFieldsPerLog = flag.Int("jsonFieldsPerLog", 1, "The number of JSON fields to generate per each log entry; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||||
|
|
||||||
statInterval = flag.Duration("statInterval", 10*time.Second, "The interval between publishing the stats")
|
statInterval = flag.Duration("statInterval", 10*time.Second, "The interval between publishing the stats")
|
||||||
)
|
)
|
||||||
|
|
|
@ -30,7 +30,7 @@ var (
|
||||||
"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
|
"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
|
||||||
"Smaller intervals increase disk IO load. Minimum supported value is 1s")
|
"Smaller intervals increase disk IO load. Minimum supported value is 1s")
|
||||||
logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
|
logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
|
||||||
"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields ; see also -logIngestedRows")
|
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields ; see also -logIngestedRows")
|
||||||
logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
|
logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
|
||||||
"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
|
"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
|
||||||
minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
|
minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
|
||||||
|
|
|
@ -51,4 +51,4 @@ http:
|
||||||
port: 5066
|
port: 5066
|
||||||
```
|
```
|
||||||
|
|
||||||
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance.
|
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.
|
||||||
|
|
|
@ -42,4 +42,4 @@ output.elasticsearch:
|
||||||
_stream_fields: "host.name,process.program,process.pid,container.name"
|
_stream_fields: "host.name,process.program,process.pid,container.name"
|
||||||
```
|
```
|
||||||
|
|
||||||
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance.
|
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.
|
||||||
|
|
|
@ -58,4 +58,4 @@ the example of fluentbit configuration(`filebeat.yml`)
|
||||||
header ProjectID 0
|
header ProjectID 0
|
||||||
```
|
```
|
||||||
|
|
||||||
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance.
|
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.
|
||||||
|
|
|
@ -53,4 +53,4 @@ output {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance.
|
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.
|
||||||
|
|
|
@ -58,4 +58,4 @@ the example of vector configuration(`vector.toml`)
|
||||||
ProjectID = "0"
|
ProjectID = "0"
|
||||||
```
|
```
|
||||||
|
|
||||||
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance.
|
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.
|
||||||
|
|
|
@ -166,8 +166,8 @@ Released at 2023-10-03
|
||||||
|
|
||||||
* FEATURE: add `-elasticsearch.version` command-line flag, which can be used for specifying Elasticsearch version returned by VictoriaLogs to Filebeat at [elasticsearch bulk API](https://docs.victoriametrics.com/victorialogs/data-ingestion/#elasticsearch-bulk-api). This helps resolving [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4777).
|
* FEATURE: add `-elasticsearch.version` command-line flag, which can be used for specifying Elasticsearch version returned by VictoriaLogs to Filebeat at [elasticsearch bulk API](https://docs.victoriametrics.com/victorialogs/data-ingestion/#elasticsearch-bulk-api). This helps resolving [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4777).
|
||||||
* FEATURE: expose the following metrics at [/metrics](https://docs.victoriametrics.com/VictoriaLogs/#monitoring) page:
|
* FEATURE: expose the following metrics at [/metrics](https://docs.victoriametrics.com/VictoriaLogs/#monitoring) page:
|
||||||
* `vl_data_size_bytes{type="storage"}` - on-disk size for data excluding [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes.
|
* `vl_data_size_bytes{type="storage"}` - on-disk size for data excluding [log stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) indexes.
|
||||||
* `vl_data_size_bytes{type="indexdb"}` - on-disk size for [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes.
|
* `vl_data_size_bytes{type="indexdb"}` - on-disk size for [log stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) indexes.
|
||||||
* FEATURE: add `-insert.maxFieldsPerLine` command-line flag, which can be used for limiting the number of fields per line in logs sent to VictoriaLogs via ingestion protocols. This helps to avoid issues like [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4762).
|
* FEATURE: add `-insert.maxFieldsPerLine` command-line flag, which can be used for limiting the number of fields per line in logs sent to VictoriaLogs via ingestion protocols. This helps to avoid issues like [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4762).
|
||||||
* FEATURE: expose `vl_http_request_duration_seconds` histogram at the [/metrics](https://docs.victoriametrics.com/VictoriaLogs/#monitoring) page. Thanks to @crossoverJie for [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4934).
|
* FEATURE: expose `vl_http_request_duration_seconds` histogram at the [/metrics](https://docs.victoriametrics.com/VictoriaLogs/#monitoring) page. Thanks to @crossoverJie for [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4934).
|
||||||
* FEATURE: add support of `-storage.minFreeDiskSpaceBytes` command-line flag to allow switching to read-only mode when running out of disk space at `-storageDataPath`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4737).
|
* FEATURE: add support of `-storage.minFreeDiskSpaceBytes` command-line flag to allow switching to read-only mode when running out of disk space at `-storageDataPath`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4737).
|
||||||
|
@ -186,7 +186,7 @@ Released at 2023-07-20
|
||||||
|
|
||||||
Released at 2023-07-17
|
Released at 2023-07-17
|
||||||
|
|
||||||
* FEATURE: support short form of `_time` filters over the last X minutes/hours/days/etc. For example, `_time:5m` is a short form for `_time:(now-5m, now]`, which matches logs with [timestamps](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field) for the last 5 minutes. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#time-filter) for details.
|
* FEATURE: support short form of `_time` filters over the last X minutes/hours/days/etc. For example, `_time:5m` is a short form for `_time:(now-5m, now]`, which matches logs with [timestamps](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field) for the last 5 minutes. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#time-filter) for details.
|
||||||
* FEATURE: add ability to specify offset for the selected time range. For example, `_time:5m offset 1h` is equivalent to `_time:(now-5m-1h, now-1h]`. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#time-filter) for details.
|
* FEATURE: add ability to specify offset for the selected time range. For example, `_time:5m offset 1h` is equivalent to `_time:(now-5m-1h, now-1h]`. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#time-filter) for details.
|
||||||
* FEATURE: [LogsQL](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html): replace `exact_prefix("...")` with `exact("..."*)`. This makes it consistent with [i()](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#case-insensitive-filter) filter, which can accept phrases and prefixes, e.g. `i("phrase")` and `i("phrase"*)`. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#exact-prefix-filter).
|
* FEATURE: [LogsQL](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html): replace `exact_prefix("...")` with `exact("..."*)`. This makes it consistent with [i()](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#case-insensitive-filter) filter, which can accept phrases and prefixes, e.g. `i("phrase")` and `i("phrase"*)`. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#exact-prefix-filter).
|
||||||
|
|
||||||
|
|
|
@ -36,14 +36,14 @@ VictoriaLogs is optimized specifically for logs. So it provides the following fe
|
||||||
- Up to 15x less disk space usage than Elasticsearch for the same amounts of stored logs.
|
- Up to 15x less disk space usage than Elasticsearch for the same amounts of stored logs.
|
||||||
- Ability to work with hundreds of terabytes of logs on a single node.
|
- Ability to work with hundreds of terabytes of logs on a single node.
|
||||||
- Very easy to use query language optimized for typical log analysis tasks - [LogsQL](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html).
|
- Very easy to use query language optimized for typical log analysis tasks - [LogsQL](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html).
|
||||||
- Fast full-text search over all the [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) out of the box.
|
- Fast full-text search over all the [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) out of the box.
|
||||||
- Good integration with traditional command-line tools for log analysis. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/querying/#command-line).
|
- Good integration with traditional command-line tools for log analysis. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/querying/#command-line).
|
||||||
|
|
||||||
|
|
||||||
## What is the difference between VictoriaLogs and Grafana Loki?
|
## What is the difference between VictoriaLogs and Grafana Loki?
|
||||||
|
|
||||||
Both Grafana Loki and VictoriaLogs are designed for log management and processing.
|
Both Grafana Loki and VictoriaLogs are designed for log management and processing.
|
||||||
Both systems support [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) concept.
|
Both systems support [log stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) concept.
|
||||||
|
|
||||||
VictoriaLogs and Grafana Loki have the following differences:
|
VictoriaLogs and Grafana Loki have the following differences:
|
||||||
|
|
||||||
|
@ -51,7 +51,7 @@ VictoriaLogs and Grafana Loki have the following differences:
|
||||||
It starts consuming huge amounts of RAM and working very slow when logs with high-cardinality fields are ingested into it.
|
It starts consuming huge amounts of RAM and working very slow when logs with high-cardinality fields are ingested into it.
|
||||||
See [these docs](https://grafana.com/docs/loki/latest/best-practices/) for details.
|
See [these docs](https://grafana.com/docs/loki/latest/best-practices/) for details.
|
||||||
|
|
||||||
VictoriaMetrics supports high-cardinality [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
VictoriaMetrics supports high-cardinality [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
It automatically indexes all the ingested log fields and allows performing fast full-text search over any field.
|
It automatically indexes all the ingested log fields and allows performing fast full-text search over any field.
|
||||||
|
|
||||||
- Grafana Loki provides very inconvenient query language - [LogQL](https://grafana.com/docs/loki/latest/logql/).
|
- Grafana Loki provides very inconvenient query language - [LogQL](https://grafana.com/docs/loki/latest/logql/).
|
||||||
|
@ -71,7 +71,7 @@ VictoriaLogs and Grafana Loki have the following differences:
|
||||||
ClickHouse is an extremely fast and efficient analytical database. It can be used for logs storage, analysis and processing.
|
ClickHouse is an extremely fast and efficient analytical database. It can be used for logs storage, analysis and processing.
|
||||||
VictoriaLogs is designed solely for logs. VictoriaLogs uses [similar design ideas as ClickHouse](#how-does-victorialogs-work) for achieving high performance.
|
VictoriaLogs is designed solely for logs. VictoriaLogs uses [similar design ideas as ClickHouse](#how-does-victorialogs-work) for achieving high performance.
|
||||||
|
|
||||||
- ClickHouse is good for logs if you know the set of [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) beforehand.
|
- ClickHouse is good for logs if you know the set of [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) beforehand.
|
||||||
Then you can create a table with a column per each log field and achieve the maximum possible query performance.
|
Then you can create a table with a column per each log field and achieve the maximum possible query performance.
|
||||||
|
|
||||||
If the set of log fields isn't known beforehand, or if it can change at any time, then ClickHouse can still be used,
|
If the set of log fields isn't known beforehand, or if it can change at any time, then ClickHouse can still be used,
|
||||||
|
@ -81,7 +81,7 @@ VictoriaLogs is designed solely for logs. VictoriaLogs uses [similar design idea
|
||||||
for achieving high efficiency and query performance.
|
for achieving high efficiency and query performance.
|
||||||
|
|
||||||
VictoriaLogs works optimally with any log types out of the box - structured, unstructured and mixed.
|
VictoriaLogs works optimally with any log types out of the box - structured, unstructured and mixed.
|
||||||
It works optimally with any sets of [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model),
|
It works optimally with any sets of [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model),
|
||||||
which can change in any way across different log sources.
|
which can change in any way across different log sources.
|
||||||
|
|
||||||
- ClickHouse provides SQL dialect with additional analytical functionality. It allows performing arbitrary complex analytical queries
|
- ClickHouse provides SQL dialect with additional analytical functionality. It allows performing arbitrary complex analytical queries
|
||||||
|
@ -100,7 +100,7 @@ VictoriaLogs is designed solely for logs. VictoriaLogs uses [similar design idea
|
||||||
|
|
||||||
## How does VictoriaLogs work?
|
## How does VictoriaLogs work?
|
||||||
|
|
||||||
VictoriaLogs accepts logs as [JSON entries](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
VictoriaLogs accepts logs as [JSON entries](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
It then stores every field value into a distinct data block. E.g. values for the same field across multiple log entries
|
It then stores every field value into a distinct data block. E.g. values for the same field across multiple log entries
|
||||||
are stored in a single data block. This allow reading data blocks only for the needed fields during querying.
|
are stored in a single data block. This allow reading data blocks only for the needed fields during querying.
|
||||||
|
|
||||||
|
@ -122,8 +122,8 @@ On top of this, VictoriaLogs employs additional optimizations for achieving high
|
||||||
[word](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#word-filter) or [phrase](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#phrase-filter).
|
[word](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#word-filter) or [phrase](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#phrase-filter).
|
||||||
- It uses custom encoding and compression for fields with different data types.
|
- It uses custom encoding and compression for fields with different data types.
|
||||||
For example, it encodes IP addresses as 4-byte tuples. Custom fields' encoding reduces data size on disk and improves query performance.
|
For example, it encodes IP addresses as 4-byte tuples. Custom fields' encoding reduces data size on disk and improves query performance.
|
||||||
- It physically groups logs for the same [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields)
|
- It physically groups logs for the same [log stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
|
||||||
close to each other. This improves compression ratio, which helps reducing disk space usage. This also improves query performance
|
close to each other. This improves compression ratio, which helps reducing disk space usage. This also improves query performance
|
||||||
by skipping blocks for unneeded streams when [stream filter](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#stream-filter) is used.
|
by skipping blocks for unneeded streams when [stream filter](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#stream-filter) is used.
|
||||||
- It maintains sparse index for [log timestamps](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field),
|
- It maintains sparse index for [log timestamps](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field),
|
||||||
which allow improving query performance when [time filter](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#time-filter) is used.
|
which allow improving query performance when [time filter](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#time-filter) is used.
|
||||||
|
|
|
@ -15,7 +15,7 @@ aliases:
|
||||||
LogsQL is a simple yet powerful query language for [VictoriaLogs](https://docs.victoriametrics.com/VictoriaLogs/).
|
LogsQL is a simple yet powerful query language for [VictoriaLogs](https://docs.victoriametrics.com/VictoriaLogs/).
|
||||||
It provides the following features:
|
It provides the following features:
|
||||||
|
|
||||||
- Full-text search across [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- Full-text search across [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
See [word filter](#word-filter), [phrase filter](#phrase-filter) and [prefix filter](#prefix-filter).
|
See [word filter](#word-filter), [phrase filter](#phrase-filter) and [prefix filter](#prefix-filter).
|
||||||
- Ability to combine filters into arbitrary complex [logical filters](#logical-filter).
|
- Ability to combine filters into arbitrary complex [logical filters](#logical-filter).
|
||||||
- Ability to extract structured fields from unstructured logs at query time. See [these docs](#transformations).
|
- Ability to extract structured fields from unstructured logs at query time. See [these docs](#transformations).
|
||||||
|
@ -23,14 +23,14 @@ It provides the following features:
|
||||||
|
|
||||||
## LogsQL tutorial
|
## LogsQL tutorial
|
||||||
|
|
||||||
If you aren't familiar with VictoriaLogs, then start with [key concepts docs](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html).
|
If you aren't familiar with VictoriaLogs, then start with [key concepts docs](https://docs.victoriametrics.com/victorialogs/keyconcepts/).
|
||||||
|
|
||||||
Then follow these docs:
|
Then follow these docs:
|
||||||
- [How to run VictoriaLogs](https://docs.victoriametrics.com/victorialogs/quickstart/).
|
- [How to run VictoriaLogs](https://docs.victoriametrics.com/victorialogs/quickstart/).
|
||||||
- [how to ingest data into VictoriaLogs](https://docs.victoriametrics.com/victorialogs/data-ingestion/).
|
- [how to ingest data into VictoriaLogs](https://docs.victoriametrics.com/victorialogs/data-ingestion/).
|
||||||
- [How to query VictoriaLogs](https://docs.victoriametrics.com/VictoriaLogs/querying/).
|
- [How to query VictoriaLogs](https://docs.victoriametrics.com/VictoriaLogs/querying/).
|
||||||
|
|
||||||
The simplest LogsQL query is just a [word](#word), which must be found in the [log message](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
The simplest LogsQL query is just a [word](#word), which must be found in the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
For example, the following query finds all the logs with `error` word:
|
For example, the following query finds all the logs with `error` word:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -57,7 +57,7 @@ finds log messages with the `error: cannot find file` phrase:
|
||||||
"error: cannot find file"
|
"error: cannot find file"
|
||||||
```
|
```
|
||||||
|
|
||||||
Queries above match logs with any [timestamp](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field),
|
Queries above match logs with any [timestamp](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field),
|
||||||
e.g. they may return logs from the previous year alongside recently ingested logs.
|
e.g. they may return logs from the previous year alongside recently ingested logs.
|
||||||
|
|
||||||
Usually logs from the previous year aren't so interesting comparing to the recently ingested logs.
|
Usually logs from the previous year aren't so interesting comparing to the recently ingested logs.
|
||||||
|
@ -72,9 +72,9 @@ error AND _time:5m
|
||||||
This query consists of two [filters](#filters) joined with `AND` [operator](#logical-filter):
|
This query consists of two [filters](#filters) joined with `AND` [operator](#logical-filter):
|
||||||
|
|
||||||
- The filter on the `error` [word](#word).
|
- The filter on the `error` [word](#word).
|
||||||
- The filter on the [`_time` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field).
|
- The filter on the [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field).
|
||||||
|
|
||||||
The `AND` operator means that the [log entry](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) must match both filters in order to be selected.
|
The `AND` operator means that the [log entry](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must match both filters in order to be selected.
|
||||||
|
|
||||||
Typical LogsQL query constists of multiple [filters](#filters) joined with `AND` operator. It may be tiresome typing and then reading all these `AND` words.
|
Typical LogsQL query constists of multiple [filters](#filters) joined with `AND` operator. It may be tiresome typing and then reading all these `AND` words.
|
||||||
So LogsQL allows omitting `AND` words. For example, the following query is equivalent to the query above:
|
So LogsQL allows omitting `AND` words. For example, the following query is equivalent to the query above:
|
||||||
|
@ -83,7 +83,7 @@ So LogsQL allows omitting `AND` words. For example, the following query is equiv
|
||||||
error _time:5m
|
error _time:5m
|
||||||
```
|
```
|
||||||
|
|
||||||
The query returns all the [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) by default.
|
The query returns all the [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) by default.
|
||||||
See [how to query specific fields](#querying-specific-fields).
|
See [how to query specific fields](#querying-specific-fields).
|
||||||
|
|
||||||
Suppose the query above selects too many rows because some buggy app pushes invalid error logs to VictoriaLogs. Suppose the app adds `buggy_app` [word](#word) to every log line.
|
Suppose the query above selects too many rows because some buggy app pushes invalid error logs to VictoriaLogs. Suppose the app adds `buggy_app` [word](#word) to every log line.
|
||||||
|
@ -121,8 +121,8 @@ Read [these docs](#logical-filter) for more details. There is no need in remembe
|
||||||
just wrap the needed query parts into explicit parentheses if you aren't sure in priority rules.
|
just wrap the needed query parts into explicit parentheses if you aren't sure in priority rules.
|
||||||
As an additional bonus, explicit parentheses make queries easier to read and maintain.
|
As an additional bonus, explicit parentheses make queries easier to read and maintain.
|
||||||
|
|
||||||
Queries above assume that the `error` [word](#word) is stored in the [log message](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
Queries above assume that the `error` [word](#word) is stored in the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
This word can be stored in other [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) such as `log.level`.
|
This word can be stored in other [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) such as `log.level`.
|
||||||
How to select error logs in this case? Just add the `log.level:` prefix in front of the `error` word:
|
How to select error logs in this case? Just add the `log.level:` prefix in front of the `error` word:
|
||||||
|
|
||||||
```logsq
|
```logsq
|
||||||
|
@ -149,7 +149,7 @@ _time:5m log.level:error !app:(buggy_app OR foobar)
|
||||||
```
|
```
|
||||||
|
|
||||||
The `app` field uniquely identifies the application instance if a single instance runs per each unique `app`.
|
The `app` field uniquely identifies the application instance if a single instance runs per each unique `app`.
|
||||||
In this case it is recommended associating the `app` field with [log stream fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields)
|
In this case it is recommended associating the `app` field with [log stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
|
||||||
during [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/). This usually improves both compression rate
|
during [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/). This usually improves both compression rate
|
||||||
and query performance when querying the needed streams via [`_stream` filter](#stream-filter).
|
and query performance when querying the needed streams via [`_stream` filter](#stream-filter).
|
||||||
If the `app` field is associated with the log stream, then the query above can be rewritten to more performant one:
|
If the `app` field is associated with the log stream, then the query above can be rewritten to more performant one:
|
||||||
|
@ -169,7 +169,7 @@ Now you are familiar with LogsQL basics. Read [query syntax](#query-syntax) if y
|
||||||
|
|
||||||
#### Word
|
#### Word
|
||||||
|
|
||||||
LogsQL splits all the [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) into words
|
LogsQL splits all the [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) into words
|
||||||
delimited by non-word chars such as whitespace, parens, punctuation chars, etc. For example, the `foo: (bar,"тест")!` string
|
delimited by non-word chars such as whitespace, parens, punctuation chars, etc. For example, the `foo: (bar,"тест")!` string
|
||||||
is split into `foo`, `bar` and `тест` words. Words can contain arbitrary [utf-8](https://en.wikipedia.org/wiki/UTF-8) chars.
|
is split into `foo`, `bar` and `тест` words. Words can contain arbitrary [utf-8](https://en.wikipedia.org/wiki/UTF-8) chars.
|
||||||
These words are taken into account by full-text search filters such as
|
These words are taken into account by full-text search filters such as
|
||||||
|
@ -199,8 +199,8 @@ See [the list of supported pipes in LogsQL](#pipes).
|
||||||
LogsQL supports various filters for searching for log messages (see below).
|
LogsQL supports various filters for searching for log messages (see below).
|
||||||
They can be combined into arbitrary complex queries via [logical filters](#logical-filter).
|
They can be combined into arbitrary complex queries via [logical filters](#logical-filter).
|
||||||
|
|
||||||
Filters are applied to [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field) by default.
|
Filters are applied to [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) by default.
|
||||||
If the filter must be applied to other [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model),
|
If the filter must be applied to other [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model),
|
||||||
then its' name followed by the colon must be put in front of the filter. For example, if `error` [word filter](#word-filter) must be applied
|
then its' name followed by the colon must be put in front of the filter. For example, if `error` [word filter](#word-filter) must be applied
|
||||||
to the `log.level` field, then use `log.level:error` query.
|
to the `log.level` field, then use `log.level:error` query.
|
||||||
|
|
||||||
|
@ -215,34 +215,34 @@ If doubt, it is recommended quoting field names and filter args.
|
||||||
|
|
||||||
The list of LogsQL filters:
|
The list of LogsQL filters:
|
||||||
|
|
||||||
- [Time filter](#time-filter) - matches logs with [`_time` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field) in the given time range
|
- [Time filter](#time-filter) - matches logs with [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field) in the given time range
|
||||||
- [Stream filter](#stream-filter) - matches logs, which belong to the given [streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields)
|
- [Stream filter](#stream-filter) - matches logs, which belong to the given [streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
|
||||||
- [Word filter](#word-filter) - matches logs with the given [word](#word)
|
- [Word filter](#word-filter) - matches logs with the given [word](#word)
|
||||||
- [Phrase filter](#phrase-filter) - matches logs with the given phrase
|
- [Phrase filter](#phrase-filter) - matches logs with the given phrase
|
||||||
- [Prefix filter](#prefix-filter) - matches logs with the given word prefix or phrase prefix
|
- [Prefix filter](#prefix-filter) - matches logs with the given word prefix or phrase prefix
|
||||||
- [Empty value filter](#empty-value-filter) - matches logs without the given [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
- [Empty value filter](#empty-value-filter) - matches logs without the given [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
- [Any value filter](#any-value-filter) - matches logs with the given non-empty [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
- [Any value filter](#any-value-filter) - matches logs with the given non-empty [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
- [Exact filter](#exact-filter) - matches logs with the exact value
|
- [Exact filter](#exact-filter) - matches logs with the exact value
|
||||||
- [Exact prefix filter](#exact-prefix-filter) - matches logs starting with the given prefix
|
- [Exact prefix filter](#exact-prefix-filter) - matches logs starting with the given prefix
|
||||||
- [Multi-exact filter](#multi-exact-filter) - matches logs with one of the specified exact values
|
- [Multi-exact filter](#multi-exact-filter) - matches logs with one of the specified exact values
|
||||||
- [Case-insensitive filter](#case-insensitive-filter) - matches logs with the given case-insensitive word, phrase or prefix
|
- [Case-insensitive filter](#case-insensitive-filter) - matches logs with the given case-insensitive word, phrase or prefix
|
||||||
- [Sequence filter](#sequence-filter) - matches logs with the given sequence of words or phrases
|
- [Sequence filter](#sequence-filter) - matches logs with the given sequence of words or phrases
|
||||||
- [Regexp filter](#regexp-filter) - matches logs for the given regexp
|
- [Regexp filter](#regexp-filter) - matches logs for the given regexp
|
||||||
- [Range filter](#range-filter) - matches logs with numeric [field values](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in the given range
|
- [Range filter](#range-filter) - matches logs with numeric [field values](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in the given range
|
||||||
- [IPv4 range filter](#ipv4-range-filter) - matches logs with ip address [field values](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in the given range
|
- [IPv4 range filter](#ipv4-range-filter) - matches logs with ip address [field values](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in the given range
|
||||||
- [String range filter](#string-range-filter) - matches logs with [field values](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in the given string range
|
- [String range filter](#string-range-filter) - matches logs with [field values](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in the given string range
|
||||||
- [Length range filter](#length-range-filter) - matches logs with [field values](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) of the given length range
|
- [Length range filter](#length-range-filter) - matches logs with [field values](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) of the given length range
|
||||||
- [Logical filter](#logical-filter) - allows combining other filters
|
- [Logical filter](#logical-filter) - allows combining other filters
|
||||||
|
|
||||||
|
|
||||||
### Time filter
|
### Time filter
|
||||||
|
|
||||||
VictoriaLogs scans all the logs per each query if it doesn't contain the filter on [`_time` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field).
|
VictoriaLogs scans all the logs per each query if it doesn't contain the filter on [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field).
|
||||||
It uses various optimizations in order to accelerate full scan queries without the `_time` filter,
|
It uses various optimizations in order to accelerate full scan queries without the `_time` filter,
|
||||||
but such queries can be slow if the storage contains large number of logs over long time range. The easiest way to optimize queries
|
but such queries can be slow if the storage contains large number of logs over long time range. The easiest way to optimize queries
|
||||||
is to narrow down the search with the filter on [`_time` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field).
|
is to narrow down the search with the filter on [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field).
|
||||||
|
|
||||||
For example, the following query returns [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field)
|
For example, the following query returns [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field)
|
||||||
ingested into VictoriaLogs during the last hour, which contain the `error` [word](#word):
|
ingested into VictoriaLogs during the last hour, which contain the `error` [word](#word):
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -295,11 +295,11 @@ See also:
|
||||||
|
|
||||||
### Stream filter
|
### Stream filter
|
||||||
|
|
||||||
VictoriaLogs provides an optimized way to select log entries, which belong to particular [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields).
|
VictoriaLogs provides an optimized way to select log entries, which belong to particular [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||||
This can be done via `_stream:{...}` filter. The `{...}` may contain arbitrary
|
This can be done via `_stream:{...}` filter. The `{...}` may contain arbitrary
|
||||||
[Prometheus-compatible label selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
|
[Prometheus-compatible label selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
|
||||||
over fields associated with [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields).
|
over fields associated with [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||||
For example, the following query selects [log entries](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
For example, the following query selects [log entries](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
with `app` field equal to `nginx`:
|
with `app` field equal to `nginx`:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -330,13 +330,13 @@ See also:
|
||||||
### Word filter
|
### Word filter
|
||||||
|
|
||||||
The simplest LogsQL query consists of a single [word](#word) to search in log messages. For example, the following query matches
|
The simplest LogsQL query consists of a single [word](#word) to search in log messages. For example, the following query matches
|
||||||
[log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field) with `error` [word](#word) inside them:
|
[log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) with `error` [word](#word) inside them:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
error
|
error
|
||||||
```
|
```
|
||||||
|
|
||||||
This query matches the following [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field):
|
This query matches the following [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field):
|
||||||
|
|
||||||
- `error`
|
- `error`
|
||||||
- `an error happened`
|
- `an error happened`
|
||||||
|
@ -347,8 +347,8 @@ This query doesn't match the following log messages:
|
||||||
- `ERROR`, since the filter is case-sensitive by default. Use `i(error)` for this case. See [these docs](#case-insensitive-filter) for details.
|
- `ERROR`, since the filter is case-sensitive by default. Use `i(error)` for this case. See [these docs](#case-insensitive-filter) for details.
|
||||||
- `multiple errors occurred`, since the `errors` word doesn't match `error` word. Use `error*` for this case. See [these docs](#prefix-filter) for details.
|
- `multiple errors occurred`, since the `errors` word doesn't match `error` word. Use `error*` for this case. See [these docs](#prefix-filter) for details.
|
||||||
|
|
||||||
By default the given [word](#word) is searched in the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
By default the given [word](#word) is searched in the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
Specify the [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the word and put a colon after it
|
Specify the [field name](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in front of the word and put a colon after it
|
||||||
if it must be searched in the given field. For example, the following query returns log entries containing the `error` [word](#word) in the `log.level` field:
|
if it must be searched in the given field. For example, the following query returns log entries containing the `error` [word](#word) in the `log.level` field:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -380,14 +380,14 @@ See also:
|
||||||
|
|
||||||
Is you need to search for log messages with the specific phrase inside them, then just wrap the phrase in quotes.
|
Is you need to search for log messages with the specific phrase inside them, then just wrap the phrase in quotes.
|
||||||
The phrase can contain any chars, including whitespace, punctuation, parens, etc. They are taken into account during the search.
|
The phrase can contain any chars, including whitespace, punctuation, parens, etc. They are taken into account during the search.
|
||||||
For example, the following query matches [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field)
|
For example, the following query matches [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field)
|
||||||
with `ssh: login fail` phrase inside them:
|
with `ssh: login fail` phrase inside them:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
"ssh: login fail"
|
"ssh: login fail"
|
||||||
```
|
```
|
||||||
|
|
||||||
This query matches the following [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field):
|
This query matches the following [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field):
|
||||||
|
|
||||||
- `ERROR: ssh: login fail for user "foobar"`
|
- `ERROR: ssh: login fail for user "foobar"`
|
||||||
- `ssh: login fail!`
|
- `ssh: login fail!`
|
||||||
|
@ -410,8 +410,8 @@ logs with `"foo":"bar"` phrase:
|
||||||
'"foo":"bar"'
|
'"foo":"bar"'
|
||||||
```
|
```
|
||||||
|
|
||||||
By default the given phrase is searched in the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
By default the given phrase is searched in the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
Specify the [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the phrase and put a colon after it
|
Specify the [field name](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in front of the phrase and put a colon after it
|
||||||
if it must be searched in the given field. For example, the following query returns log entries containing the `cannot open file` phrase in the `event.original` field:
|
if it must be searched in the given field. For example, the following query returns log entries containing the `cannot open file` phrase in the `event.original` field:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -442,13 +442,13 @@ See also:
|
||||||
### Prefix filter
|
### Prefix filter
|
||||||
|
|
||||||
If you need to search for log messages with [words](#word) / phrases containing some prefix, then just add `*` char to the end of the [word](#word) / phrase in the query.
|
If you need to search for log messages with [words](#word) / phrases containing some prefix, then just add `*` char to the end of the [word](#word) / phrase in the query.
|
||||||
For example, the following query returns [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field), which contain [words](#word) with `err` prefix:
|
For example, the following query returns [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field), which contain [words](#word) with `err` prefix:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
err*
|
err*
|
||||||
```
|
```
|
||||||
|
|
||||||
This query matches the following [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field):
|
This query matches the following [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field):
|
||||||
|
|
||||||
- `err: foobar`
|
- `err: foobar`
|
||||||
- `cannot open file: error occurred`
|
- `cannot open file: error occurred`
|
||||||
|
@ -459,13 +459,13 @@ This query doesn't match the following log messages:
|
||||||
- `fooerror`, since the `fooerror` [word](#word) doesn't start with `err`. Use `~"err"` for this case. See [these docs](#regexp-filter) for details.
|
- `fooerror`, since the `fooerror` [word](#word) doesn't start with `err`. Use `~"err"` for this case. See [these docs](#regexp-filter) for details.
|
||||||
|
|
||||||
Prefix filter can be applied to [phrases](#phrase-filter). For example, the following query matches
|
Prefix filter can be applied to [phrases](#phrase-filter). For example, the following query matches
|
||||||
[log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field) containing phrases with `unexpected fail` prefix:
|
[log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) containing phrases with `unexpected fail` prefix:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
"unexpected fail"*
|
"unexpected fail"*
|
||||||
```
|
```
|
||||||
|
|
||||||
This query matches the following [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field):
|
This query matches the following [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field):
|
||||||
|
|
||||||
- `unexpected fail: IO error`
|
- `unexpected fail: IO error`
|
||||||
- `error:unexpected failure`
|
- `error:unexpected failure`
|
||||||
|
@ -484,8 +484,8 @@ logs with `"foo":"bar` prefix:
|
||||||
'"foo":"bar'*
|
'"foo":"bar'*
|
||||||
```
|
```
|
||||||
|
|
||||||
By default the prefix filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
By default the prefix filter is applied to the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
Specify the needed [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the prefix filter
|
Specify the needed [field name](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in front of the prefix filter
|
||||||
in order to apply it to the given field. For example, the following query matches `log.level` field containing any word with the `err` prefix:
|
in order to apply it to the given field. For example, the following query matches `log.level` field containing any word with the `err` prefix:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -517,7 +517,7 @@ See also:
|
||||||
|
|
||||||
### Empty value filter
|
### Empty value filter
|
||||||
|
|
||||||
Sometimes it is needed to find log entries without the given [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
Sometimes it is needed to find log entries without the given [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
This can be performed with `log_field:""` syntax. For example, the following query matches log entries without `host.hostname` field:
|
This can be performed with `log_field:""` syntax. For example, the following query matches log entries without `host.hostname` field:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -533,7 +533,7 @@ See also:
|
||||||
|
|
||||||
### Any value filter
|
### Any value filter
|
||||||
|
|
||||||
Sometimes it is needed to find log entries containing any non-empty value for the given [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
Sometimes it is needed to find log entries containing any non-empty value for the given [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
This can be performed with `log_field:*` syntax. For example, the following query matches log entries with non-empty `host.hostname` field:
|
This can be performed with `log_field:*` syntax. For example, the following query matches log entries with non-empty `host.hostname` field:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -549,9 +549,9 @@ See also:
|
||||||
|
|
||||||
### Exact filter
|
### Exact filter
|
||||||
|
|
||||||
The [word filter](#word-filter) and [phrase filter](#phrase-filter) return [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field),
|
The [word filter](#word-filter) and [phrase filter](#phrase-filter) return [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field),
|
||||||
which contain the given word or phrase inside them. The message may contain additional text other than the requested word or phrase. If you need searching for log messages
|
which contain the given word or phrase inside them. The message may contain additional text other than the requested word or phrase. If you need searching for log messages
|
||||||
or [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field) with the exact value, then use the `exact` filter.
|
or [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) with the exact value, then use the `exact` filter.
|
||||||
For example, the following query returns log messages wih the exact value `fatal error: cannot find /foo/bar`:
|
For example, the following query returns log messages wih the exact value `fatal error: cannot find /foo/bar`:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -566,8 +566,8 @@ The query doesn't match the following log messages:
|
||||||
- `FATAL ERROR: cannot find /foo/bar`, since the `exact` filter is case-sensitive. Use `i("fatal error: cannot find /foo/bar")` in this case.
|
- `FATAL ERROR: cannot find /foo/bar`, since the `exact` filter is case-sensitive. Use `i("fatal error: cannot find /foo/bar")` in this case.
|
||||||
See [these docs](#case-insensitive-filter) for details.
|
See [these docs](#case-insensitive-filter) for details.
|
||||||
|
|
||||||
By default the `exact` filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
By default the `exact` filter is applied to the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
Specify the [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the `exact` filter and put a colon after it
|
Specify the [field name](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in front of the `exact` filter and put a colon after it
|
||||||
if it must be searched in the given field. For example, the following query returns log entries with the exact `error` value at `log.level` field:
|
if it must be searched in the given field. For example, the following query returns log entries with the exact `error` value at `log.level` field:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -606,7 +606,7 @@ For example, the following query matches log messages, which start from `Process
|
||||||
="Processing request"*
|
="Processing request"*
|
||||||
```
|
```
|
||||||
|
|
||||||
This filter matches the following [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field):
|
This filter matches the following [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field):
|
||||||
|
|
||||||
- `Processing request foobar`
|
- `Processing request foobar`
|
||||||
- `Processing requests from ...`
|
- `Processing requests from ...`
|
||||||
|
@ -618,8 +618,8 @@ It doesn't match the following log messages:
|
||||||
- `start: Processing request`, since the log message doesn't start with `Processing request`. Use `"Processing request"` query in this case.
|
- `start: Processing request`, since the log message doesn't start with `Processing request`. Use `"Processing request"` query in this case.
|
||||||
See [these docs](#phrase-filter) for details.
|
See [these docs](#phrase-filter) for details.
|
||||||
|
|
||||||
By default the `exact` filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
By default the `exact` filter is applied to the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
Specify the [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the `exact` filter and put a colon after it
|
Specify the [field name](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in front of the `exact` filter and put a colon after it
|
||||||
if it must be searched in the given field. For example, the following query returns log entries with `log.level` field, which starts with `err` prefix:
|
if it must be searched in the given field. For example, the following query returns log entries with `log.level` field, which starts with `err` prefix:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -695,7 +695,7 @@ log messages with `error` word in any case:
|
||||||
i(error)
|
i(error)
|
||||||
```
|
```
|
||||||
|
|
||||||
The query matches the following [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field):
|
The query matches the following [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field):
|
||||||
|
|
||||||
- `unknown error happened`
|
- `unknown error happened`
|
||||||
- `ERROR: cannot read file`
|
- `ERROR: cannot read file`
|
||||||
|
@ -707,8 +707,8 @@ The query doesn't match the following log messages:
|
||||||
- `FooError`, since the `FooError` [word](#word) has superflouos prefix `Foo`. Use `~"(?i)error"` for this case. See [these docs](#regexp-filter) for details.
|
- `FooError`, since the `FooError` [word](#word) has superflouos prefix `Foo`. Use `~"(?i)error"` for this case. See [these docs](#regexp-filter) for details.
|
||||||
- `too many Errors`, since the `Errors` [word](#word) has superflouos suffix `s`. Use `i(error*)` for this case.
|
- `too many Errors`, since the `Errors` [word](#word) has superflouos suffix `s`. Use `i(error*)` for this case.
|
||||||
|
|
||||||
By default the `i()` filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
By default the `i()` filter is applied to the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
Specify the needed [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the filter
|
Specify the needed [field name](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in front of the filter
|
||||||
in order to apply it to the given field. For example, the following query matches `log.level` field containing `error` [word](#word) in any case:
|
in order to apply it to the given field. For example, the following query matches `log.level` field containing `error` [word](#word) in any case:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -740,7 +740,7 @@ See also:
|
||||||
|
|
||||||
### Sequence filter
|
### Sequence filter
|
||||||
|
|
||||||
Sometimes it is needed to find [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field)
|
Sometimes it is needed to find [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field)
|
||||||
with [words](#word) or phrases in a particular order. For example, if log messages with `error` word followed by `open file` phrase
|
with [words](#word) or phrases in a particular order. For example, if log messages with `error` word followed by `open file` phrase
|
||||||
must be found, then the following LogsQL query can be used:
|
must be found, then the following LogsQL query can be used:
|
||||||
|
|
||||||
|
@ -753,8 +753,8 @@ The query doesn't match the `cannot open file: error` message, since the `open f
|
||||||
If you need matching log messages with both `error` word and `open file` phrase, then use `error AND "open file"` query. See [these docs](#logical-filter)
|
If you need matching log messages with both `error` word and `open file` phrase, then use `error AND "open file"` query. See [these docs](#logical-filter)
|
||||||
for details.
|
for details.
|
||||||
|
|
||||||
By default the `seq()` filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
By default the `seq()` filter is applied to the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
Specify the needed [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the filter
|
Specify the needed [field name](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in front of the filter
|
||||||
in order to apply it to the given field. For example, the following query matches `event.original` field containing `(error, "open file")` sequence:
|
in order to apply it to the given field. For example, the following query matches `event.original` field containing `(error, "open file")` sequence:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -785,7 +785,7 @@ For example, the following query returns all the log messages containing `err` o
|
||||||
~"err|warn"
|
~"err|warn"
|
||||||
```
|
```
|
||||||
|
|
||||||
The query matches the following [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field), which contain either `err` or `warn` substrings:
|
The query matches the following [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field), which contain either `err` or `warn` substrings:
|
||||||
|
|
||||||
- `error: cannot read data`
|
- `error: cannot read data`
|
||||||
- `2 warnings have been raised`
|
- `2 warnings have been raised`
|
||||||
|
@ -804,8 +804,8 @@ logs matching `"foo":"(bar|baz)"` regexp:
|
||||||
'"foo":"(bar|baz)"'
|
'"foo":"(bar|baz)"'
|
||||||
```
|
```
|
||||||
|
|
||||||
By default the regexp filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
By default the regexp filter is applied to the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
Specify the needed [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the filter
|
Specify the needed [field name](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in front of the filter
|
||||||
in order to apply it to the given field. For example, the following query matches `event.original` field containing either `err` or `warn` substrings:
|
in order to apply it to the given field. For example, the following query matches `event.original` field containing either `err` or `warn` substrings:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -866,15 +866,15 @@ parentheses with square brackets. For example:
|
||||||
|
|
||||||
The range boundaries can contain any [supported numeric values](#numeric-values).
|
The range boundaries can contain any [supported numeric values](#numeric-values).
|
||||||
|
|
||||||
Note that the `range()` filter doesn't match [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
Note that the `range()` filter doesn't match [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
with non-numeric values alongside numeric values. For example, `range(1, 10)` doesn't match `the request took 4.2 seconds`
|
with non-numeric values alongside numeric values. For example, `range(1, 10)` doesn't match `the request took 4.2 seconds`
|
||||||
[log message](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field), since the `4.2` number is surrounded by other text.
|
[log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field), since the `4.2` number is surrounded by other text.
|
||||||
Extract the numeric value from the message with `parse(_msg, "the request took <request_duration> seconds")` [transformation](#transformations)
|
Extract the numeric value from the message with `parse(_msg, "the request took <request_duration> seconds")` [transformation](#transformations)
|
||||||
and then apply the `range()` [filter pipe](#filter-pipe) to the extracted `request_duration` field.
|
and then apply the `range()` [filter pipe](#filter-pipe) to the extracted `request_duration` field.
|
||||||
|
|
||||||
Performance tips:
|
Performance tips:
|
||||||
|
|
||||||
- It is better to query pure numeric [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
- It is better to query pure numeric [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
instead of extracting numeric field from text field via [transformations](#transformations) at query time.
|
instead of extracting numeric field from text field via [transformations](#transformations) at query time.
|
||||||
- See [other performance tips](#performance-tips).
|
- See [other performance tips](#performance-tips).
|
||||||
|
|
||||||
|
@ -903,29 +903,29 @@ user.ip:ipv4_range("127.0.0.0/8")
|
||||||
```
|
```
|
||||||
|
|
||||||
If you need matching a single IPv4 address, then just put it inside `ipv4_range()`. For example, the following query matches `1.2.3.4` IP
|
If you need matching a single IPv4 address, then just put it inside `ipv4_range()`. For example, the following query matches `1.2.3.4` IP
|
||||||
at `user.ip` [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model):
|
at `user.ip` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model):
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
user.ip:ipv4_range("1.2.3.4")
|
user.ip:ipv4_range("1.2.3.4")
|
||||||
```
|
```
|
||||||
|
|
||||||
Note that the `ipv4_range()` doesn't match a string with IPv4 address if this string contains other text. For example, `ipv4_range("127.0.0.0/24")`
|
Note that the `ipv4_range()` doesn't match a string with IPv4 address if this string contains other text. For example, `ipv4_range("127.0.0.0/24")`
|
||||||
doesn't match `request from 127.0.0.1: done` [log message](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field),
|
doesn't match `request from 127.0.0.1: done` [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field),
|
||||||
since the `127.0.0.1` ip is surrounded by other text. Extract the IP from the message with `parse(_msg, "request from <ip>: done")` [transformation](#transformations)
|
since the `127.0.0.1` ip is surrounded by other text. Extract the IP from the message with `parse(_msg, "request from <ip>: done")` [transformation](#transformations)
|
||||||
and then apply the `ipv4_range()` [filter pipe](#filter-pipe) to the extracted `ip` field.
|
and then apply the `ipv4_range()` [filter pipe](#filter-pipe) to the extracted `ip` field.
|
||||||
|
|
||||||
Hints:
|
Hints:
|
||||||
|
|
||||||
- If you need searching for [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field) containing the given `X.Y.Z.Q` IPv4 address,
|
- If you need searching for [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) containing the given `X.Y.Z.Q` IPv4 address,
|
||||||
then `"X.Y.Z.Q"` query can be used. See [these docs](#phrase-filter) for details.
|
then `"X.Y.Z.Q"` query can be used. See [these docs](#phrase-filter) for details.
|
||||||
- If you need searching for [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field) containing
|
- If you need searching for [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) containing
|
||||||
at least a single IPv4 address out of the given list, then `"ip1" OR "ip2" ... OR "ipN"` query can be used. See [these docs](#logical-filter) for details.
|
at least a single IPv4 address out of the given list, then `"ip1" OR "ip2" ... OR "ipN"` query can be used. See [these docs](#logical-filter) for details.
|
||||||
- If you need finding log entries with `ip` field in multiple ranges, then use `ip:(ipv4_range(range1) OR ipv4_range(range2) ... OR ipv4_range(rangeN))` query.
|
- If you need finding log entries with `ip` field in multiple ranges, then use `ip:(ipv4_range(range1) OR ipv4_range(range2) ... OR ipv4_range(rangeN))` query.
|
||||||
See [these docs](#logical-filter) for details.
|
See [these docs](#logical-filter) for details.
|
||||||
|
|
||||||
Performance tips:
|
Performance tips:
|
||||||
|
|
||||||
- It is better querying pure IPv4 [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
- It is better querying pure IPv4 [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
instead of extracting IPv4 from text field via [transformations](#transformations) at query time.
|
instead of extracting IPv4 from text field via [transformations](#transformations) at query time.
|
||||||
- See [other performance tips](#performance-tips).
|
- See [other performance tips](#performance-tips).
|
||||||
|
|
||||||
|
@ -960,7 +960,7 @@ See also:
|
||||||
### Length range filter
|
### Length range filter
|
||||||
|
|
||||||
If you need to filter log message by its length, then `len_range()` filter can be used.
|
If you need to filter log message by its length, then `len_range()` filter can be used.
|
||||||
For example, the following LogsQL query matches [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field)
|
For example, the following LogsQL query matches [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field)
|
||||||
with lengths in the range `[5, 10]` chars:
|
with lengths in the range `[5, 10]` chars:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -977,7 +977,7 @@ This query doesn't match the following log messages:
|
||||||
- `foo`, since it is too short
|
- `foo`, since it is too short
|
||||||
- `foo bar baz abc`, sinc it is too long
|
- `foo bar baz abc`, sinc it is too long
|
||||||
|
|
||||||
It is possible to use `inf` as the upper bound. For example, the following query matches [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field)
|
It is possible to use `inf` as the upper bound. For example, the following query matches [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field)
|
||||||
with the length bigger or equal to 5 chars:
|
with the length bigger or equal to 5 chars:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -990,8 +990,8 @@ The range boundaries can be expressed in the following forms:
|
||||||
- Binary form. Form example, `len_range(0b100110, 0b11111101)`
|
- Binary form. Form example, `len_range(0b100110, 0b11111101)`
|
||||||
- Integer form with `_` delimiters for better readability. For example, `len_range(1_000, 2_345_678)`.
|
- Integer form with `_` delimiters for better readability. For example, `len_range(1_000, 2_345_678)`.
|
||||||
|
|
||||||
By default the `len_range()` is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
|
By default the `len_range()` is applied to the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field).
|
||||||
Put the [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the `len_range()` in order to apply
|
Put the [field name](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) in front of the `len_range()` in order to apply
|
||||||
the filter to the needed field. For example, the following query matches log entries with the `foo` field length in the range `[10, 20]` chars:
|
the filter to the needed field. For example, the following query matches log entries with the `foo` field length in the range `[10, 20]` chars:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -1009,34 +1009,34 @@ See also:
|
||||||
Simpler LogsQL [filters](#filters) can be combined into more complex filters with the following logical operations:
|
Simpler LogsQL [filters](#filters) can be combined into more complex filters with the following logical operations:
|
||||||
|
|
||||||
- `q1 AND q2` - matches common log entries returned by both `q1` and `q2`. Arbitrary number of [filters](#filters) can be combined with `AND` operation.
|
- `q1 AND q2` - matches common log entries returned by both `q1` and `q2`. Arbitrary number of [filters](#filters) can be combined with `AND` operation.
|
||||||
For example, `error AND file AND app` matches [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field),
|
For example, `error AND file AND app` matches [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field),
|
||||||
which simultaneously contain `error`, `file` and `app` [words](#word).
|
which simultaneously contain `error`, `file` and `app` [words](#word).
|
||||||
The `AND` operation is frequently used in LogsQL queries, so it is allowed to skip the `AND` word.
|
The `AND` operation is frequently used in LogsQL queries, so it is allowed to skip the `AND` word.
|
||||||
For example, `error file app` is equivalent to `error AND file AND app`.
|
For example, `error file app` is equivalent to `error AND file AND app`.
|
||||||
|
|
||||||
- `q1 OR q2` - merges log entries returned by both `q1` and `q2`. Aribtrary number of [filters](#filters) can be combined with `OR` operation.
|
- `q1 OR q2` - merges log entries returned by both `q1` and `q2`. Aribtrary number of [filters](#filters) can be combined with `OR` operation.
|
||||||
For example, `error OR warning OR info` matches [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field),
|
For example, `error OR warning OR info` matches [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field),
|
||||||
which contain at least one of `error`, `warning` or `info` [words](#word).
|
which contain at least one of `error`, `warning` or `info` [words](#word).
|
||||||
|
|
||||||
- `NOT q` - returns all the log entries except of those which match `q`. For example, `NOT info` returns all the
|
- `NOT q` - returns all the log entries except of those which match `q`. For example, `NOT info` returns all the
|
||||||
[log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field),
|
[log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field),
|
||||||
which do not contain `info` [word](#word). The `NOT` operation is frequently used in LogsQL queries, so it is allowed substituting `NOT` with `!` in queries.
|
which do not contain `info` [word](#word). The `NOT` operation is frequently used in LogsQL queries, so it is allowed substituting `NOT` with `!` in queries.
|
||||||
For example, `!info` is equivalent to `NOT info`.
|
For example, `!info` is equivalent to `NOT info`.
|
||||||
|
|
||||||
The `NOT` operation has the highest priority, `AND` has the middle priority and `OR` has the lowest priority.
|
The `NOT` operation has the highest priority, `AND` has the middle priority and `OR` has the lowest priority.
|
||||||
The priority order can be changed with parentheses. For example, `NOT info OR debug` is interpreted as `(NOT info) OR debug`,
|
The priority order can be changed with parentheses. For example, `NOT info OR debug` is interpreted as `(NOT info) OR debug`,
|
||||||
so it matches [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field),
|
so it matches [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field),
|
||||||
which do not contain `info` [word](#word), while it also matches messages with `debug` word (which may contain the `info` word).
|
which do not contain `info` [word](#word), while it also matches messages with `debug` word (which may contain the `info` word).
|
||||||
This is not what most users expect. In this case the query can be rewritten to `NOT (info OR debug)`,
|
This is not what most users expect. In this case the query can be rewritten to `NOT (info OR debug)`,
|
||||||
which correctly returns log messages without `info` and `debug` [words](#word).
|
which correctly returns log messages without `info` and `debug` [words](#word).
|
||||||
|
|
||||||
LogsQL supports arbitrary complex logical queries with arbitrary mix of `AND`, `OR` and `NOT` operations and parentheses.
|
LogsQL supports arbitrary complex logical queries with arbitrary mix of `AND`, `OR` and `NOT` operations and parentheses.
|
||||||
|
|
||||||
By default logical filters apply to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field)
|
By default logical filters apply to the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field)
|
||||||
unless the inner filters explicitly specify the needed [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) via `field_name:filter` syntax.
|
unless the inner filters explicitly specify the needed [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) via `field_name:filter` syntax.
|
||||||
For example, `(error OR warn) AND host.hostname:host123` is interpreted as `(_msg:error OR _msg:warn) AND host.hostname:host123`.
|
For example, `(error OR warn) AND host.hostname:host123` is interpreted as `(_msg:error OR _msg:warn) AND host.hostname:host123`.
|
||||||
|
|
||||||
It is possible to specify a single [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) for multiple filters
|
It is possible to specify a single [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) for multiple filters
|
||||||
with the following syntax:
|
with the following syntax:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -1050,7 +1050,7 @@ Performance tips:
|
||||||
- VictoriaLogs executes logical operations from the left to the right, so it is recommended moving the most specific
|
- VictoriaLogs executes logical operations from the left to the right, so it is recommended moving the most specific
|
||||||
and the fastest filters (such as [word filter](#word-filter) and [phrase filter](#phrase-filter)) to the left,
|
and the fastest filters (such as [word filter](#word-filter) and [phrase filter](#phrase-filter)) to the left,
|
||||||
while moving less specific and the slowest filters (such as [regexp filter](#regexp-filter) and [case-insensitive filter](#case-insensitive-filter))
|
while moving less specific and the slowest filters (such as [regexp filter](#regexp-filter) and [case-insensitive filter](#case-insensitive-filter))
|
||||||
to the right. For example, if you need to find [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field)
|
to the right. For example, if you need to find [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field)
|
||||||
with the `error` word, which match some `/foo/(bar|baz)` regexp,
|
with the `error` word, which match some `/foo/(bar|baz)` regexp,
|
||||||
it is better from performance PoV to use the query `error ~"/foo/(bar|baz)"` instead of `~"/foo/(bar|baz)" error`.
|
it is better from performance PoV to use the query `error ~"/foo/(bar|baz)"` instead of `~"/foo/(bar|baz)" error`.
|
||||||
|
|
||||||
|
@ -1071,25 +1071,25 @@ _time:5m | stats by (_stream) count() per_stream_logs | sort by (per_stream_logs
|
||||||
|
|
||||||
LogsQL supports the following pipes:
|
LogsQL supports the following pipes:
|
||||||
|
|
||||||
- [`copy`](#copy-pipe) copies [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- [`copy`](#copy-pipe) copies [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
- [`delete`](#delete-pipe) deletes [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- [`delete`](#delete-pipe) deletes [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
- [`extract`](#extract-pipe) extracts the sepcified text into the given log fields.
|
- [`extract`](#extract-pipe) extracts the sepcified text into the given log fields.
|
||||||
- [`field_names`](#field_names-pipe) returns all the names of [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- [`field_names`](#field_names-pipe) returns all the names of [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
- [`fields`](#fields-pipe) selects the given set of [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- [`fields`](#fields-pipe) selects the given set of [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
- [`filter`](#filter-pipe) applies additional [filters](#filters) to results.
|
- [`filter`](#filter-pipe) applies additional [filters](#filters) to results.
|
||||||
- [`format`](#format-pipe) formats ouptut field from input [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- [`format`](#format-pipe) formats ouptut field from input [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
- [`limit`](#limit-pipe) limits the number selected logs.
|
- [`limit`](#limit-pipe) limits the number selected logs.
|
||||||
- [`offset`](#offset-pipe) skips the given number of selected logs.
|
- [`offset`](#offset-pipe) skips the given number of selected logs.
|
||||||
- [`rename`](#rename-pipe) renames [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- [`rename`](#rename-pipe) renames [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
- [`sort`](#sort-pipe) sorts logs by the given [fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- [`sort`](#sort-pipe) sorts logs by the given [fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
- [`stats`](#stats-pipe) calculates various stats over the selected logs.
|
- [`stats`](#stats-pipe) calculates various stats over the selected logs.
|
||||||
- [`uniq`](#uniq-pipe) returns unique log entires.
|
- [`uniq`](#uniq-pipe) returns unique log entires.
|
||||||
- [`unpack_json`](#unpack_json-pipe) unpacks JSON fields from [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- [`unpack_json`](#unpack_json-pipe) unpacks JSON fields from [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
- [`unpack_logfmt`](#unpack_logfmt-pipe) unpacks [logfmt](https://brandur.org/logfmt) fields from [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- [`unpack_logfmt`](#unpack_logfmt-pipe) unpacks [logfmt](https://brandur.org/logfmt) fields from [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
|
|
||||||
### copy pipe
|
### copy pipe
|
||||||
|
|
||||||
If some [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) must be copied, then `| copy src1 as dst1, ..., srcN as dstN` [pipe](#pipes) can be used.
|
If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must be copied, then `| copy src1 as dst1, ..., srcN as dstN` [pipe](#pipes) can be used.
|
||||||
For example, the following query copies `host` field to `server` for logs over the last 5 minutes, so the output contains both `host` and `server` fields:
|
For example, the following query copies `host` field to `server` for logs over the last 5 minutes, so the output contains both `host` and `server` fields:
|
||||||
|
|
||||||
```logsq
|
```logsq
|
||||||
|
@ -1116,7 +1116,7 @@ See also:
|
||||||
|
|
||||||
### delete pipe
|
### delete pipe
|
||||||
|
|
||||||
If some [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) must be deleted, then `| delete field1, ..., fieldN` [pipe](#pipes) can be used.
|
If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must be deleted, then `| delete field1, ..., fieldN` [pipe](#pipes) can be used.
|
||||||
For example, the following query deletes `host` and `app` fields from the logs over the last 5 minutes:
|
For example, the following query deletes `host` and `app` fields from the logs over the last 5 minutes:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -1133,7 +1133,7 @@ See also:
|
||||||
### extract pipe
|
### extract pipe
|
||||||
|
|
||||||
`| extract "pattern" from field_name` [pipe](#pipes) allows extracting abitrary text into output fields according to the [`pattern`](#format-for-extract-pipe-pattern) from the given
|
`| extract "pattern" from field_name` [pipe](#pipes) allows extracting abitrary text into output fields according to the [`pattern`](#format-for-extract-pipe-pattern) from the given
|
||||||
[`field_name`](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model). Existing log fields remain unchanged after the `| extract ...` pipe.
|
[`field_name`](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). Existing log fields remain unchanged after the `| extract ...` pipe.
|
||||||
|
|
||||||
`| extract ...` can be useful for extracting additional fields needed for further data processing with other pipes such as [`stats` pipe](#stats-pipe) or [`sort` pipe](#sort-pipe).
|
`| extract ...` can be useful for extracting additional fields needed for further data processing with other pipes such as [`stats` pipe](#stats-pipe) or [`sort` pipe](#sort-pipe).
|
||||||
|
|
||||||
|
@ -1245,7 +1245,7 @@ For example, the following `pattern` properly matches `a < b` text by extracting
|
||||||
If some log entries must be skipped from [`extract` pipe](#extract-pipe), then add `if (<filters>)` filter after the `extract` word.
|
If some log entries must be skipped from [`extract` pipe](#extract-pipe), then add `if (<filters>)` filter after the `extract` word.
|
||||||
The `<filters>` can contain arbitrary [filters](#filters). For example, the following query extracts `ip` field
|
The `<filters>` can contain arbitrary [filters](#filters). For example, the following query extracts `ip` field
|
||||||
from [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) only
|
from [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) only
|
||||||
if the input [log entry](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) doesn't contain `ip` field or this field is empty:
|
if the input [log entry](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) doesn't contain `ip` field or this field is empty:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
_time:5m | extract if (ip:"") "ip=<ip> "
|
_time:5m | extract if (ip:"") "ip=<ip> "
|
||||||
|
@ -1260,7 +1260,7 @@ _time:5m | extract "ip=<ip> " keep_original_fields
|
||||||
|
|
||||||
### field_names pipe
|
### field_names pipe
|
||||||
|
|
||||||
`| field_names` [pipe](#pipes) returns all the names of [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
`| field_names` [pipe](#pipes) returns all the names of [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
with an estimated number of logs per each field name.
|
with an estimated number of logs per each field name.
|
||||||
For example, the following query returns all the field names with the number of matching logs over the last 5 minutes:
|
For example, the following query returns all the field names with the number of matching logs over the last 5 minutes:
|
||||||
|
|
||||||
|
@ -1276,7 +1276,7 @@ See also:
|
||||||
|
|
||||||
### fields pipe
|
### fields pipe
|
||||||
|
|
||||||
By default all the [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) are returned in the response.
|
By default all the [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) are returned in the response.
|
||||||
It is possible to select the given set of log fields with `| fields field1, ..., fieldN` [pipe](#pipes). For example, the following query selects only `host`
|
It is possible to select the given set of log fields with `| fields field1, ..., fieldN` [pipe](#pipes). For example, the following query selects only `host`
|
||||||
and [`_msg`](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) fields from logs for the last 5 minutes:
|
and [`_msg`](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) fields from logs for the last 5 minutes:
|
||||||
|
|
||||||
|
@ -1301,7 +1301,7 @@ See also:
|
||||||
Sometimes it is needed to apply additional filters on the calculated results. This can be done with `| filter ...` [pipe](#pipes).
|
Sometimes it is needed to apply additional filters on the calculated results. This can be done with `| filter ...` [pipe](#pipes).
|
||||||
The `filter` pipe can contain arbitrary [filters](#filters).
|
The `filter` pipe can contain arbitrary [filters](#filters).
|
||||||
|
|
||||||
For example, the following query returns `host` [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) values
|
For example, the following query returns `host` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) values
|
||||||
if the number of log messages with the `error` [word](#word) for them over the last hour exceeds `1_000`:
|
if the number of log messages with the `error` [word](#word) for them over the last hour exceeds `1_000`:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -1404,7 +1404,7 @@ See also:
|
||||||
|
|
||||||
### rename pipe
|
### rename pipe
|
||||||
|
|
||||||
If some [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) must be renamed, then `| rename src1 as dst1, ..., srcN as dstN` [pipe](#pipes) can be used.
|
If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must be renamed, then `| rename src1 as dst1, ..., srcN as dstN` [pipe](#pipes) can be used.
|
||||||
For example, the following query renames `host` field to `server` for logs over the last 5 minutes, so the output contains `server` field instead of `host` field:
|
For example, the following query renames `host` field to `server` for logs over the last 5 minutes, so the output contains `server` field instead of `host` field:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -1430,7 +1430,7 @@ See also:
|
||||||
### sort pipe
|
### sort pipe
|
||||||
|
|
||||||
By default logs are selected in arbitrary order because of performance reasons. If logs must be sorted, then `| sort by (field1, ..., fieldN)` [pipe](#pipes) can be used.
|
By default logs are selected in arbitrary order because of performance reasons. If logs must be sorted, then `| sort by (field1, ..., fieldN)` [pipe](#pipes) can be used.
|
||||||
The returned logs are sorted by the given [fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
The returned logs are sorted by the given [fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
using [natural sorting](https://en.wikipedia.org/wiki/Natural_sort_order).
|
using [natural sorting](https://en.wikipedia.org/wiki/Natural_sort_order).
|
||||||
|
|
||||||
For example, the following query returns logs for the last 5 minutes sorted by [`_stream`](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
|
For example, the following query returns logs for the last 5 minutes sorted by [`_stream`](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
|
||||||
|
@ -1462,14 +1462,14 @@ Sorting of big number of logs can consume a lot of CPU time and memory. Sometime
|
||||||
or the smallest values. This can be done by adding `limit N` to the end of `sort ...` pipe.
|
or the smallest values. This can be done by adding `limit N` to the end of `sort ...` pipe.
|
||||||
Such a query consumes lower amounts of memory when sorting big number of logs, since it keeps in memory only `N` log entries.
|
Such a query consumes lower amounts of memory when sorting big number of logs, since it keeps in memory only `N` log entries.
|
||||||
For example, the following query returns top 10 log entries with the biggest values
|
For example, the following query returns top 10 log entries with the biggest values
|
||||||
for the `request_duration` [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) during the last hour:
|
for the `request_duration` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) during the last hour:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
_time:1h | sort by (request_duration desc) limit 10
|
_time:1h | sort by (request_duration desc) limit 10
|
||||||
```
|
```
|
||||||
|
|
||||||
If the first `N` sorted results must be skipped, then `offset N` can be added to `sort` pipe. For example,
|
If the first `N` sorted results must be skipped, then `offset N` can be added to `sort` pipe. For example,
|
||||||
the following query skips the first 10 logs with the biggest `request_duration` [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model),
|
the following query skips the first 10 logs with the biggest `request_duration` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model),
|
||||||
and then returns the next 20 sorted logs for the last 5 minutes:
|
and then returns the next 20 sorted logs for the last 5 minutes:
|
||||||
|
|
||||||
```logsql
|
```logsql
|
||||||
|
@ -1482,7 +1482,7 @@ It is recommended limiting the number of logs before sorting with the following
|
||||||
- Adding `limit N` to the end of `sort ...` pipe.
|
- Adding `limit N` to the end of `sort ...` pipe.
|
||||||
- Reducing the selected time range with [time filter](#time-filter).
|
- Reducing the selected time range with [time filter](#time-filter).
|
||||||
- Using more specific [filters](#filters), so they select less logs.
|
- Using more specific [filters](#filters), so they select less logs.
|
||||||
- Limiting the number of selected [fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) via [`fields` pipe](#fields-pipe).
|
- Limiting the number of selected [fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) via [`fields` pipe](#fields-pipe).
|
||||||
|
|
||||||
See also:
|
See also:
|
||||||
|
|
||||||
|
@ -2171,7 +2171,7 @@ See also:
|
||||||
## Stream context
|
## Stream context
|
||||||
|
|
||||||
LogsQL will support the ability to select the given number of surrounding log lines for the selected log lines
|
LogsQL will support the ability to select the given number of surrounding log lines for the selected log lines
|
||||||
on a [per-stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) basis.
|
on a [per-stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) basis.
|
||||||
|
|
||||||
See the [Roadmap](https://docs.victoriametrics.com/VictoriaLogs/Roadmap.html) for details.
|
See the [Roadmap](https://docs.victoriametrics.com/VictoriaLogs/Roadmap.html) for details.
|
||||||
|
|
||||||
|
@ -2179,15 +2179,15 @@ See the [Roadmap](https://docs.victoriametrics.com/VictoriaLogs/Roadmap.html) fo
|
||||||
|
|
||||||
LogsQL supports the following transformations on the log entries selected with [filters](#filters):
|
LogsQL supports the following transformations on the log entries selected with [filters](#filters):
|
||||||
|
|
||||||
- Extracting arbitrary text from [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) according to the provided pattern.
|
- Extracting arbitrary text from [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) according to the provided pattern.
|
||||||
See [these docs](#extract-pipe) for details.
|
See [these docs](#extract-pipe) for details.
|
||||||
- Unpacking JSON fields from [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model). See [these docs](#unpack_json-pipe).
|
- Unpacking JSON fields from [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). See [these docs](#unpack_json-pipe).
|
||||||
- Unpacking [logfmt](https://brandur.org/logfmt) fields from [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model). See [these docs](#unpack_logfmt-pipe).
|
- Unpacking [logfmt](https://brandur.org/logfmt) fields from [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). See [these docs](#unpack_logfmt-pipe).
|
||||||
- Creating a new field from existing [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) according to the provided format. See [these docs](#format-pipe).
|
- Creating a new field from existing [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) according to the provided format. See [these docs](#format-pipe).
|
||||||
|
|
||||||
LogsQL will support the following transformations in the future:
|
LogsQL will support the following transformations in the future:
|
||||||
|
|
||||||
- Creating a new field according to math calculations over existing [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
- Creating a new field according to math calculations over existing [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
|
|
||||||
See the [Roadmap](https://docs.victoriametrics.com/VictoriaLogs/Roadmap.html) for details.
|
See the [Roadmap](https://docs.victoriametrics.com/VictoriaLogs/Roadmap.html) for details.
|
||||||
|
|
||||||
|
@ -2216,7 +2216,7 @@ By default VictoriaLogs doesn't sort the returned results because of performance
|
||||||
|
|
||||||
LogsQL provides the following [pipes](#pipes) for limiting the number of returned log entries:
|
LogsQL provides the following [pipes](#pipes) for limiting the number of returned log entries:
|
||||||
|
|
||||||
- [`fields`](#fields-pipe) and [`delete`](#delete-pipe) pipes allow limiting the set of [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) to return.
|
- [`fields`](#fields-pipe) and [`delete`](#delete-pipe) pipes allow limiting the set of [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) to return.
|
||||||
- [`limit` pipe](#limit-pipe) allows limiting the number of log entries to return.
|
- [`limit` pipe](#limit-pipe) allows limiting the number of log entries to return.
|
||||||
|
|
||||||
## Querying specific fields
|
## Querying specific fields
|
||||||
|
@ -2270,7 +2270,7 @@ Internally duration values are converted into nanoseconds.
|
||||||
|
|
||||||
- It is highly recommended specifying [time filter](#time-filter) in order to narrow down the search to specific time range.
|
- It is highly recommended specifying [time filter](#time-filter) in order to narrow down the search to specific time range.
|
||||||
- It is highly recommended specifying [stream filter](#stream-filter) in order to narrow down the search
|
- It is highly recommended specifying [stream filter](#stream-filter) in order to narrow down the search
|
||||||
to specific [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields).
|
to specific [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||||
- Move faster filters such as [word filter](#word-filter) and [phrase filter](#phrase-filter) to the beginning of the query.
|
- Move faster filters such as [word filter](#word-filter) and [phrase filter](#phrase-filter) to the beginning of the query.
|
||||||
This rule doesn't apply to [time filter](#time-filter) and [stream filter](#stream-filter), which can be put at any place of the query.
|
This rule doesn't apply to [time filter](#time-filter) and [stream filter](#stream-filter), which can be put at any place of the query.
|
||||||
- Move more specific filters, which match lower number of log entries, to the beginning of the query.
|
- Move more specific filters, which match lower number of log entries, to the beginning of the query.
|
||||||
|
|
|
@ -14,7 +14,7 @@ aliases:
|
||||||
# VictoriaLogs Quick Start
|
# VictoriaLogs Quick Start
|
||||||
|
|
||||||
It is recommended to read [README](https://docs.victoriametrics.com/VictoriaLogs/)
|
It is recommended to read [README](https://docs.victoriametrics.com/VictoriaLogs/)
|
||||||
and [Key Concepts](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html)
|
and [Key Concepts](https://docs.victoriametrics.com/victorialogs/keyconcepts/)
|
||||||
before you start working with VictoriaLogs.
|
before you start working with VictoriaLogs.
|
||||||
|
|
||||||
## How to install and run VictoriaLogs
|
## How to install and run VictoriaLogs
|
||||||
|
@ -124,7 +124,7 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line
|
||||||
VictoriaLogs stores the ingested data to the `victoria-logs-data` directory by default. The directory can be changed
|
VictoriaLogs stores the ingested data to the `victoria-logs-data` directory by default. The directory can be changed
|
||||||
via `-storageDataPath` command-line flag. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/#storage) for details.
|
via `-storageDataPath` command-line flag. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/#storage) for details.
|
||||||
|
|
||||||
By default VictoriaLogs stores [log entries](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html) with timestamps
|
By default VictoriaLogs stores [log entries](https://docs.victoriametrics.com/victorialogs/keyconcepts/) with timestamps
|
||||||
in the time range `[now-7d, now]`, while dropping logs outside the given time range.
|
in the time range `[now-7d, now]`, while dropping logs outside the given time range.
|
||||||
E.g. it uses the retention of 7 days. Read [these docs](https://docs.victoriametrics.com/VictoriaLogs/#retention) on how to control the retention
|
E.g. it uses the retention of 7 days. Read [these docs](https://docs.victoriametrics.com/VictoriaLogs/#retention) on how to control the retention
|
||||||
for the [ingested](https://docs.victoriametrics.com/victorialogs/data-ingestion/) logs.
|
for the [ingested](https://docs.victoriametrics.com/victorialogs/data-ingestion/) logs.
|
||||||
|
|
|
@ -16,7 +16,7 @@ VictoriaLogs provides the following key features:
|
||||||
- VictoriaLogs is much easier to set up and operate compared to Elasticsearch and Grafana Loki.
|
- VictoriaLogs is much easier to set up and operate compared to Elasticsearch and Grafana Loki.
|
||||||
See [these docs](https://docs.victoriametrics.com/victorialogs/quickstart/).
|
See [these docs](https://docs.victoriametrics.com/victorialogs/quickstart/).
|
||||||
- VictoriaLogs provides easy yet powerful query language with full-text search capabilities across
|
- VictoriaLogs provides easy yet powerful query language with full-text search capabilities across
|
||||||
all the [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) -
|
all the [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) -
|
||||||
see [LogsQL docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html).
|
see [LogsQL docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html).
|
||||||
- VictoriaLogs can be seamlessly combined with good old Unix tools for log analysis such as `grep`, `less`, `sort`, `jq`, etc.
|
- VictoriaLogs can be seamlessly combined with good old Unix tools for log analysis such as `grep`, `less`, `sort`, `jq`, etc.
|
||||||
See [these docs](https://docs.victoriametrics.com/VictoriaLogs/querying/#command-line) for details.
|
See [these docs](https://docs.victoriametrics.com/VictoriaLogs/querying/#command-line) for details.
|
||||||
|
@ -24,7 +24,7 @@ VictoriaLogs provides the following key features:
|
||||||
It runs smoothly on both Raspberry PI and a server with hundreds of CPU cores and terabytes of RAM.
|
It runs smoothly on both Raspberry PI and a server with hundreds of CPU cores and terabytes of RAM.
|
||||||
- VictoriaLogs can handle up to 30x bigger data volumes than Elasticsearch and Grafana Loki when running on the same hardware.
|
- VictoriaLogs can handle up to 30x bigger data volumes than Elasticsearch and Grafana Loki when running on the same hardware.
|
||||||
See [these docs](#benchmarks).
|
See [these docs](#benchmarks).
|
||||||
- VictoriaLogs supports fast full-text search over high-cardinality [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
- VictoriaLogs supports fast full-text search over high-cardinality [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
such as `trace_id`, `user_id` and `ip`.
|
such as `trace_id`, `user_id` and `ip`.
|
||||||
- VictoriaLogs supports multitenancy - see [these docs](#multitenancy).
|
- VictoriaLogs supports multitenancy - see [these docs](#multitenancy).
|
||||||
- VictoriaLogs supports out-of-order logs' ingestion aka backfilling.
|
- VictoriaLogs supports out-of-order logs' ingestion aka backfilling.
|
||||||
|
@ -203,7 +203,7 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line
|
||||||
-logIngestedRows
|
-logIngestedRows
|
||||||
Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams
|
Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams
|
||||||
-logNewStreams
|
-logNewStreams
|
||||||
Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields ; see also -logIngestedRows
|
Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields ; see also -logIngestedRows
|
||||||
-loggerDisableTimestamps
|
-loggerDisableTimestamps
|
||||||
Whether to disable writing timestamps in logs
|
Whether to disable writing timestamps in logs
|
||||||
-loggerErrorsPerSecondLimit int
|
-loggerErrorsPerSecondLimit int
|
||||||
|
|
|
@ -28,8 +28,8 @@ Substitute the `localhost:9428` address inside `hosts` section with the real TCP
|
||||||
|
|
||||||
See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on the `parameters` section.
|
See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on the `parameters` section.
|
||||||
|
|
||||||
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
and uses the correct [stream fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields).
|
and uses the correct [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||||
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters)
|
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters)
|
||||||
and inspecting VictoriaLogs logs then:
|
and inspecting VictoriaLogs logs then:
|
||||||
|
|
||||||
|
@ -43,7 +43,7 @@ output.elasticsearch:
|
||||||
debug: "1"
|
debug: "1"
|
||||||
```
|
```
|
||||||
|
|
||||||
If some [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) must be skipped
|
If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must be skipped
|
||||||
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
||||||
For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
|
For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
|
||||||
|
|
||||||
|
|
|
@ -30,8 +30,8 @@ Substitute the host (`localhost`) and port (`9428`) with the real TCP address of
|
||||||
|
|
||||||
See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on the query args specified in the `uri`.
|
See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on the query args specified in the `uri`.
|
||||||
|
|
||||||
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
and uses the correct [stream fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields).
|
and uses the correct [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||||
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) in the `uri`
|
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) in the `uri`
|
||||||
and inspecting VictoriaLogs logs then:
|
and inspecting VictoriaLogs logs then:
|
||||||
|
|
||||||
|
@ -46,7 +46,7 @@ and inspecting VictoriaLogs logs then:
|
||||||
json_date_format iso8601
|
json_date_format iso8601
|
||||||
```
|
```
|
||||||
|
|
||||||
If some [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) must be skipped
|
If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must be skipped
|
||||||
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
||||||
For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
|
For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
|
||||||
|
|
||||||
|
@ -76,7 +76,7 @@ This usually allows saving network bandwidth and costs by up to 5 times:
|
||||||
compress gzip
|
compress gzip
|
||||||
```
|
```
|
||||||
|
|
||||||
By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#multitenancy).
|
By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/victorialogs/keyconcepts/#multitenancy).
|
||||||
If you need storing logs in other tenant, then specify the needed tenant via `header` options.
|
If you need storing logs in other tenant, then specify the needed tenant via `header` options.
|
||||||
For example, the following `fluentbit.conf` config instructs Fluentbit to store the data to `(AccountID=12, ProjectID=34)` tenant:
|
For example, the following `fluentbit.conf` config instructs Fluentbit to store the data to `(AccountID=12, ProjectID=34)` tenant:
|
||||||
|
|
||||||
|
|
|
@ -32,8 +32,8 @@ Substitute `localhost:9428` address inside `hosts` with the real TCP address of
|
||||||
|
|
||||||
See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on the `parameters` section.
|
See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on the `parameters` section.
|
||||||
|
|
||||||
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
and uses the correct [stream fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields).
|
and uses the correct [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||||
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters)
|
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters)
|
||||||
and inspecting VictoriaLogs logs then:
|
and inspecting VictoriaLogs logs then:
|
||||||
|
|
||||||
|
@ -51,7 +51,7 @@ output {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
If some [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) must be skipped
|
If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must be skipped
|
||||||
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
||||||
For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
|
For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
|
||||||
|
|
||||||
|
|
|
@ -24,17 +24,17 @@ clients:
|
||||||
|
|
||||||
Substitute `localhost:9428` address inside `clients` with the real TCP address of VictoriaLogs.
|
Substitute `localhost:9428` address inside `clients` with the real TCP address of VictoriaLogs.
|
||||||
|
|
||||||
By default VictoriaLogs stores all the ingested logs into a single [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields).
|
By default VictoriaLogs stores all the ingested logs into a single [log stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||||
Storing all the logs in a single log stream may be not so efficient, so it is recommended to specify `_stream_fields` query arg
|
Storing all the logs in a single log stream may be not so efficient, so it is recommended to specify `_stream_fields` query arg
|
||||||
with the list of labels, which uniquely identify log streams. There is no need in specifying all the labels Promtail generates there -
|
with the list of labels, which uniquely identify log streams. There is no need in specifying all the labels Promtail generates there -
|
||||||
it is usually enough specifying `instance` and `job` labels. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields)
|
it is usually enough specifying `instance` and `job` labels. See [these docs](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
|
||||||
for details.
|
for details.
|
||||||
|
|
||||||
See also [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on other supported query args.
|
See also [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on other supported query args.
|
||||||
There is no need in specifying `_msg_field` and `_time_field` query args, since VictoriaLogs automatically extracts log message and timestamp from the ingested Loki data.
|
There is no need in specifying `_msg_field` and `_time_field` query args, since VictoriaLogs automatically extracts log message and timestamp from the ingested Loki data.
|
||||||
|
|
||||||
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
and uses the correct [stream fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields).
|
and uses the correct [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||||
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters)
|
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters)
|
||||||
and inspecting VictoriaLogs logs then:
|
and inspecting VictoriaLogs logs then:
|
||||||
|
|
||||||
|
@ -43,7 +43,7 @@ clients:
|
||||||
- url: http://localhost:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app&debug=1
|
- url: http://localhost:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app&debug=1
|
||||||
```
|
```
|
||||||
|
|
||||||
If some [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) must be skipped
|
If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must be skipped
|
||||||
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
||||||
For example, the following config instructs VictoriaLogs to ignore `filename` and `stream` fields in the ingested logs:
|
For example, the following config instructs VictoriaLogs to ignore `filename` and `stream` fields in the ingested logs:
|
||||||
|
|
||||||
|
|
|
@ -56,13 +56,13 @@ echo '{"create":{}}
|
||||||
|
|
||||||
It is possible to push thousands of log lines in a single request to this API.
|
It is possible to push thousands of log lines in a single request to this API.
|
||||||
|
|
||||||
If the [timestamp field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field) is set to `"0"`,
|
If the [timestamp field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field) is set to `"0"`,
|
||||||
then the current timestamp at VictoriaLogs side is used per each ingested log line.
|
then the current timestamp at VictoriaLogs side is used per each ingested log line.
|
||||||
Otherwise the timestamp field must be in the [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) format. For example, `2023-06-20T15:32:10Z`.
|
Otherwise the timestamp field must be in the [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) format. For example, `2023-06-20T15:32:10Z`.
|
||||||
Optional fractional part of seconds can be specified after the dot - `2023-06-20T15:32:10.123Z`.
|
Optional fractional part of seconds can be specified after the dot - `2023-06-20T15:32:10.123Z`.
|
||||||
Timezone can be specified instead of `Z` suffix - `2023-06-20T15:32:10+02:00`.
|
Timezone can be specified instead of `Z` suffix - `2023-06-20T15:32:10+02:00`.
|
||||||
|
|
||||||
See [these docs](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) for details on fields,
|
See [these docs](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) for details on fields,
|
||||||
which must be present in the ingested log messages.
|
which must be present in the ingested log messages.
|
||||||
|
|
||||||
The API accepts various http parameters, which can change the data ingestion behavior - [these docs](#http-parameters) for details.
|
The API accepts various http parameters, which can change the data ingestion behavior - [these docs](#http-parameters) for details.
|
||||||
|
@ -106,13 +106,13 @@ echo '{ "log": { "level": "info", "message": "hello world" }, "date": "0", "stre
|
||||||
|
|
||||||
It is possible to push unlimited number of log lines in a single request to this API.
|
It is possible to push unlimited number of log lines in a single request to this API.
|
||||||
|
|
||||||
If the [timestamp field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field) is set to `"0"`,
|
If the [timestamp field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field) is set to `"0"`,
|
||||||
then the current timestamp at VictoriaLogs side is used per each ingested log line.
|
then the current timestamp at VictoriaLogs side is used per each ingested log line.
|
||||||
Otherwise the timestamp field must be in the [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) format. For example, `2023-06-20T15:32:10Z`.
|
Otherwise the timestamp field must be in the [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) format. For example, `2023-06-20T15:32:10Z`.
|
||||||
Optional fractional part of seconds can be specified after the dot - `2023-06-20T15:32:10.123Z`.
|
Optional fractional part of seconds can be specified after the dot - `2023-06-20T15:32:10.123Z`.
|
||||||
Timezone can be specified instead of `Z` suffix - `2023-06-20T15:32:10+02:00`.
|
Timezone can be specified instead of `Z` suffix - `2023-06-20T15:32:10+02:00`.
|
||||||
|
|
||||||
See [these docs](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) for details on fields,
|
See [these docs](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) for details on fields,
|
||||||
which must be present in the ingested log messages.
|
which must be present in the ingested log messages.
|
||||||
|
|
||||||
The API accepts various http parameters, which can change the data ingestion behavior - [these docs](#http-parameters) for details.
|
The API accepts various http parameters, which can change the data ingestion behavior - [these docs](#http-parameters) for details.
|
||||||
|
@ -185,22 +185,22 @@ See also:
|
||||||
|
|
||||||
VictoriaLogs accepts the following parameters at [data ingestion HTTP APIs](#http-apis):
|
VictoriaLogs accepts the following parameters at [data ingestion HTTP APIs](#http-apis):
|
||||||
|
|
||||||
- `_msg_field` - it must contain the name of the [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
- `_msg_field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
with the [log message](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field) generated by the log shipper.
|
with the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) generated by the log shipper.
|
||||||
This is usually the `message` field for Filebeat and Logstash.
|
This is usually the `message` field for Filebeat and Logstash.
|
||||||
If the `_msg_field` parameter isn't set, then VictoriaLogs reads the log message from the `_msg` field.
|
If the `_msg_field` parameter isn't set, then VictoriaLogs reads the log message from the `_msg` field.
|
||||||
|
|
||||||
- `_time_field` - it must contain the name of the [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
- `_time_field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
with the [log timestamp](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field) generated by the log shipper.
|
with the [log timestamp](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field) generated by the log shipper.
|
||||||
This is usually the `@timestamp` field for Filebeat and Logstash.
|
This is usually the `@timestamp` field for Filebeat and Logstash.
|
||||||
If the `_time_field` parameter isn't set, then VictoriaLogs reads the timestamp from the `_time` field.
|
If the `_time_field` parameter isn't set, then VictoriaLogs reads the timestamp from the `_time` field.
|
||||||
If this field doesn't exist, then the current timestamp is used.
|
If this field doesn't exist, then the current timestamp is used.
|
||||||
|
|
||||||
- `_stream_fields` - it should contain comma-separated list of [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) names,
|
- `_stream_fields` - it should contain comma-separated list of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names,
|
||||||
which uniquely identify every [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) collected the log shipper.
|
which uniquely identify every [log stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) collected the log shipper.
|
||||||
If the `_stream_fields` parameter isn't set, then all the ingested logs are written to default log stream - `{}`.
|
If the `_stream_fields` parameter isn't set, then all the ingested logs are written to default log stream - `{}`.
|
||||||
|
|
||||||
- `ignore_fields` - this parameter may contain the list of [log field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) names,
|
- `ignore_fields` - this parameter may contain the list of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names,
|
||||||
which must be ignored during data ingestion.
|
which must be ignored during data ingestion.
|
||||||
|
|
||||||
- `debug` - if this parameter is set to `1`, then the ingested logs aren't stored in VictoriaLogs. Instead,
|
- `debug` - if this parameter is set to `1`, then the ingested logs aren't stored in VictoriaLogs. Instead,
|
||||||
|
@ -232,24 +232,24 @@ See [how to query specific fields](https://docs.victoriametrics.com/VictoriaLogs
|
||||||
VictoriaLogs provides the following command-line flags, which can help debugging data ingestion issues:
|
VictoriaLogs provides the following command-line flags, which can help debugging data ingestion issues:
|
||||||
|
|
||||||
- `-logNewStreams` - if this flag is passed to VictoriaLogs, then it logs all the newly
|
- `-logNewStreams` - if this flag is passed to VictoriaLogs, then it logs all the newly
|
||||||
registered [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields).
|
registered [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||||
This may help debugging [high cardinality issues](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#high-cardinality).
|
This may help debugging [high cardinality issues](https://docs.victoriametrics.com/victorialogs/keyconcepts/#high-cardinality).
|
||||||
- `-logIngestedRows` - if this flag is passed to VictoriaLogs, then it logs all the ingested
|
- `-logIngestedRows` - if this flag is passed to VictoriaLogs, then it logs all the ingested
|
||||||
[log entries](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model).
|
[log entries](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
|
||||||
See also `debug` [parameter](#http-parameters).
|
See also `debug` [parameter](#http-parameters).
|
||||||
|
|
||||||
VictoriaLogs exposes various [metrics](https://docs.victoriametrics.com/VictoriaLogs/#monitoring), which may help debugging data ingestion issues:
|
VictoriaLogs exposes various [metrics](https://docs.victoriametrics.com/VictoriaLogs/#monitoring), which may help debugging data ingestion issues:
|
||||||
|
|
||||||
- `vl_rows_ingested_total` - the number of ingested [log entries](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
- `vl_rows_ingested_total` - the number of ingested [log entries](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
since the last VictoriaLogs restart. If this number icreases over time, then logs are successfully ingested into VictoriaLogs.
|
since the last VictoriaLogs restart. If this number icreases over time, then logs are successfully ingested into VictoriaLogs.
|
||||||
The ingested logs can be inspected in the following ways:
|
The ingested logs can be inspected in the following ways:
|
||||||
- By passing `debug=1` parameter to every request to [data ingestion APIs](#http-apis). The ingested rows aren't stored in VictoriaLogs
|
- By passing `debug=1` parameter to every request to [data ingestion APIs](#http-apis). The ingested rows aren't stored in VictoriaLogs
|
||||||
in this case. Instead, they are logged, so they can be investigated later.
|
in this case. Instead, they are logged, so they can be investigated later.
|
||||||
The `vl_rows_dropped_total` [metric](https://docs.victoriametrics.com/VictoriaLogs/#monitoring) is incremented for each logged row.
|
The `vl_rows_dropped_total` [metric](https://docs.victoriametrics.com/VictoriaLogs/#monitoring) is incremented for each logged row.
|
||||||
- By passing `-logIngestedRows` command-line flag to VictoriaLogs. In this case it logs all the ingested data, so it can be investigated later.
|
- By passing `-logIngestedRows` command-line flag to VictoriaLogs. In this case it logs all the ingested data, so it can be investigated later.
|
||||||
- `vl_streams_created_total` - the number of created [log streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields)
|
- `vl_streams_created_total` - the number of created [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
|
||||||
since the last VictoriaLogs restart. If this metric grows rapidly during extended periods of time, then this may lead
|
since the last VictoriaLogs restart. If this metric grows rapidly during extended periods of time, then this may lead
|
||||||
to [high cardinality issues](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#high-cardinality).
|
to [high cardinality issues](https://docs.victoriametrics.com/victorialogs/keyconcepts/#high-cardinality).
|
||||||
The newly created log streams can be inspected in logs by passing `-logNewStreams` command-line flag to VictoriaLogs.
|
The newly created log streams can be inspected in logs by passing `-logNewStreams` command-line flag to VictoriaLogs.
|
||||||
|
|
||||||
## Log collectors and data ingestion formats
|
## Log collectors and data ingestion formats
|
||||||
|
|
|
@ -39,8 +39,8 @@ Replace `your_input` with the name of the `inputs` section, which collects logs.
|
||||||
See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on parameters specified
|
See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on parameters specified
|
||||||
in the `[sinks.vlogs.query]` section.
|
in the `[sinks.vlogs.query]` section.
|
||||||
|
|
||||||
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model)
|
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||||
and uses the correct [stream fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields).
|
and uses the correct [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||||
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters)
|
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters)
|
||||||
in the `[sinks.vlogs.query]` section and inspecting VictoriaLogs logs then:
|
in the `[sinks.vlogs.query]` section and inspecting VictoriaLogs logs then:
|
||||||
|
|
||||||
|
@ -60,7 +60,7 @@ in the `[sinks.vlogs.query]` section and inspecting VictoriaLogs logs then:
|
||||||
debug = "1"
|
debug = "1"
|
||||||
```
|
```
|
||||||
|
|
||||||
If some [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) must be skipped
|
If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must be skipped
|
||||||
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
||||||
For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
|
For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
|
||||||
|
|
||||||
|
@ -120,7 +120,7 @@ This usually allows saving network bandwidth and costs by up to 5 times:
|
||||||
_stream_fields = "host,container_name"
|
_stream_fields = "host,container_name"
|
||||||
```
|
```
|
||||||
|
|
||||||
By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#multitenancy).
|
By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/victorialogs/keyconcepts/#multitenancy).
|
||||||
If you need storing logs in other tenant, then specify the needed tenant via `[sinks.vlogq.request.headers]` section.
|
If you need storing logs in other tenant, then specify the needed tenant via `[sinks.vlogq.request.headers]` section.
|
||||||
For example, the following `vector.toml` config instructs Vector to store the data to `(AccountID=12, ProjectID=34)` tenant:
|
For example, the following `vector.toml` config instructs Vector to store the data to `(AccountID=12, ProjectID=34)` tenant:
|
||||||
|
|
||||||
|
|
|
@ -532,7 +532,7 @@ which are usually used for log analysis - `grep`, `jq`, `awk`, `sort`, `uniq`, `
|
||||||
|
|
||||||
For example, the following command uses `wc -l` Unix command for counting the number of log messages
|
For example, the following command uses `wc -l` Unix command for counting the number of log messages
|
||||||
with the `error` [word](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#word)
|
with the `error` [word](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#word)
|
||||||
received from [streams](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) with `app="nginx"` field
|
received from [streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) with `app="nginx"` field
|
||||||
during the last 5 minutes:
|
during the last 5 minutes:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
@ -543,14 +543,14 @@ See [these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#strea
|
||||||
[these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#time-filter) about `_time` filter
|
[these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#time-filter) about `_time` filter
|
||||||
and [these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#logical-filter) about `AND` operator.
|
and [these docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#logical-filter) about `AND` operator.
|
||||||
|
|
||||||
The following example shows how to sort query results by the [`_time` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field):
|
The following example shows how to sort query results by the [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field):
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
curl http://localhost:9428/select/logsql/query -d 'query=error' | jq -r '._time + " " + ._msg' | sort | less
|
curl http://localhost:9428/select/logsql/query -d 'query=error' | jq -r '._time + " " + ._msg' | sort | less
|
||||||
```
|
```
|
||||||
|
|
||||||
This command uses `jq` for extracting [`_time`](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#time-field)
|
This command uses `jq` for extracting [`_time`](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field)
|
||||||
and [`_msg`](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field) fields from the returned results,
|
and [`_msg`](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) fields from the returned results,
|
||||||
and piping them to `sort` command.
|
and piping them to `sort` command.
|
||||||
|
|
||||||
Note that the `sort` command needs to read all the response stream before returning the sorted results. So the command above
|
Note that the `sort` command needs to read all the response stream before returning the sorted results. So the command above
|
||||||
|
@ -559,7 +559,7 @@ before sorting the results. See [these tips](https://docs.victoriametrics.com/Vi
|
||||||
on how to narrow down query results.
|
on how to narrow down query results.
|
||||||
|
|
||||||
The following example calculates stats on the number of log messages received during the last 5 minutes
|
The following example calculates stats on the number of log messages received during the last 5 minutes
|
||||||
grouped by `log.level` [field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model):
|
grouped by `log.level` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model):
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
curl http://localhost:9428/select/logsql/query -d 'query=_time:5m log.level:*' | jq -r '."log.level"' | sort | uniq -c
|
curl http://localhost:9428/select/logsql/query -d 'query=_time:5m log.level:*' | jq -r '."log.level"' | sort | uniq -c
|
||||||
|
@ -571,5 +571,5 @@ are sorted with `sort` command and, finally, they are passed to `uniq -c` comman
|
||||||
|
|
||||||
See also:
|
See also:
|
||||||
|
|
||||||
- [Key concepts](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html).
|
- [Key concepts](https://docs.victoriametrics.com/victorialogs/keyconcepts/).
|
||||||
- [LogsQL docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html).
|
- [LogsQL docs](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html).
|
||||||
|
|
|
@ -11,7 +11,7 @@ import (
|
||||||
|
|
||||||
// JSONParser parses a single JSON log message into Fields.
|
// JSONParser parses a single JSON log message into Fields.
|
||||||
//
|
//
|
||||||
// See https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model
|
// See https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model
|
||||||
//
|
//
|
||||||
// Use GetParser() for obtaining the parser.
|
// Use GetParser() for obtaining the parser.
|
||||||
type JSONParser struct {
|
type JSONParser struct {
|
||||||
|
|
|
@ -59,7 +59,7 @@ type StorageConfig struct {
|
||||||
// LogNewStreams indicates whether to log newly created log streams.
|
// LogNewStreams indicates whether to log newly created log streams.
|
||||||
//
|
//
|
||||||
// This can be useful for debugging of high cardinality issues.
|
// This can be useful for debugging of high cardinality issues.
|
||||||
// https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#high-cardinality
|
// https://docs.victoriametrics.com/victorialogs/keyconcepts/#high-cardinality
|
||||||
LogNewStreams bool
|
LogNewStreams bool
|
||||||
|
|
||||||
// LogIngestedRows indicates whether to log the ingested log entries.
|
// LogIngestedRows indicates whether to log the ingested log entries.
|
||||||
|
|
Loading…
Reference in a new issue