From 116623a0f0993ba45407ffe93d8b621dd1c3ecb7 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 28 Jul 2023 09:46:50 -0700 Subject: [PATCH] docs/VictoriaLogs: add missing information about Promtail support for data ingestion Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4728 --- .../docker/victorialogs/promtail/config.yml | 2 +- docs/VictoriaLogs/QuickStart.md | 1 + docs/VictoriaLogs/Roadmap.md | 2 +- docs/VictoriaLogs/data-ingestion/Promtail.md | 16 +++++++++++----- docs/VictoriaLogs/data-ingestion/README.md | 1 + 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/deployment/docker/victorialogs/promtail/config.yml b/deployment/docker/victorialogs/promtail/config.yml index 4a5974a7d..98a78e3c9 100644 --- a/deployment/docker/victorialogs/promtail/config.yml +++ b/deployment/docker/victorialogs/promtail/config.yml @@ -6,7 +6,7 @@ positions: filename: /tmp/positions.yaml clients: - - url: http://vlogs:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app,pid + - url: http://vlogs:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app tenant_id: "0:0" scrape_configs: diff --git a/docs/VictoriaLogs/QuickStart.md b/docs/VictoriaLogs/QuickStart.md index 96d65d091..582bef687 100644 --- a/docs/VictoriaLogs/QuickStart.md +++ b/docs/VictoriaLogs/QuickStart.md @@ -143,6 +143,7 @@ Here are a Docker-compose demos, which start VictoriaLogs and push logs to it vi - [Fluentbit demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentbit-docker) - [Logstash demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/logstash) - [Vector demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector-docker) +- [Promtail demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/promtail) You can use [this Helm chart](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-logs-single/README.md) as a demo for running Fluentbit in Kubernetes with VictoriaLogs. diff --git a/docs/VictoriaLogs/Roadmap.md b/docs/VictoriaLogs/Roadmap.md index d3ab17afb..7a820463f 100644 --- a/docs/VictoriaLogs/Roadmap.md +++ b/docs/VictoriaLogs/Roadmap.md @@ -29,9 +29,9 @@ See [these docs](https://docs.victoriametrics.com/VictoriaLogs/) for details. The following functionality is planned in the future versions of VictoriaLogs: - Support for [data ingestion](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/) from popular log collectors and formats: - - Promtail (aka Grafana Loki) - Fluentd - Syslog + - Journald (systemd) - Add missing functionality to [LogsQL](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html): - [Stream context](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#stream-context). - [Transformation functions](https://docs.victoriametrics.com/VictoriaLogs/LogsQL.html#transformations). diff --git a/docs/VictoriaLogs/data-ingestion/Promtail.md b/docs/VictoriaLogs/data-ingestion/Promtail.md index 7d72e9a99..b8b34febd 100644 --- a/docs/VictoriaLogs/data-ingestion/Promtail.md +++ b/docs/VictoriaLogs/data-ingestion/Promtail.md @@ -8,12 +8,18 @@ for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.co ```yaml clients: - - url: http://localhost:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app,pid + - url: http://localhost:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app ``` Substitute `localhost:9428` address inside `clients` with the real TCP address of VictoriaLogs. -See [these docs](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/#http-parameters) for details on the used URL query parameter section. +By default VictoriaLogs stores all the ingested logs into a single [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields). +Storing all the logs in a single log stream may be not so efficient, so it is recommended to specify `_stream_fields` query arg +with the list of labels, which uniquely identify log streams. There is no need in specifying all the labels Promtail generates there - +it is usually enough specifying `instance` and `job` labels. See [these docs](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) +for details. + +See also [these docs](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/#http-parameters) for details on other supported query args. There is no need in specifying `_msg_field` and `_time_field` query args, since VictoriaLogs automatically extracts log message and timestamp from the ingested Loki data. It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) @@ -23,7 +29,7 @@ and inspecting VictoriaLogs logs then: ```yaml clients: - - url: http://localhost:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app,pid&debug=1 + - url: http://localhost:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app&debug=1 ``` If some [log fields](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) must be skipped @@ -32,7 +38,7 @@ For example, the following config instructs VictoriaLogs to ignore `filename` an ```yaml clients: - - url: http://localhost:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app,pid&ignore_fields=filename,stream + - url: http://localhost:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app&ignore_fields=filename,stream ``` By default the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/VictoriaLogs/#multitenancy). @@ -43,7 +49,7 @@ For example, the following config instructs VictoriaLogs to store logs in the `( ```yaml clients: - - url: http://localhost:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app,pid&debug=1 + - url: http://localhost:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app&debug=1 tenant_id: "12:34" ``` diff --git a/docs/VictoriaLogs/data-ingestion/README.md b/docs/VictoriaLogs/data-ingestion/README.md index cb56428b1..aa86c705a 100644 --- a/docs/VictoriaLogs/data-ingestion/README.md +++ b/docs/VictoriaLogs/data-ingestion/README.md @@ -261,3 +261,4 @@ Here is the list of log collectors and their ingestion formats supported by Vict | [Fluentbit](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Fluentbit.html) | No | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/http) | | [Logstash](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Logstash.html) | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) | No | | [Vector](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Vector.html) | [Yes](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) | No | +| [Promtail](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Promtail.html) | No | No |