From 4e5a68ed08d8285a2d135400d8f10eba2fa293b0 Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Fri, 1 Sep 2023 12:45:43 +0400 Subject: [PATCH] deployment/docker: add VictoriaLogs (#4929) * deployment/docker: add VictoriaLogs configuration Signed-off-by: Zakhar Bessarab * deployment/docker/victorialogs: remove outdated comment It was added in order to indicate that it is required to build VictoriaLogs manually before starting it at the time there was no public release available. Currently, there is a public tag and it is not required to build it from sources. Signed-off-by: Zakhar Bessarab * deployment/docker/victorialogs/fluentbit: include log path in stream configuration Signed-off-by: Zakhar Bessarab * deployment/docker: add reference to monitoring setup for VictoriaLogs Signed-off-by: Zakhar Bessarab --------- Signed-off-by: Zakhar Bessarab --- deployment/docker/README.md | 13 ++++++++ deployment/docker/docker-compose.yml | 18 +++++++++++ deployment/docker/fluent-bit.conf | 30 +++++++++++++++++++ .../filebeat-docker/docker-compose.yml | 1 - .../filebeat-syslog/docker-compose.yml | 1 - .../fluentbit-docker/docker-compose.yml | 1 - .../fluentbit-docker/fluent-bit.conf | 5 ++-- .../victorialogs/logstash/docker-compose.yml | 1 - .../victorialogs/promtail/docker-compose.yml | 1 - .../vector-docker/docker-compose.yml | 1 - 10 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 deployment/docker/fluent-bit.conf diff --git a/deployment/docker/README.md b/deployment/docker/README.md index d4ab1a9cf..ed9364557 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -128,3 +128,16 @@ of alerting rules is the following: Please, also see [how to monitor](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#monitoring) VictoriaMetrics installations. + +## VictoriaLogs server + +VictoriaLogs will be accessible on the following port: `--httpListenAddr=:9428` + +[Fluent Bit](https://docs.fluentbit.io/manual) is used to send logs to VictoriaLogs instance. +Fluent Bit is configured to send logs from running containers to VictoriaLogs instance. +Additionally, it is configured to listen for syslog logs on port `5140` and send them to VictoriaLogs instance. + +To access VictoriaLogs UI use link [http://localhost:9428/select/vmui/](http://localhost:9428/select/vmui/). + +Please, also see [how to monitor](https://docs.victoriametrics.com/VictoriaLogs/#monitoring) +VictoriaLogs installations. diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index a1a59dd33..568b3b9d2 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -92,9 +92,27 @@ services: networks: - vm_net restart: always + fluentbit: + image: cr.fluentbit.io/fluent/fluent-bit:2.1.4 + volumes: + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf + depends_on: [victorialogs] + ports: + - "5140:5140" + victorialogs: + image: docker.io/victoriametrics/victoria-logs:v0.3.0-victorialogs + command: + - "--storageDataPath=/vlogs" + - "--httpListenAddr=:9428" + volumes: + - victorialogs-fluentbit:/vlogs + ports: + - "9428:9428" volumes: vmagentdata: {} vmdata: {} grafanadata: {} + victorialogs-fluentbit: {} networks: vm_net: diff --git a/deployment/docker/fluent-bit.conf b/deployment/docker/fluent-bit.conf new file mode 100644 index 000000000..42d349338 --- /dev/null +++ b/deployment/docker/fluent-bit.conf @@ -0,0 +1,30 @@ +[INPUT] + name tail + path /var/lib/docker/containers/**/*.log + path_key path + multiline.parser docker, cri + Parser docker + Docker_Mode On + +[INPUT] + Name syslog + Listen 0.0.0.0 + Port 5140 + Parser syslog-rfc3164 + Mode tcp + +[SERVICE] + Flush 1 + Parsers_File parsers.conf + +[Output] + Name http + Match * + host victorialogs + port 9428 + compress gzip + uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date + format json_lines + json_date_format iso8601 + header AccountID 0 + header ProjectID 0 diff --git a/deployment/docker/victorialogs/filebeat-docker/docker-compose.yml b/deployment/docker/victorialogs/filebeat-docker/docker-compose.yml index 734106467..58d482aea 100644 --- a/deployment/docker/victorialogs/filebeat-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/filebeat-docker/docker-compose.yml @@ -21,7 +21,6 @@ services: command: - -beat.uri=http://filebeat-victorialogs:5066 - # Run `make package-victoria-logs` to build victoria-logs image victorialogs: image: docker.io/victoriametrics/victoria-logs:v0.3.0-victorialogs volumes: diff --git a/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml b/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml index 471376e68..ba5fb63e7 100644 --- a/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml +++ b/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml @@ -12,7 +12,6 @@ services: ports: - "5140:5140" - # Run `make package-victoria-logs` to build victoria-logs image victorialogs: image: docker.io/victoriametrics/victoria-logs:v0.3.0-victorialogs volumes: diff --git a/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml b/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml index 76b76a457..31b1341e7 100644 --- a/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml @@ -10,7 +10,6 @@ services: ports: - "5140:5140" - # Run `make package-victoria-logs` to build victoria-logs image victorialogs: image: docker.io/victoriametrics/victoria-logs:v0.3.0-victorialogs volumes: diff --git a/deployment/docker/victorialogs/fluentbit-docker/fluent-bit.conf b/deployment/docker/victorialogs/fluentbit-docker/fluent-bit.conf index 8cca3a70e..42d349338 100644 --- a/deployment/docker/victorialogs/fluentbit-docker/fluent-bit.conf +++ b/deployment/docker/victorialogs/fluentbit-docker/fluent-bit.conf @@ -1,6 +1,7 @@ [INPUT] name tail path /var/lib/docker/containers/**/*.log + path_key path multiline.parser docker, cri Parser docker Docker_Mode On @@ -22,8 +23,8 @@ host victorialogs port 9428 compress gzip - uri /insert/jsonline?_stream_fields=stream&_msg_field=log&_time_field=date + uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date format json_lines json_date_format iso8601 header AccountID 0 - header ProjectID 0 \ No newline at end of file + header ProjectID 0 diff --git a/deployment/docker/victorialogs/logstash/docker-compose.yml b/deployment/docker/victorialogs/logstash/docker-compose.yml index d45b6f33c..f2bb4db47 100644 --- a/deployment/docker/victorialogs/logstash/docker-compose.yml +++ b/deployment/docker/victorialogs/logstash/docker-compose.yml @@ -13,7 +13,6 @@ services: ports: - "5140:5140" - # Run `make package-victoria-logs` to build victoria-logs image victorialogs: image: docker.io/victoriametrics/victoria-logs:v0.3.0-victorialogs volumes: diff --git a/deployment/docker/victorialogs/promtail/docker-compose.yml b/deployment/docker/victorialogs/promtail/docker-compose.yml index 6a51c81ed..4ba245197 100644 --- a/deployment/docker/victorialogs/promtail/docker-compose.yml +++ b/deployment/docker/victorialogs/promtail/docker-compose.yml @@ -11,7 +11,6 @@ services: ports: - "5140:5140" - # Run `make package-victoria-logs` to build victoria-logs image vlogs: image: docker.io/victoriametrics/victoria-logs:v0.3.0-victorialogs volumes: diff --git a/deployment/docker/victorialogs/vector-docker/docker-compose.yml b/deployment/docker/victorialogs/vector-docker/docker-compose.yml index 39b636093..d4677c2b7 100644 --- a/deployment/docker/victorialogs/vector-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/vector-docker/docker-compose.yml @@ -21,7 +21,6 @@ services: victoriametrics: condition: service_healthy - # Run `make package-victoria-logs` to build victoria-logs image victorialogs: image: docker.io/victoriametrics/victoria-logs:v0.3.0-victorialogs volumes: