From 8271094c98e463e07cb4af68302b9fb62099e47b Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Thu, 28 Nov 2024 14:25:25 +0200 Subject: [PATCH] deployment/docker: use vector for local setup (#7681) --- deployment/docker/README.md | 16 +++++--- .../docker/docker-compose-victorialogs.yml | 19 +++++---- deployment/docker/fluent-bit.conf | 33 --------------- deployment/docker/vector.yaml | 41 +++++++++++++++++++ .../victorialogs/vector/compose-base.yml | 2 +- .../victorialogs/vector/datadog/vector.yaml | 28 +++++++++---- .../vector/elasticsearch/vector.yaml | 18 +++----- .../vector/jsonline-ha/vector.yaml | 22 +++++----- .../victorialogs/vector/jsonline/vector.yaml | 11 +++-- .../victorialogs/vector/loki/vector.yaml | 30 +++++++++----- 10 files changed, 126 insertions(+), 94 deletions(-) delete mode 100644 deployment/docker/fluent-bit.conf create mode 100644 deployment/docker/vector.yaml diff --git a/deployment/docker/README.md b/deployment/docker/README.md index 56110fd0e..b55ead020 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -182,8 +182,8 @@ make docker-victorialogs-up VictoriaLogs will be accessible on the `--httpListenAddr=:9428` port. In addition to VictoriaLogs server, the docker compose contains the following componetns: -* [fluentbit](https://docs.fluentbit.io/manual) service for collecting docker logs and sending them to VictoriaLogs; -* VictoriaMetrics single server to collect metrics from `VictoriaLogs` and `fluentbit`; +* [vector](https://vector.dev/guides/) service for collecting docker logs and sending them to VictoriaLogs; +* VictoriaMetrics single server to collect metrics from `VictoriaLogs` and `vector`; * [grafana](#grafana) is configured with [VictoriaLogs datasource](https://github.com/VictoriaMetrics/victorialogs-datasource). To access Grafana use link [http://localhost:3000](http://localhost:3000). @@ -200,9 +200,13 @@ make docker-victorialogs-down ``` Please see more examples on integration of VictoriaLogs with other log shippers below: -* [filebeat-docker](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/filebeat-docker) -* [filebeat-syslog](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/filebeat-syslog) -* [fluentbit-docker](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentbit-docker) +* [filebeat](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/filebeat) +* [fluentbit](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentbit) * [logstash](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/logstash) * [promtail](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/promtail) -* [vector-docker](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector-docker) +* [vector](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector) +* [datadog-agent](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/datadog-agent) +* [journald](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/journald) +* [opentelemetry-collector](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/opentelemetry-collector) +* [telegraf](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/telegraf) +* [fluentd]((https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentd) diff --git a/deployment/docker/docker-compose-victorialogs.yml b/deployment/docker/docker-compose-victorialogs.yml index 4cedfdb54..81ff4baeb 100644 --- a/deployment/docker/docker-compose-victorialogs.yml +++ b/deployment/docker/docker-compose-victorialogs.yml @@ -22,17 +22,22 @@ services: - vm_net restart: always - # fluentbit is logs collector. It collects logs according to fluent-bit.conf + # vector is logs collector. It collects logs according to vector.yaml # and forwards them to VictoriaLogs - fluentbit: - container_name: fluentbit - image: fluent/fluent-bit:2.1.4 + vector: + image: docker.io/timberio/vector:0.42.X-distroless-libc volumes: - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + - type: bind + source: /var/lib/docker + target: /var/lib/docker + - ./vector.yaml:/etc/vector/vector.yaml:ro depends_on: [victorialogs] ports: - - "5140:5140" + - "8686:8686" + user: root networks: - vm_net diff --git a/deployment/docker/fluent-bit.conf b/deployment/docker/fluent-bit.conf deleted file mode 100644 index 44e3d9429..000000000 --- a/deployment/docker/fluent-bit.conf +++ /dev/null @@ -1,33 +0,0 @@ -[INPUT] - name tail - path /var/lib/docker/containers/**/*.log - path_key path - multiline.parser docker, cri - Parser docker - Docker_Mode On - -[INPUT] - Name syslog - Listen 0.0.0.0 - Port 5140 - Parser syslog-rfc3164 - Mode tcp - -[SERVICE] - Flush 1 - Parsers_File parsers.conf - HTTP_Server On - HTTP_Listen 0.0.0.0 - HTTP_PORT 2020 - -[Output] - Name http - Match * - host victorialogs - port 9428 - compress gzip - uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date - format json_lines - json_date_format iso8601 - header AccountID 0 - header ProjectID 0 diff --git a/deployment/docker/vector.yaml b/deployment/docker/vector.yaml new file mode 100644 index 000000000..adb6d1af7 --- /dev/null +++ b/deployment/docker/vector.yaml @@ -0,0 +1,41 @@ +api: + enabled: true + address: 0.0.0.0:8686 +sources: + docker: + type: docker_logs + demo: + type: demo_logs + format: apache_common + interval: 10 + vector_metrics: + type: internal_metrics +transforms: + msg_parser: + type: remap + inputs: [docker] + source: | + .message = parse_json(.message) ?? .message +sinks: + elasticsearch: + type: elasticsearch + inputs: [demo, msg_parser] + endpoints: [http://victorialogs:9428/insert/elasticsearch/] + mode: bulk + api_version: v8 + compression: gzip + healthcheck: + enabled: false + request: + headers: + VL-Stream-Fields: source_type,label.com.docker.compose.service + VL-Time-Field: timestamp + VL-Msg-Field: message,log + AccountID: "0" + ProjectID: "0" + victoriametrics: + type: prometheus_remote_write + endpoint: http://victoriametrics:8428/api/v1/write + inputs: [vector_metrics] + healthcheck: + enabled: false diff --git a/deployment/docker/victorialogs/vector/compose-base.yml b/deployment/docker/victorialogs/vector/compose-base.yml index bc035bef7..f04d982bf 100644 --- a/deployment/docker/victorialogs/vector/compose-base.yml +++ b/deployment/docker/victorialogs/vector/compose-base.yml @@ -2,7 +2,7 @@ include: - ../compose-base.yml services: vector: - image: docker.io/timberio/vector:0.40.0-distroless-static + image: docker.io/timberio/vector:0.42.X-distroless-libc restart: on-failure volumes: - type: bind diff --git a/deployment/docker/victorialogs/vector/datadog/vector.yaml b/deployment/docker/victorialogs/vector/datadog/vector.yaml index cf6c112c8..4cde7e796 100644 --- a/deployment/docker/victorialogs/vector/datadog/vector.yaml +++ b/deployment/docker/victorialogs/vector/datadog/vector.yaml @@ -2,27 +2,39 @@ api: enabled: true address: 0.0.0.0:8686 sources: - vector_metrics: + docker: + type: docker_logs + metrics: type: internal_metrics - demo: - type: demo_logs - format: json +transforms: + parser: + type: remap + inputs: + - docker + source: | + .ddsource = .source_type + ddtags = [] + for_each(object!(.label)) -> |key, value| { + ddtags = append(ddtags, [to_string(key) + ":" + to_string!(value)]) + } + .ddtags = join!(ddtags, ",") + del(.source_type) + del(.label) sinks: datadog: type: datadog_logs - inputs: [demo] + inputs: [parser] default_api_key: test endpoint: http://dd-logs:8427 compression: gzip request: headers: - dd-protocol: test # required by VictoriaLogs AccountID: "0" ProjectID: "0" - VL-Stream-Fields: "service,host" + VL-Stream-Fields: "ddsource" victoriametrics: type: prometheus_remote_write endpoint: http://victoriametrics:8428/api/v1/write - inputs: [vector_metrics] + inputs: [metrics] healthcheck: enabled: false diff --git a/deployment/docker/victorialogs/vector/elasticsearch/vector.yaml b/deployment/docker/victorialogs/vector/elasticsearch/vector.yaml index 520a7a9bd..8ee84c489 100644 --- a/deployment/docker/victorialogs/vector/elasticsearch/vector.yaml +++ b/deployment/docker/victorialogs/vector/elasticsearch/vector.yaml @@ -4,37 +4,31 @@ api: sources: docker: type: docker_logs - demo: - type: demo_logs - format: apache_common - interval: 10 vector_metrics: type: internal_metrics transforms: - msg_parser: + parser: type: remap inputs: [docker] source: | - .log = parse_json!(.message) - del(.message) + .message = parse_json(.message) ?? .message sinks: elasticsearch: type: elasticsearch - inputs: [demo] + inputs: [parser] endpoints: [http://victorialogs:9428/insert/elasticsearch/] mode: bulk api_version: v8 compression: gzip healthcheck: enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: source_type request: headers: AccountID: "0" ProjectID: "0" + VL-Stream-Fields: source_type,label.com.docker.compose.service + VL-Time-Field: timestamp + VL-Msg-Field: message.msg,message victoriametrics: type: prometheus_remote_write endpoint: http://victoriametrics:8428/api/v1/write diff --git a/deployment/docker/victorialogs/vector/jsonline-ha/vector.yaml b/deployment/docker/victorialogs/vector/jsonline-ha/vector.yaml index 331a4f05a..ffb8c06e3 100644 --- a/deployment/docker/victorialogs/vector/jsonline-ha/vector.yaml +++ b/deployment/docker/victorialogs/vector/jsonline-ha/vector.yaml @@ -4,25 +4,21 @@ api: sources: docker: type: docker_logs - demo: - type: demo_logs - format: json metrics: type: internal_metrics transforms: - msg_parser: + parser: type: remap inputs: - docker source: | - .log = parse_json!(.message) - del(.message) + .message = parse_json(.message) ?? .message sinks: vlogs-1: type: http inputs: - - msg_parser - uri: http://victorialogs:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp + - parser + uri: http://victorialogs:9428/insert/jsonline encoding: codec: json framing: @@ -34,11 +30,14 @@ sinks: headers: AccountID: '0' ProjectID: '0' + VL-Stream-Fields: source_type,host,container_name,label.com.docker.compose.service + VL-Msg-Field: message.msg + VL-Time-Field: timestamp vlogs-2: type: http inputs: - - msg_parser - uri: http://victorialogs-2:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp + - parser + uri: http://victorialogs-2:9428/insert/jsonline encoding: codec: json framing: @@ -50,6 +49,9 @@ sinks: headers: AccountID: '0' ProjectID: '0' + VL-Stream-Fields: source_type,host,container_name,label.com.docker.compose.service + VL-Msg-Field: message.msg + VL-Time-Field: timestamp victoriametrics: type: prometheus_remote_write endpoint: http://victoriametrics:8428/api/v1/write diff --git a/deployment/docker/victorialogs/vector/jsonline/vector.yaml b/deployment/docker/victorialogs/vector/jsonline/vector.yaml index 3c09a818f..ae2eb400d 100644 --- a/deployment/docker/victorialogs/vector/jsonline/vector.yaml +++ b/deployment/docker/victorialogs/vector/jsonline/vector.yaml @@ -4,9 +4,6 @@ api: sources: docker: type: docker_logs - demo: - type: demo_logs - format: json metrics: type: internal_metrics transforms: @@ -15,14 +12,13 @@ transforms: inputs: - docker source: | - .log = parse_json!(.message) - del(.message) + .message = parse_json(.message) ?? .message sinks: http: type: http inputs: - msg_parser - uri: http://victorialogs:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp + uri: http://victorialogs:9428/insert/jsonline encoding: codec: json framing: @@ -34,6 +30,9 @@ sinks: headers: AccountID: '0' ProjectID: '0' + VL-Stream-Fields: source_type,host,container_name,label.com.docker.compose.service + VL-Msg-Field: message.msg + VL-Time-Field: timestamp victoriametrics: type: prometheus_remote_write endpoint: http://victoriametrics:8428/api/v1/write diff --git a/deployment/docker/victorialogs/vector/loki/vector.yaml b/deployment/docker/victorialogs/vector/loki/vector.yaml index 3c8911386..5a4f09ae8 100644 --- a/deployment/docker/victorialogs/vector/loki/vector.yaml +++ b/deployment/docker/victorialogs/vector/loki/vector.yaml @@ -4,31 +4,39 @@ api: sources: docker: type: docker_logs - demo: - type: demo_logs - format: json metrics: type: internal_metrics transforms: - msg_parser: + parser: type: remap inputs: - docker source: | - .log = parse_json!(.message) - del(.message) + message, err = parse_json(.message) + if (err == null) { + if exists(message.msg) { + message = message.msg + } else if exists(message.message) { + message = message.message + } else { + message = message + } + } else { + message = .message + } + .message = message sinks: loki: type: loki inputs: - - demo + - parser endpoint: http://victorialogs:9428/insert/loki/ compression: gzip - path: /api/v1/push?_msg_field=message.message&_time_field=timestamp&_stream_fields=source - encoding: - codec: json + path: /api/v1/push?_msg_field=message.msg&_time_field=timestamp&_stream_fields=com_docker_compose_service labels: - source: vector + "*": "{{ label }}" + encoding: + codec: raw_message victoriametrics: type: prometheus_remote_write endpoint: http://victoriametrics:8428/api/v1/write