deployment/docker: use vector for local setup (#7681)

deployment/docker/README.md

@@ -182,8 +182,8 @@ make docker-victorialogs-up
 
 VictoriaLogs will be accessible on the `--httpListenAddr=:9428` port.
 In addition to VictoriaLogs server, the docker compose contains the following componetns:
-* [fluentbit](https://docs.fluentbit.io/manual) service for collecting docker logs and sending them to VictoriaLogs;
+* [vector](https://vector.dev/guides/) service for collecting docker logs and sending them to VictoriaLogs;
-* VictoriaMetrics single server to collect metrics from `VictoriaLogs` and `fluentbit`;
+* VictoriaMetrics single server to collect metrics from `VictoriaLogs` and `vector`;
 * [grafana](#grafana) is configured with [VictoriaLogs datasource](https://github.com/VictoriaMetrics/victorialogs-datasource).
 
 To access Grafana use link [http://localhost:3000](http://localhost:3000).
@@ -200,9 +200,13 @@ make docker-victorialogs-down
 ```
 
 Please see more examples on integration of VictoriaLogs with other log shippers below:
-* [filebeat-docker](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/filebeat-docker) 
+* [filebeat](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/filebeat) 
-* [filebeat-syslog](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/filebeat-syslog) 
+* [fluentbit](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentbit) 
-* [fluentbit-docker](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentbit-docker) 
 * [logstash](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/logstash) 
 * [promtail](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/promtail) 
-* [vector-docker](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector-docker)
+* [vector](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector)
+* [datadog-agent](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/datadog-agent)
+* [journald](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/journald)
+* [opentelemetry-collector](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/opentelemetry-collector)
+* [telegraf](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/telegraf)
+* [fluentd]((https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentd)

deployment/docker/docker-compose-victorialogs.yml

@@ -22,17 +22,22 @@ services:
       - vm_net
     restart: always
 
-  # fluentbit is logs collector. It collects logs according to fluent-bit.conf
+  # vector is logs collector. It collects logs according to vector.yaml
   # and forwards them to VictoriaLogs
-  fluentbit:
+  vector:
-    container_name: fluentbit
+    image: docker.io/timberio/vector:0.42.X-distroless-libc
-    image: fluent/fluent-bit:2.1.4
     volumes:
-      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - type: bind
-      - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+      - type: bind
+        source: /var/lib/docker
+        target: /var/lib/docker
+      - ./vector.yaml:/etc/vector/vector.yaml:ro
     depends_on: [victorialogs]
     ports:
-      - "5140:5140"
+      - "8686:8686"
+    user: root
     networks:
       - vm_net
 

deployment/docker/fluent-bit.conf

@@ -1,33 +0,0 @@
-[INPUT]
-    name              tail
-    path              /var/lib/docker/containers/**/*.log
-    path_key         path
-    multiline.parser  docker, cri
-    Parser docker
-    Docker_Mode  On
-
-[INPUT]
-    Name     syslog
-    Listen   0.0.0.0
-    Port     5140
-    Parser   syslog-rfc3164
-    Mode     tcp
-
-[SERVICE]
-    Flush        1
-    Parsers_File parsers.conf
-    HTTP_Server  On
-    HTTP_Listen  0.0.0.0
-    HTTP_PORT    2020
-
-[Output]
-    Name http
-    Match *
-    host victorialogs
-    port 9428
-    compress gzip
-    uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date
-    format json_lines
-    json_date_format iso8601
-    header AccountID 0
-    header ProjectID 0

deployment/docker/vector.yaml

@@ -0,0 +1,41 @@
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+sources:
+  docker:
+    type: docker_logs
+  demo:
+    type: demo_logs
+    format: apache_common
+    interval: 10
+  vector_metrics:
+    type: internal_metrics
+transforms:
+  msg_parser:
+    type: remap
+    inputs: [docker]
+    source: |
+      .message = parse_json(.message) ?? .message
+sinks:
+  elasticsearch:
+    type: elasticsearch
+    inputs: [demo, msg_parser]
+    endpoints: [http://victorialogs:9428/insert/elasticsearch/]
+    mode: bulk
+    api_version: v8
+    compression: gzip
+    healthcheck:
+      enabled: false
+    request:
+      headers:
+        VL-Stream-Fields: source_type,label.com.docker.compose.service
+        VL-Time-Field: timestamp
+        VL-Msg-Field: message,log
+        AccountID: "0"
+        ProjectID: "0"
+  victoriametrics:
+    type: prometheus_remote_write
+    endpoint: http://victoriametrics:8428/api/v1/write
+    inputs: [vector_metrics]
+    healthcheck:
+      enabled: false

deployment/docker/victorialogs/vector/compose-base.yml

@@ -2,7 +2,7 @@ include:
  - ../compose-base.yml
 services:
   vector:
-    image: docker.io/timberio/vector:0.40.0-distroless-static
+    image: docker.io/timberio/vector:0.42.X-distroless-libc
     restart: on-failure
     volumes:
       - type: bind

deployment/docker/victorialogs/vector/datadog/vector.yaml

@@ -2,27 +2,39 @@ api:
   enabled: true
   address: 0.0.0.0:8686
 sources:
-  vector_metrics:
+  docker:
+    type: docker_logs
+  metrics:
     type: internal_metrics
-  demo:
+transforms:
-    type: demo_logs
+  parser:
-    format: json
+    type: remap
+    inputs:
+      - docker
+    source: |
+      .ddsource = .source_type
+      ddtags = []
+      for_each(object!(.label)) -> |key, value| {
+        ddtags = append(ddtags, [to_string(key) + ":" + to_string!(value)])
+      }
+      .ddtags = join!(ddtags, ",")
+      del(.source_type)
+      del(.label)
 sinks:
   datadog:
     type: datadog_logs
-    inputs: [demo]
+    inputs: [parser]
     default_api_key: test
     endpoint: http://dd-logs:8427
     compression: gzip
     request:
       headers:
-        dd-protocol: test  # required by VictoriaLogs
         AccountID: "0"
         ProjectID: "0"
-        VL-Stream-Fields: "service,host"
+        VL-Stream-Fields: "ddsource"
   victoriametrics:
     type: prometheus_remote_write
     endpoint: http://victoriametrics:8428/api/v1/write
-    inputs: [vector_metrics]
+    inputs: [metrics]
     healthcheck:
       enabled: false

deployment/docker/victorialogs/vector/elasticsearch/vector.yaml

@@ -4,37 +4,31 @@ api:
 sources:
   docker:
     type: docker_logs
-  demo:
-    type: demo_logs
-    format: apache_common
-    interval: 10
   vector_metrics:
     type: internal_metrics
 transforms:
-  msg_parser:
+  parser:
     type: remap
     inputs: [docker]
     source: |
-      .log = parse_json!(.message)
+      .message = parse_json(.message) ?? .message
-      del(.message)
 sinks:
   elasticsearch:
     type: elasticsearch
-    inputs: [demo]
+    inputs: [parser]
     endpoints: [http://victorialogs:9428/insert/elasticsearch/]
     mode: bulk
     api_version: v8
     compression: gzip
     healthcheck:
       enabled: false
-    query:
-      _msg_field: message
-      _time_field: timestamp
-      _stream_fields: source_type
     request:
       headers:
         AccountID: "0"
         ProjectID: "0"
+        VL-Stream-Fields: source_type,label.com.docker.compose.service
+        VL-Time-Field: timestamp
+        VL-Msg-Field: message.msg,message
   victoriametrics:
     type: prometheus_remote_write
     endpoint: http://victoriametrics:8428/api/v1/write

deployment/docker/victorialogs/vector/jsonline-ha/vector.yaml

@@ -4,25 +4,21 @@ api:
 sources:
   docker:
     type: docker_logs
-  demo:
-    type: demo_logs
-    format: json
   metrics:
     type: internal_metrics
 transforms:
-  msg_parser:
+  parser:
     type: remap
     inputs:
       - docker
     source: |
-      .log = parse_json!(.message)
+      .message = parse_json(.message) ?? .message
-      del(.message)
 sinks:
   vlogs-1:
     type: http
     inputs:
-      - msg_parser
+      - parser
-    uri: http://victorialogs:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp
+    uri: http://victorialogs:9428/insert/jsonline
     encoding:
       codec: json
     framing:
@@ -34,11 +30,14 @@ sinks:
       headers:
         AccountID: '0'
         ProjectID: '0'
+        VL-Stream-Fields: source_type,host,container_name,label.com.docker.compose.service
+        VL-Msg-Field: message.msg
+        VL-Time-Field: timestamp
   vlogs-2:
     type: http
     inputs:
-      - msg_parser
+      - parser
-    uri: http://victorialogs-2:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp
+    uri: http://victorialogs-2:9428/insert/jsonline
     encoding:
       codec: json
     framing:
@@ -50,6 +49,9 @@ sinks:
       headers:
         AccountID: '0'
         ProjectID: '0'
+        VL-Stream-Fields: source_type,host,container_name,label.com.docker.compose.service
+        VL-Msg-Field: message.msg
+        VL-Time-Field: timestamp
   victoriametrics:
     type: prometheus_remote_write
     endpoint: http://victoriametrics:8428/api/v1/write

deployment/docker/victorialogs/vector/jsonline/vector.yaml

@@ -4,9 +4,6 @@ api:
 sources:
   docker:
     type: docker_logs
-  demo:
-    type: demo_logs
-    format: json
   metrics:
     type: internal_metrics
 transforms:
@@ -15,14 +12,13 @@ transforms:
     inputs:
       - docker
     source: |
-      .log = parse_json!(.message)
+      .message = parse_json(.message) ?? .message
-      del(.message)
 sinks:
   http:
     type: http
     inputs:
       - msg_parser
-    uri: http://victorialogs:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp
+    uri: http://victorialogs:9428/insert/jsonline
     encoding:
       codec: json
     framing:
@@ -34,6 +30,9 @@ sinks:
       headers:
         AccountID: '0'
         ProjectID: '0'
+        VL-Stream-Fields: source_type,host,container_name,label.com.docker.compose.service
+        VL-Msg-Field: message.msg
+        VL-Time-Field: timestamp
   victoriametrics:
     type: prometheus_remote_write
     endpoint: http://victoriametrics:8428/api/v1/write

deployment/docker/victorialogs/vector/loki/vector.yaml

@@ -4,31 +4,39 @@ api:
 sources:
   docker:
     type: docker_logs
-  demo:
-    type: demo_logs
-    format: json
   metrics:
     type: internal_metrics
 transforms:
-  msg_parser:
+  parser:
     type: remap
     inputs:
       - docker
     source: |
-      .log = parse_json!(.message)
+      message, err = parse_json(.message) 
-      del(.message)
+      if (err == null) {
+        if exists(message.msg) {
+          message = message.msg
+        } else if exists(message.message) {
+          message = message.message
+        } else {
+          message = message
+        }
+      } else {
+        message = .message
+      }
+      .message = message
 sinks:
   loki:
     type: loki
     inputs:
-      - demo
+      - parser
     endpoint: http://victorialogs:9428/insert/loki/
     compression: gzip
-    path: /api/v1/push?_msg_field=message.message&_time_field=timestamp&_stream_fields=source
+    path: /api/v1/push?_msg_field=message.msg&_time_field=timestamp&_stream_fields=com_docker_compose_service
-    encoding:
-      codec: json
     labels:
-      source: vector
+      "*": "{{ label }}"
+    encoding:
+      codec: raw_message
   victoriametrics:
     type: prometheus_remote_write
     endpoint: http://victoriametrics:8428/api/v1/write