From 05ac9618c0d781de7da50c9d8856f239d4a2142b Mon Sep 17 00:00:00 2001
From: Alexander Marshalov <_@marshalov.org>
Date: Wed, 21 Jun 2023 12:51:38 +0200
Subject: [PATCH] improved vector example for victoria logs

Signed-off-by: Alexander Marshalov <_@marshalov.org>
---
 .../victorialogs/vector-docker/README.md      | 40 +++++-----
 .../vector-docker/docker-compose.yml          |  2 +-
 .../victorialogs/vector-docker/vector.toml    | 74 ++++++++++++++++---
 3 files changed, 84 insertions(+), 32 deletions(-)

diff --git a/deployment/docker/victorialogs/vector-docker/README.md b/deployment/docker/victorialogs/vector-docker/README.md
index c8902675a6..092786982c 100644
--- a/deployment/docker/victorialogs/vector-docker/README.md
+++ b/deployment/docker/victorialogs/vector-docker/README.md
@@ -22,34 +22,34 @@ The docker compose file contains the following components:
 the example of vector configuration(`vector.toml`)
 
 ```
-[api]
-  enabled = true
-  address = "0.0.0.0:8686"
-
-  [sources.docker]
+[sources.docker]
   type = "docker_logs"
 
-  [sinks.vlogs]
+[transforms.msg_parser]
+  type = "remap"
+  inputs = ["docker"]
+  source = '''
+  .log = parse_json!(.message)
+  del(.message)
+  '''
+
+[sinks.vlogs]
   type = "elasticsearch"
-  inputs = [ "docker" ]
+  inputs = [ "msg_parser" ]
   endpoints = [ "http://victorialogs:9428/insert/elasticsearch/" ]
-  id_key = "id"
   mode = "bulk"
+  api_version = "v8"
+  compression = "gzip"
   healthcheck.enabled = false
 
   [sinks.vlogs.query]
-  _msg_field = "message"
-  _time_field = "timestamp"
-  _stream_fields = "host,container_name"
+    _msg_field = "log.msg"
+    _time_field = "timestamp"
+    _stream_fields = "source_type,host,container_name"
 
-  [sources.vector_metrics]
-  type = "internal_metrics"
-
-  [sinks.victoriametrics]
-  type = "prometheus_remote_write"
-  endpoint = "http://victoriametrics:8428/api/v1/write"
-  inputs = ["vector_metrics"]
-  healthcheck.enabled = false
+  [sinks.vlogs.request.headers]
+    AccountID = "0"
+    ProjectID = "0"
 ```
 
-Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance.
\ No newline at end of file
+Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance.
diff --git a/deployment/docker/victorialogs/vector-docker/docker-compose.yml b/deployment/docker/victorialogs/vector-docker/docker-compose.yml
index 30ff3a708e..922022772f 100644
--- a/deployment/docker/victorialogs/vector-docker/docker-compose.yml
+++ b/deployment/docker/victorialogs/vector-docker/docker-compose.yml
@@ -28,7 +28,7 @@ services:
 
   # Run `make package-victoria-logs` to build victoria-logs image
   victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:heads-master-0-g88993f312-dirty-e01fb71f
+    image: docker.io/victoriametrics/victoria-logs:heads-examples-vl-docker-compose-0-gdefae2a3f-dirty-de6e4df1
     volumes:
       - victorialogs-vector-docker-vl:/vlogs
     ports:
diff --git a/deployment/docker/victorialogs/vector-docker/vector.toml b/deployment/docker/victorialogs/vector-docker/vector.toml
index 0661647f46..b693611d61 100644
--- a/deployment/docker/victorialogs/vector-docker/vector.toml
+++ b/deployment/docker/victorialogs/vector-docker/vector.toml
@@ -1,28 +1,80 @@
 [api]
-enabled = true
-address = "0.0.0.0:8686"
+  enabled = true
+  address = "0.0.0.0:8686"
+
+
+# ---------------------------------------------
+# Docker logs -> VictoriaLogs
+# ---------------------------------------------
 
 [sources.docker]
-type = "docker_logs"
+  type = "docker_logs"
+
+[transforms.msg_parser]
+  type = "remap"
+  inputs = ["docker"]
+  source = '''
+  .log = parse_json!(.message)
+  del(.message)
+  '''
 
 [sinks.vlogs]
   type = "elasticsearch"
-  inputs = [ "docker" ]
+  inputs = [ "msg_parser" ]
   endpoints = [ "http://victorialogs:9428/insert/elasticsearch/" ]
-  id_key = "id"
   mode = "bulk"
+  api_version = "v8"
+  compression = "gzip"
   healthcheck.enabled = false
 
   [sinks.vlogs.query]
+    _msg_field = "log.msg"
+    _time_field = "timestamp"
+    _stream_fields = "source_type,host,container_name"
+
+  [sinks.vlogs.request.headers]
+    AccountID = "0"
+    ProjectID = "0"
+
+
+# ---------------------------------------------
+# Generted demo logs -> VictoriaLogs
+# ---------------------------------------------
+
+[sources.demo]
+  type = "demo_logs"
+  format = "apache_common"
+  interval = 10
+
+[sinks.vlogs_demo]
+  type = "elasticsearch"
+  inputs = [ "demo" ]
+  endpoints = [ "http://victorialogs:9428/insert/elasticsearch/" ]
+  mode = "bulk"
+  api_version = "v8"
+  compression = "gzip"
+  healthcheck.enabled = false
+
+  [sinks.vlogs_demo.query]
     _msg_field = "message"
     _time_field = "timestamp"
-    _stream_fields = "host,container_name"
+    _stream_fields = "source_type"
+
+  [sinks.vlogs_demo.request.headers]
+    AccountID = "0"
+    ProjectID = "0"
+
+
+
+# ---------------------------------------------
+# Vector Metrics -> VictoriaMetrics
+# ---------------------------------------------
 
 [sources.vector_metrics]
-type = "internal_metrics"
+  type = "internal_metrics"
 
 [sinks.victoriametrics]
-type = "prometheus_remote_write"
-endpoint = "http://victoriametrics:8428/api/v1/write"
-inputs = ["vector_metrics"]
-healthcheck.enabled = false
\ No newline at end of file
+  type = "prometheus_remote_write"
+  endpoint = "http://victoriametrics:8428/api/v1/write"
+  inputs = ["vector_metrics"]
+  healthcheck.enabled = false