From d5fe4566e5bbba7b74cacd270faea13eeffef0e1 Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Tue, 3 Sep 2024 18:43:26 +0300 Subject: [PATCH] app/vlinsert: support getting _msg_field, _time_field, _stream_fields and _ignore_fields from headers * Many collectors don't support forwarding url query params to the remote system. It makes impossible to define stream fields for it. Workaround with proxy between VictoriaLogs and log shipper is too complicated solution. * This commit adds the following changes: * Adds fallback to to headers params, if query param is empty for: _msg_field -> VL-Msg-Field _stream_fields -> VL-Stream-Fields _ignore_fields -> VL-Ignore-Fields _time_field -> VL-Time-Field * removes deprecations from victorialogs compose files, added more output format examples for logstash, telegraf, fluent-bit related issue: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5310 --- app/vlinsert/insertutils/common_params.go | 19 +- .../{docker-compose.yml => compose.yml} | 2 - .../{filebeat-docker => filebeat}/README.md | 0 .../compose.yml} | 2 - .../filebeat.yml | 0 .../{filebeat-docker => filebeat}/scrape.yml | 0 .../fluentbit-docker/fluent-bit.conf | 30 -- .../{fluentbit-docker => fluentbit}/README.md | 0 .../compose.yml} | 4 +- .../victorialogs/fluentbit/fluent-bit.conf | 54 ++++ .../docker/victorialogs/logstash/Dockerfile | 4 +- .../{docker-compose.yml => compose.yml} | 2 - .../victorialogs/logstash/pipeline.conf | 10 +- .../{docker-compose.yml => compose.yml} | 2 - .../victorialogs/telegraf-docker/README.md | 25 ++ .../victorialogs/telegraf-docker/compose.yml | 55 ++++ .../scrape.yml | 0 .../telegraf-docker/telegraf.conf | 70 +++++ .../victorialogs/vector-docker/vector.toml | 75 ----- .../{vector-docker => vector}/README.md | 62 +++-- .../docker-compose.yml => vector/compose.yml} | 10 +- .../docker/victorialogs/vector/scrape.yml | 11 + .../docker/victorialogs/vector/vector.yaml | 73 +++++ docs/VictoriaLogs/data-ingestion/Fluentbit.md | 45 +++ docs/VictoriaLogs/data-ingestion/Logstash.md | 33 +++ docs/VictoriaLogs/data-ingestion/README.md | 26 +- docs/VictoriaLogs/data-ingestion/Telegraf.md | 123 +++++++++ docs/VictoriaLogs/data-ingestion/Vector.md | 259 +++++++++++------- 28 files changed, 731 insertions(+), 265 deletions(-) rename deployment/docker/victorialogs/filebeat-syslog/{docker-compose.yml => compose.yml} (97%) rename deployment/docker/victorialogs/{filebeat-docker => filebeat}/README.md (100%) rename deployment/docker/victorialogs/{filebeat-docker/docker-compose.yml => filebeat/compose.yml} (99%) rename deployment/docker/victorialogs/{filebeat-docker => filebeat}/filebeat.yml (100%) rename deployment/docker/victorialogs/{filebeat-docker => filebeat}/scrape.yml (100%) delete mode 100644 deployment/docker/victorialogs/fluentbit-docker/fluent-bit.conf rename deployment/docker/victorialogs/{fluentbit-docker => fluentbit}/README.md (100%) rename deployment/docker/victorialogs/{fluentbit-docker/docker-compose.yml => fluentbit/compose.yml} (88%) create mode 100644 deployment/docker/victorialogs/fluentbit/fluent-bit.conf rename deployment/docker/victorialogs/logstash/{docker-compose.yml => compose.yml} (97%) rename deployment/docker/victorialogs/promtail/{docker-compose.yml => compose.yml} (97%) create mode 100644 deployment/docker/victorialogs/telegraf-docker/README.md create mode 100644 deployment/docker/victorialogs/telegraf-docker/compose.yml rename deployment/docker/victorialogs/{vector-docker => telegraf-docker}/scrape.yml (100%) create mode 100644 deployment/docker/victorialogs/telegraf-docker/telegraf.conf delete mode 100644 deployment/docker/victorialogs/vector-docker/vector.toml rename deployment/docker/victorialogs/{vector-docker => vector}/README.md (62%) rename deployment/docker/victorialogs/{vector-docker/docker-compose.yml => vector/compose.yml} (82%) create mode 100644 deployment/docker/victorialogs/vector/scrape.yml create mode 100644 deployment/docker/victorialogs/vector/vector.yaml create mode 100644 docs/VictoriaLogs/data-ingestion/Telegraf.md diff --git a/app/vlinsert/insertutils/common_params.go b/app/vlinsert/insertutils/common_params.go index 8092ffa520..91c4dbcf9a 100644 --- a/app/vlinsert/insertutils/common_params.go +++ b/app/vlinsert/insertutils/common_params.go @@ -38,20 +38,34 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) { return nil, err } - // Extract time field name from _time_field query arg + // Extract time field name from _time_field query arg or header var timeField = "_time" if tf := r.FormValue("_time_field"); tf != "" { timeField = tf + } else if tf = r.Header.Get("VL-Time-Field"); tf != "" { + timeField = tf } - // Extract message field name from _msg_field query arg + // Extract message field name from _msg_field query arg or header var msgField = "" if msgf := r.FormValue("_msg_field"); msgf != "" { msgField = msgf + } else if msgf = r.Header.Get("VL-Msg-Field"); msgf != "" { + msgField = msgf } streamFields := httputils.GetArray(r, "_stream_fields") + if len(streamFields) == 0 { + if sf := r.Header.Values("VL-Stream-Fields"); len(sf) > 0 { + streamFields = sf + } + } ignoreFields := httputils.GetArray(r, "ignore_fields") + if len(ignoreFields) == 0 { + if f := r.Header.Values("VL-Ignore-Fields"); len(f) > 0 { + ignoreFields = f + } + } debug := httputils.GetBool(r, "debug") debugRequestURI := "" @@ -71,6 +85,7 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) { DebugRequestURI: debugRequestURI, DebugRemoteAddr: debugRemoteAddr, } + return cp, nil } diff --git a/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml b/deployment/docker/victorialogs/filebeat-syslog/compose.yml similarity index 97% rename from deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml rename to deployment/docker/victorialogs/filebeat-syslog/compose.yml index 6a22010dd9..df833f560d 100644 --- a/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml +++ b/deployment/docker/victorialogs/filebeat-syslog/compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: filebeat-vlogs: image: docker.elastic.co/beats/filebeat:8.8.1 diff --git a/deployment/docker/victorialogs/filebeat-docker/README.md b/deployment/docker/victorialogs/filebeat/README.md similarity index 100% rename from deployment/docker/victorialogs/filebeat-docker/README.md rename to deployment/docker/victorialogs/filebeat/README.md diff --git a/deployment/docker/victorialogs/filebeat-docker/docker-compose.yml b/deployment/docker/victorialogs/filebeat/compose.yml similarity index 99% rename from deployment/docker/victorialogs/filebeat-docker/docker-compose.yml rename to deployment/docker/victorialogs/filebeat/compose.yml index becb693f40..4545da3c9d 100644 --- a/deployment/docker/victorialogs/filebeat-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/filebeat/compose.yml @@ -1,5 +1,3 @@ -version: '3' - services: filebeat-victorialogs: image: docker.elastic.co/beats/filebeat:8.8.1 diff --git a/deployment/docker/victorialogs/filebeat-docker/filebeat.yml b/deployment/docker/victorialogs/filebeat/filebeat.yml similarity index 100% rename from deployment/docker/victorialogs/filebeat-docker/filebeat.yml rename to deployment/docker/victorialogs/filebeat/filebeat.yml diff --git a/deployment/docker/victorialogs/filebeat-docker/scrape.yml b/deployment/docker/victorialogs/filebeat/scrape.yml similarity index 100% rename from deployment/docker/victorialogs/filebeat-docker/scrape.yml rename to deployment/docker/victorialogs/filebeat/scrape.yml diff --git a/deployment/docker/victorialogs/fluentbit-docker/fluent-bit.conf b/deployment/docker/victorialogs/fluentbit-docker/fluent-bit.conf deleted file mode 100644 index 42d349338b..0000000000 --- a/deployment/docker/victorialogs/fluentbit-docker/fluent-bit.conf +++ /dev/null @@ -1,30 +0,0 @@ -[INPUT] - name tail - path /var/lib/docker/containers/**/*.log - path_key path - multiline.parser docker, cri - Parser docker - Docker_Mode On - -[INPUT] - Name syslog - Listen 0.0.0.0 - Port 5140 - Parser syslog-rfc3164 - Mode tcp - -[SERVICE] - Flush 1 - Parsers_File parsers.conf - -[Output] - Name http - Match * - host victorialogs - port 9428 - compress gzip - uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date - format json_lines - json_date_format iso8601 - header AccountID 0 - header ProjectID 0 diff --git a/deployment/docker/victorialogs/fluentbit-docker/README.md b/deployment/docker/victorialogs/fluentbit/README.md similarity index 100% rename from deployment/docker/victorialogs/fluentbit-docker/README.md rename to deployment/docker/victorialogs/fluentbit/README.md diff --git a/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml b/deployment/docker/victorialogs/fluentbit/compose.yml similarity index 88% rename from deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml rename to deployment/docker/victorialogs/fluentbit/compose.yml index bdd6c25d29..808877c608 100644 --- a/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/fluentbit/compose.yml @@ -1,8 +1,6 @@ -version: "3" - services: fluentbit: - image: cr.fluentbit.io/fluent/fluent-bit:3.0.2 + image: cr.fluentbit.io/fluent/fluent-bit:3.0.7 volumes: - /var/lib/docker/containers:/var/lib/docker/containers:ro - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf diff --git a/deployment/docker/victorialogs/fluentbit/fluent-bit.conf b/deployment/docker/victorialogs/fluentbit/fluent-bit.conf new file mode 100644 index 0000000000..0c78d93646 --- /dev/null +++ b/deployment/docker/victorialogs/fluentbit/fluent-bit.conf @@ -0,0 +1,54 @@ +[INPUT] + name tail + path /var/lib/docker/containers/**/*.log + path_key path + multiline.parser docker, cri + Parser docker + Docker_Mode On + +[INPUT] + Name syslog + Listen 0.0.0.0 + Port 5140 + Parser syslog-rfc3164 + Mode tcp + +[SERVICE] + Flush 1 + Parsers_File parsers.conf + +[OUTPUT] + Name http + Match * + host victorialogs + port 9428 + compress gzip + uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date + format json_lines + json_date_format iso8601 + header AccountID 0 + header ProjectID 0 + +[OUTPUT] + Name es + Match * + host victorialogs + port 9428 + compress gzip + path /insert/elasticsearch + header AccountID 0 + header ProjectID 0 + header VL-Stream-Fields path + header VL-Msg-Field log + header VL-Time-Field @timestamp + +[OUTPUT] + name loki + match * + host victorialogs + uri /insert/loki/api/v1/push + port 9428 + label_keys $path,$log,$time + header VL-Msg-Field log + header VL-Time-Field time + header VL-Stream-Fields path diff --git a/deployment/docker/victorialogs/logstash/Dockerfile b/deployment/docker/victorialogs/logstash/Dockerfile index 0b25920052..a6e809fd16 100644 --- a/deployment/docker/victorialogs/logstash/Dockerfile +++ b/deployment/docker/victorialogs/logstash/Dockerfile @@ -1,3 +1,5 @@ FROM docker.elastic.co/logstash/logstash:8.8.1 -RUN bin/logstash-plugin install logstash-output-opensearch \ No newline at end of file +RUN bin/logstash-plugin install \ + logstash-output-opensearch \ + logstash-output-loki diff --git a/deployment/docker/victorialogs/logstash/docker-compose.yml b/deployment/docker/victorialogs/logstash/compose.yml similarity index 97% rename from deployment/docker/victorialogs/logstash/docker-compose.yml rename to deployment/docker/victorialogs/logstash/compose.yml index dc112c1e39..9f101a97cf 100644 --- a/deployment/docker/victorialogs/logstash/docker-compose.yml +++ b/deployment/docker/victorialogs/logstash/compose.yml @@ -1,5 +1,3 @@ -version: '3' - services: logstash: build: diff --git a/deployment/docker/victorialogs/logstash/pipeline.conf b/deployment/docker/victorialogs/logstash/pipeline.conf index 8b62c68192..ab6cd901de 100644 --- a/deployment/docker/victorialogs/logstash/pipeline.conf +++ b/deployment/docker/victorialogs/logstash/pipeline.conf @@ -17,4 +17,12 @@ output { "_time_field" => "@timestamp" } } -} \ No newline at end of file + http { + url => "http://victorialogs:9428/insert/jsonline?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp" + format => "json" + http_method => "post" + } + loki { + url => "http://victorialogs:9428/insert/loki/api/v1/push?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp" + } +} diff --git a/deployment/docker/victorialogs/promtail/docker-compose.yml b/deployment/docker/victorialogs/promtail/compose.yml similarity index 97% rename from deployment/docker/victorialogs/promtail/docker-compose.yml rename to deployment/docker/victorialogs/promtail/compose.yml index 5d4665c128..bf2553dbc6 100644 --- a/deployment/docker/victorialogs/promtail/docker-compose.yml +++ b/deployment/docker/victorialogs/promtail/compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: promtail: image: grafana/promtail:2.8.2 diff --git a/deployment/docker/victorialogs/telegraf-docker/README.md b/deployment/docker/victorialogs/telegraf-docker/README.md new file mode 100644 index 0000000000..d460ba9fcd --- /dev/null +++ b/deployment/docker/victorialogs/telegraf-docker/README.md @@ -0,0 +1,25 @@ +# Docker compose Telegraf integration with VictoriaLogs for docker + +The folder contains the example of integration of [telegraf](https://www.influxdata.com/time-series-platform/telegraf/) with VictoriaLogs + +To spin-up environment run the following command: +``` +docker compose up -d +``` + +To shut down the docker-compose environment run the following command: +``` +docker compose down +docker compose rm -f +``` + +The docker compose file contains the following components: + +* telegraf - telegraf is configured to collect logs from the `docker`, you can find configuration in the `telegraf.conf`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics. +* VictoriaLogs - the log database, it accepts the data from `telegraf` by elastic protocol +* VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics` + +Querying the data + +* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` +* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) diff --git a/deployment/docker/victorialogs/telegraf-docker/compose.yml b/deployment/docker/victorialogs/telegraf-docker/compose.yml new file mode 100644 index 0000000000..616aa6e270 --- /dev/null +++ b/deployment/docker/victorialogs/telegraf-docker/compose.yml @@ -0,0 +1,55 @@ +services: + telegraf: + image: bitnami/telegraf:1.31.0 + restart: on-failure + volumes: + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + - type: bind + source: /var/lib/docker + target: /var/lib/docker + - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro + command: --config /etc/telegraf/telegraf.conf + depends_on: + victorialogs: + condition: service_healthy + victoriametrics: + condition: service_healthy + + victorialogs: + image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs + volumes: + - victorialogs-vector-docker-vl:/vlogs + ports: + - '9428:9428' + command: + - -storageDataPath=/vlogs + - -loggerFormat=json + - -syslog.listenAddr.tcp=0.0.0.0:8094 + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"] + interval: 1s + timeout: 1s + retries: 10 + + victoriametrics: + image: victoriametrics/victoria-metrics:latest + ports: + - '8428:8428' + command: + - -storageDataPath=/vmsingle + - -promscrape.config=/promscrape.yml + - -loggerFormat=json + volumes: + - victorialogs-vector-docker-vm:/vmsingle + - ./scrape.yml:/promscrape.yml + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"] + interval: 1s + timeout: 1s + retries: 10 + +volumes: + victorialogs-vector-docker-vl: + victorialogs-vector-docker-vm: diff --git a/deployment/docker/victorialogs/vector-docker/scrape.yml b/deployment/docker/victorialogs/telegraf-docker/scrape.yml similarity index 100% rename from deployment/docker/victorialogs/vector-docker/scrape.yml rename to deployment/docker/victorialogs/telegraf-docker/scrape.yml diff --git a/deployment/docker/victorialogs/telegraf-docker/telegraf.conf b/deployment/docker/victorialogs/telegraf-docker/telegraf.conf new file mode 100644 index 0000000000..3eafb51b58 --- /dev/null +++ b/deployment/docker/victorialogs/telegraf-docker/telegraf.conf @@ -0,0 +1,70 @@ +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 100000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + logtarget = "file" + logfile = "/dev/null" + hostname = "pop-os" + omit_hostname = false + +[[inputs.cpu]] + +[[outputs.http]] + url = "http://victorialogs:9428/insert/jsonline?_msg_field=fields.msg&_stream_fields=tags.log_source,tags.metric_type" + data_format = "json" + namepass = ["docker_log"] + use_batch_format = false + +[[outputs.loki]] + domain = "http://victorialogs:9428" + endpoint = "/insert/loki/api/v1/push?_msg_field=docker_log.msg&_time_field=@timestamp&_stream_fields=log_source,metric_type" + namepass = ["docker_log"] + gzip_request = true + sanitize_label_names = true + +[[outputs.syslog]] + address = "tcp://victorialogs:8094" + namepass = ["docker_log"] + +[[outputs.elasticsearch]] + urls = ["http://victorialogs:9428/insert/elasticsearch"] + timeout = "1m" + flush_interval = "30s" + enable_sniffer = false + health_check_interval = "0s" + index_name = "device_log-%Y.%m.%d" + manage_template = false + template_name = "telegraf" + overwrite_template = false + namepass = ["docker_log"] + [outputs.elasticsearch.headers] + VL-Msg-Field = "docker_log.msg" + VL-Time-Field = "@timestamp" + VL-Stream-Fields = "tag.log_source,tag.metric_type" + +[[outputs.http]] + url = "http://victoriametrics:8428/api/v1/write" + data_format = "prometheusremotewrite" + namepass = ["cpu"] + [outputs.http.headers] + Content-Type = "application/x-protobuf" + Content-Encoding = "snappy" + X-Prometheus-Remote-Write-Version = "0.1.0" + +[[inputs.docker_log]] + [inputs.docker_log.tags] + metric_type = "logs" + log_source = "telegraf" + +[[processors.rename]] + namepass = ["docker_log"] + [[processors.rename.replace]] + field = "message" + dest = "msg" diff --git a/deployment/docker/victorialogs/vector-docker/vector.toml b/deployment/docker/victorialogs/vector-docker/vector.toml deleted file mode 100644 index e981bbbd6f..0000000000 --- a/deployment/docker/victorialogs/vector-docker/vector.toml +++ /dev/null @@ -1,75 +0,0 @@ -[api] - enabled = true - address = "0.0.0.0:8686" - - -# --------------------------------------------- -# Docker logs -> VictoriaLogs -# --------------------------------------------- - -[sources.docker] - type = "docker_logs" - -[transforms.msg_parser] - type = "remap" - inputs = ["docker"] - source = ''' - .log = parse_json!(.message) - del(.message) - ''' - -[sinks.vlogs] - type = "http" - inputs = [ "msg_parser" ] - uri = "http://victorialogs:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp" - encoding.codec = "json" - framing.method = "newline_delimited" - compression = "gzip" - healthcheck.enabled = false - - [sinks.vlogs.request.headers] - AccountID = "0" - ProjectID = "0" - - -# --------------------------------------------- -# Generted demo logs -> VictoriaLogs -# --------------------------------------------- - -[sources.demo] - type = "demo_logs" - format = "apache_common" - interval = 10 - -[sinks.vlogs_demo] - type = "elasticsearch" - inputs = [ "demo" ] - endpoints = [ "http://victorialogs:9428/insert/elasticsearch/" ] - mode = "bulk" - api_version = "v8" - compression = "gzip" - healthcheck.enabled = false - - [sinks.vlogs_demo.query] - _msg_field = "message" - _time_field = "timestamp" - _stream_fields = "source_type" - - [sinks.vlogs_demo.request.headers] - AccountID = "0" - ProjectID = "0" - - - -# --------------------------------------------- -# Vector Metrics -> VictoriaMetrics -# --------------------------------------------- - -[sources.vector_metrics] - type = "internal_metrics" - -[sinks.victoriametrics] - type = "prometheus_remote_write" - endpoint = "http://victoriametrics:8428/api/v1/write" - inputs = ["vector_metrics"] - healthcheck.enabled = false diff --git a/deployment/docker/victorialogs/vector-docker/README.md b/deployment/docker/victorialogs/vector/README.md similarity index 62% rename from deployment/docker/victorialogs/vector-docker/README.md rename to deployment/docker/victorialogs/vector/README.md index 9d739f3557..4acd31d361 100644 --- a/deployment/docker/victorialogs/vector-docker/README.md +++ b/deployment/docker/victorialogs/vector/README.md @@ -15,7 +15,7 @@ docker compose rm -f The docker compose file contains the following components: -* vector - vector is configured to collect logs from the `docker`, you can find configuration in the `vector.toml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics. +* vector - vector is configured to collect logs from the `docker`, you can find configuration in the `vector.yaml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics. * VictoriaLogs - the log database, it accepts the data from `vector` by elastic protocol * VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics` @@ -25,37 +25,39 @@ Querying the data * for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) -the example of vector configuration(`vector.toml`) +the example of vector configuration(`vector.yaml`) ``` -[sources.docker] - type = "docker_logs" - -[transforms.msg_parser] - type = "remap" - inputs = ["docker"] - source = ''' - .log = parse_json!(.message) - del(.message) - ''' - -[sinks.vlogs] - type = "elasticsearch" - inputs = [ "msg_parser" ] - endpoints = [ "http://victorialogs:9428/insert/elasticsearch/" ] - mode = "bulk" - api_version = "v8" - compression = "gzip" - healthcheck.enabled = false - - [sinks.vlogs.query] - _msg_field = "log.msg" - _time_field = "timestamp" - _stream_fields = "source_type,host,container_name" - - [sinks.vlogs.request.headers] - AccountID = "0" - ProjectID = "0" +sources: + docker: + type: docker_logs +transforms: + msg_parser: + type: remap + inputs: + - docker + source: | + .log = parse_json!(.message) + del(.message) +sinks: + vlogs_es: + type: elasticsearch + inputs: + - msg_parser + endpoints: + - http://victorialogs:9428/insert/elasticsearch/ + mode: bulk + api_version: v8 + compression: gzip + healthcheck.enabled: false + query: + _msg_field: log.msg + _time_field: timestamp + _stream_fields: source_type,host,container_name + request: + headers: + AccountID: "0" + ProjectID: "0" ``` Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/vector-docker/docker-compose.yml b/deployment/docker/victorialogs/vector/compose.yml similarity index 82% rename from deployment/docker/victorialogs/vector-docker/docker-compose.yml rename to deployment/docker/victorialogs/vector/compose.yml index ae0691eac3..3a5e848916 100644 --- a/deployment/docker/victorialogs/vector-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/vector/compose.yml @@ -1,8 +1,6 @@ -version: '3' - services: vector: - image: docker.io/timberio/vector:0.30.0-distroless-static + image: docker.io/timberio/vector:0.38.0-distroless-static restart: on-failure volumes: - type: bind @@ -11,7 +9,7 @@ services: - type: bind source: /var/lib/docker target: /var/lib/docker - - ./vector.toml:/etc/vector/vector.toml:ro + - ./vector.yaml:/etc/vector/vector.yaml:ro user: root ports: - '8686:8686' @@ -31,7 +29,7 @@ services: - -storageDataPath=/vlogs - -loggerFormat=json healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:9428/health"] + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"] interval: 1s timeout: 1s retries: 10 @@ -48,7 +46,7 @@ services: - victorialogs-vector-docker-vm:/vmsingle - ./scrape.yml:/promscrape.yml healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:8428/health"] + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"] interval: 1s timeout: 1s retries: 10 diff --git a/deployment/docker/victorialogs/vector/scrape.yml b/deployment/docker/victorialogs/vector/scrape.yml new file mode 100644 index 0000000000..8257db8ef0 --- /dev/null +++ b/deployment/docker/victorialogs/vector/scrape.yml @@ -0,0 +1,11 @@ +scrape_configs: + - job_name: "victoriametrics" + scrape_interval: 30s + static_configs: + - targets: + - victoriametrics:8428 + - job_name: "victorialogs" + scrape_interval: 30s + static_configs: + - targets: + - victorialogs:9428 \ No newline at end of file diff --git a/deployment/docker/victorialogs/vector/vector.yaml b/deployment/docker/victorialogs/vector/vector.yaml new file mode 100644 index 0000000000..b409823473 --- /dev/null +++ b/deployment/docker/victorialogs/vector/vector.yaml @@ -0,0 +1,73 @@ +api: + enabled: true + address: 0.0.0.0:8686 +sources: + docker: + type: docker_logs + demo: + type: demo_logs + format: json + metrics: + type: internal_metrics +transforms: + msg_parser: + type: remap + inputs: + - docker + source: | + .log = parse_json!(.message) + del(.message) +sinks: + vlogs_http: + type: http + inputs: + - msg_parser + uri: http://victorialogs:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp + encoding: + codec: json + framing: + method: newline_delimited + compression: gzip + healthcheck: + enabled: false + request: + headers: + AccountID: '0' + ProjectID: '0' + vlogs_loki: + type: loki + inputs: + - demo + endpoint: http://victorialogs:9428/insert/loki/ + compression: gzip + path: /api/v1/push?_msg_field=message.message&_time_field=timestamp&_stream_fields=source + encoding: + codec: json + labels: + source: vector + victoriametrics: + type: prometheus_remote_write + endpoint: http://victoriametrics:8428/api/v1/write + inputs: + - metrics + healthcheck: + enabled: false + vlogs_es: + type: elasticsearch + inputs: + - demo + endpoints: + - http://victorialogs:9428/insert/elasticsearch/ + mode: bulk + api_version: v8 + compression: gzip + healthcheck: + enabled: false + query: + _msg_field: message + _time_field: timestamp + _stream_fields: source_type + request: + headers: + AccountID: '0' + ProjectID: '0' diff --git a/docs/VictoriaLogs/data-ingestion/Fluentbit.md b/docs/VictoriaLogs/data-ingestion/Fluentbit.md index e126134c75..1ac1250727 100644 --- a/docs/VictoriaLogs/data-ingestion/Fluentbit.md +++ b/docs/VictoriaLogs/data-ingestion/Fluentbit.md @@ -14,6 +14,51 @@ aliases: # Fluentbit setup +VictoriaLogs supports given below Fluentbit outputs: +- [Elasticsearch](#elasticsearch) +- [Loki](#loki) +- [HTTP JSON](#http) + +## Elasticsearch + +Specify [elasticsearch output](https://docs.fluentbit.io/manual/pipeline/outputs/elasticsearch) section in the `fluentbit.conf` +for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): + +```conf +[Output] + Name es + Match * + host victorialogs + port 9428 + compress gzip + path /insert/elasticsearch + header AccountID 0 + header ProjectID 0 + header VL-Stream-Fields path + header VL-Msg-Field log + header VL-Time-Field @timestamp +``` + +## Loki + +Specify [loki output](https://docs.fluentbit.io/manual/pipeline/outputs/loki) section in the `fluentbit.conf` +for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): + +```conf +[OUTPUT] + name loki + match * + host victorialogs + uri /insert/loki/api/v1/push + port 9428 + label_keys $path,$log,$time + header VL-Msg-Field log + header VL-Time-Field time + header VL-Stream-Fields path +``` + +## HTTP + Specify [http output](https://docs.fluentbit.io/manual/pipeline/outputs/http) section in the `fluentbit.conf` for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): diff --git a/docs/VictoriaLogs/data-ingestion/Logstash.md b/docs/VictoriaLogs/data-ingestion/Logstash.md index d9d07bd671..181e98561e 100644 --- a/docs/VictoriaLogs/data-ingestion/Logstash.md +++ b/docs/VictoriaLogs/data-ingestion/Logstash.md @@ -11,6 +11,13 @@ aliases: - /victorialogs/data-ingestion/logstash.html - /victorialogs/data-ingestion/Logstash.html --- +VictoriaLogs supports given below Logstash outputs: +- [Elasticsearch](#elasticsearch) +- [Loki](#loki) +- [HTTP JSON](#http) + +## Elasticsearch + Specify [`output.elasticsearch`](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) section in the `logstash.conf` file for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): @@ -106,6 +113,32 @@ output { } ``` +## Loki + +Specify [`output.loki`](https://grafana.com/docs/loki/latest/send-data/logstash/) section in the `logstash.conf` file +for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): + +```conf +output { + loki { + url => "http://victorialogs:9428/insert/loki/api/v1/push?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp" + } +} +``` + +## HTTP + +Specify [`output.http`](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-http.html) section in the `logstash.conf` file +for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): + +```conf +output { + url => "http://victorialogs:9428/insert/jsonline?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp" + format => "json" + http_method => "post" +} +``` + See also: - [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting). diff --git a/docs/VictoriaLogs/data-ingestion/README.md b/docs/VictoriaLogs/data-ingestion/README.md index 5ec89c74e0..08e0cf24ce 100644 --- a/docs/VictoriaLogs/data-ingestion/README.md +++ b/docs/VictoriaLogs/data-ingestion/README.md @@ -6,6 +6,7 @@ - Logstash - see [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/logstash/). - Vector - see [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/vector/). - Promtail (aka Grafana Loki) - see [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/promtail/). +- Telegraf - see [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/telegraf/). The ingested logs can be queried according to [these docs](https://docs.victoriametrics.com/victorialogs/querying/). @@ -198,6 +199,24 @@ See also [HTTP headers](#http-headers). VictoriaLogs accepts optional `AccountID` and `ProjectID` headers at [data ingestion HTTP APIs](#http-apis). These headers may contain the needed tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details. +- `VL-Msg-Field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) + with the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) generated by the log shipper. + This is usually the `message` field for Filebeat and Logstash. + If the `VL-Msg-Field` header isn't set, then VictoriaLogs reads the log message from the `_msg` field. + +- `VL-Time-Field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) + with the [log timestamp](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field) generated by the log shipper. + This is usually the `@timestamp` field for Filebeat and Logstash. + If the `VL-Time-Field` header isn't set, then VictoriaLogs reads the timestamp from the `_time` field. + If this field doesn't exist, then the current timestamp is used. + +- `VL-Stream-Fields` - it should contain comma-separated list of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names, + which uniquely identify every [log stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) collected the log shipper. + If the `VL-Stream-Fields` header isn't set, then all the ingested logs are written to default log stream - `{}`. + +- `VL-Ignore-Fields` - this parameter may contain the list of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names, + which must be ignored during data ingestion. + ## Troubleshooting The following command can be used for verifying whether the data is successfully ingested into VictoriaLogs: @@ -246,7 +265,8 @@ Here is the list of log collectors and their ingestion formats supported by Vict | [Rsyslog](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/) | [Yes](https://www.rsyslog.com/doc/configuration/modules/omelasticsearch.html) | No | No | [Yes](https://www.rsyslog.com/doc/configuration/modules/omfwd.html) | | [Syslog-ng](https://docs.victoriametrics.com/victorialogs/data-ingestion/filebeat/) | Yes, [v1](https://support.oneidentity.com/technical-documents/syslog-ng-open-source-edition/3.16/administration-guide/28#TOPIC-956489), [v2](https://support.oneidentity.com/technical-documents/doc/syslog-ng-open-source-edition/3.16/administration-guide/29#TOPIC-956494) | No | No | [Yes](https://support.oneidentity.com/technical-documents/doc/syslog-ng-open-source-edition/3.16/administration-guide/44#TOPIC-956553) | | [Filebeat](https://docs.victoriametrics.com/victorialogs/data-ingestion/filebeat/) | [Yes](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html) | No | No | No | -| [Fluentbit](https://docs.victoriametrics.com/victorialogs/data-ingestion/fluentbit/) | No | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/http) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/loki) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/syslog) | -| [Logstash](https://docs.victoriametrics.com/victorialogs/data-ingestion/logstash/) | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) | No | No | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-syslog.html) | +| [Fluentbit](https://docs.victoriametrics.com/victorialogs/data-ingestion/fluentbit/) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/elasticsearch) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/http) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/loki) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/syslog) | +| [Logstash](https://docs.victoriametrics.com/victorialogs/data-ingestion/logstash/) | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-http.html) | [Yes](https://grafana.com/docs/loki/latest/send-data/logstash/) | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-syslog.html) | | [Vector](https://docs.victoriametrics.com/victorialogs/data-ingestion/vector/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/http/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/loki/) | No | -| [Promtail](https://docs.victoriametrics.com/victorialogs/data-ingestion/promtail/) | No | No | [Yes](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#clients) | No | +| [Promtail](https://docs.victoriametrics.com/victorialogs/data-ingestion/promtail/) | No | No | [Yes](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#clients) | No | +| [Telegraf](https://docs.victoriametrics.com/victorialogs/data-ingestion/telegraf/) | [Yes](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/elasticsearch) | [Yes](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/http) | [Yes](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/loki) | [Yes](https://github.com/influxdata/telegraf/blob/master/plugins/outputs/syslog) | diff --git a/docs/VictoriaLogs/data-ingestion/Telegraf.md b/docs/VictoriaLogs/data-ingestion/Telegraf.md new file mode 100644 index 0000000000..1e8fe76f9b --- /dev/null +++ b/docs/VictoriaLogs/data-ingestion/Telegraf.md @@ -0,0 +1,123 @@ +--- +weight: 5 +title: Telegraf setup +disableToc: true +menu: + docs: + parent: "victorialogs-data-ingestion" + weight: 5 +aliases: + - /VictoriaLogs/data-ingestion/Telegraf.html +--- +# Telegraf setup + +VictoriaLogs supports given below Telegraf outputs: +- [Elasticsearch](#elasticsearch) +- [Loki](#loki) +- [HTTP JSON](#http) + +## Elasticsearch + +Specify [Elasticsearch output](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/elasticsearch) in the `telegraf.toml` +for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): + +```toml +[[outputs.elasticsearch]] + urls = ["http://localhost:9428/insert/elasticsearch"] + timeout = "1m" + flush_interval = "30s" + enable_sniffer = false + health_check_interval = "0s" + index_name = "device_log-%Y.%m.%d" + manage_template = false + template_name = "telegraf" + overwrite_template = false + namepass = ["tail"] + [outputs.elasticsearch.headers] + "VL-Msg-Field" = "tail.value" + "VL-Time-Field" = "@timestamp" + "VL-Stream-Fields" = "tag.log_source,tag.metric_type" + +[[inputs.tail]] + files = ["/tmp/telegraf.log"] + from_beginning = false + interval = "10s" + pipe = false + watch_method = "inotify" + data_format = "value" + data_type = "string" + character_encoding = "utf-8" + [inputs.tail.tags] + metric_type = "logs" + log_source = "telegraf" +``` + + +## Loki + +Specify [Loki output](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/loki) in the `telegraf.toml` +for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): + +```toml +[[outputs.loki]] + domain = "http://localhost:9428" + endpoint = "/insert/loki/api/v1/push&_msg_field=tail.value&_time_field=@timefield&_stream_fields=log_source,metric_type" + namepass = ["tail"] + gzip_request = true + sanitize_label_names = true + +[[inputs.tail]] + files = ["/tmp/telegraf.log"] + from_beginning = false + interval = "10s" + pipe = false + watch_method = "inotify" + data_format = "value" + data_type = "string" + character_encoding = "utf-8" + [inputs.tail.tags] + metric_type = "logs" + log_source = "telegraf" +``` + + +## HTTP + +Specify [HTTP output](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/http) in the `telegraf.toml with batch mode disabled` +for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): + +```toml +[[inputs.tail]] + files = ["/tmp/telegraf.log"] + from_beginning = false + interval = "10s" + pipe = false + watch_method = "inotify" + data_format = "value" + data_type = "string" + character_encoding = "utf-8" + [inputs.tail.tags] + metric_type = "logs" + log_source = "telegraf" + +[[outputs.http]] + url = "http://localhost:9428/insert/jsonline?_msg_field=fields.message&_time_field=timestamp,_stream_fields=tags.log_source,tags.metric_type" + data_format = "json" + namepass = ["docker_log"] + use_batch_format = false +``` + +Substitute the `localhost:9428` address inside `endpoints` section with the real TCP address of VictoriaLogs. + +See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-headers) for details on headers specified +in the `[[output.elasticsearch]]` section. + +It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) +and uses the correct [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields). + +See also: + +- [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting). +- [How to query VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/). +- [Elasticsearch output docs for Telegraf](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/elasticsearch). +- [Docker-compose demo for Telegraf integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/telegraf-docker). diff --git a/docs/VictoriaLogs/data-ingestion/Vector.md b/docs/VictoriaLogs/data-ingestion/Vector.md index 6d52fce507..6a3fb189bb 100644 --- a/docs/VictoriaLogs/data-ingestion/Vector.md +++ b/docs/VictoriaLogs/data-ingestion/Vector.md @@ -11,24 +11,52 @@ aliases: - /victorialogs/data-ingestion/Vector.html - /victorialogs/data-ingestion/vector.html --- -## Elasticsearch sink +VictoriaLogs supports given below Vector sinks: +- [Elasticsearch](#elasticsearch) +- [Loki](#loki) +- [HTTP JSON](#http) -Specify [Elasticsearch sink type](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) in the `vector.toml` +## Elasticsearch + +Specify [Elasticsearch sink type](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) in the `vector.yaml` for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): -```toml -[sinks.vlogs] - inputs = [ "your_input" ] - type = "elasticsearch" - endpoints = [ "http://localhost:9428/insert/elasticsearch/" ] - mode = "bulk" - api_version = "v8" - healthcheck.enabled = false +```yaml +sinks: + vlogs: + inputs: + - your_input + type: elasticsearch + endpoints: + - http://localhost:9428/insert/elasticsearch/ + mode: bulk + api_version: v8 + healthcheck: + enabled: false + query: + _msg_field: message + _time_field: timestamp + _stream_fields: host,container_name +``` - [sinks.vlogs.query] - _msg_field = "message" - _time_field = "timestamp" - _stream_fields = "host,container_name" +## Loki + +Specify [Loki sink type](https://vector.dev/docs/reference/configuration/sinks/loki/) in the `vector.yaml` +for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/): + +```yaml +sinks: + vlogs: + type: "loki" + endpoint = "http://localhost:9428/insert/loki/" + inputs: + - your_input + compression: gzip + path: /api/v1/push?_msg_field=message.message&_time_field=timestamp&_stream_fields=source + encoding: + codec: json + labels: + source: vector ``` Substitute the `localhost:9428` address inside `endpoints` section with the real TCP address of VictoriaLogs. @@ -36,129 +64,148 @@ Substitute the `localhost:9428` address inside `endpoints` section with the real Replace `your_input` with the name of the `inputs` section, which collects logs. See [these docs](https://vector.dev/docs/reference/configuration/sources/) for details. See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on parameters specified -in the `[sinks.vlogs.query]` section. +in the `sinks.vlogs.query` section. It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) and uses the correct [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields). This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) -in the `[sinks.vlogs.query]` section and inspecting VictoriaLogs logs then: +in the `sinks.vlogs.query` section and inspecting VictoriaLogs logs then: -```toml -[sinks.vlogs] - inputs = [ "your_input" ] - type = "elasticsearch" - endpoints = [ "http://localhost:9428/insert/elasticsearch/" ] - mode = "bulk" - api_version = "v8" - healthcheck.enabled = false - - [sinks.vlogs.query] - _msg_field = "message" - _time_field = "timestamp" - _stream_fields = "host,container_name" - debug = "1" +```yaml +sinks: + vlogs: + inputs: + - your_input + type: elasticsearch + endpoints: + - http://localhost:9428/insert/elasticsearch/ + mode: bulk + api_version: v8 + healthcheck: + enabled: false + query: + _msg_field: message + _time_field: timestamp + _stream_fields: host,container_name + debug: "1" ``` If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must be skipped during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters). For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs: -```toml -[sinks.vlogs] - inputs = [ "your_input" ] - type = "elasticsearch" - endpoints = [ "http://localhost:9428/insert/elasticsearch/" ] - mode = "bulk" - api_version = "v8" - healthcheck.enabled = false - - [sinks.vlogs.query] - _msg_field = "message" - _time_field = "timestamp" - _stream_fields = "host,container_name" - ignore_fields = "log.offset,event.original" +```yaml +sinks: + vlogs: + inputs: + - your_input + type: elasticsearch + endpoints: + - http://localhost:9428/insert/elasticsearch/ + mode: bulk + api_version: v8 + healthcheck: + enabled: false + query: + _msg_field: message + _time_field: timestamp + _stream_fields: host,container_name + _ignore_fields: log.offset,event.original ``` When Vector ingests logs into VictoriaLogs at a high rate, then it may be needed to tune `batch.max_events` option. For example, the following config is optimized for higher than usual ingestion rate: -```toml -[sinks.vlogs] - inputs = [ "your_input" ] - type = "elasticsearch" - endpoints = [ "http://localhost:9428/insert/elasticsearch/" ] - mode = "bulk" - api_version = "v8" - healthcheck.enabled = false - - [sinks.vlogs.query] - _msg_field = "message" - _time_field = "timestamp" - _stream_fields = "host,container_name" - - [sinks.vlogs.batch] - max_events = 1000 +```yaml +sinks: + vlogs: + inputs: + - your_input + type: elasticsearch + endpoints: + - http://localhost:9428/insert/elasticsearch/ + mode: bulk + api_version: v8 + healthcheck: + enabled: false + query: + _msg_field: message + _time_field: timestamp + _stream_fields: host,container_name + batch] + max_events: 1000 ``` If the Vector sends logs to VictoriaLogs in another datacenter, then it may be useful enabling data compression via `compression = "gzip"` option. This usually allows saving network bandwidth and costs by up to 5 times: -```toml -[sinks.vlogs] - inputs = [ "your_input" ] - type = "elasticsearch" - endpoints = [ "http://localhost:9428/insert/elasticsearch/" ] - mode = "bulk" - api_version = "v8" - healthcheck.enabled = false - compression = "gzip" - - [sinks.vlogs.query] - _msg_field = "message" - _time_field = "timestamp" - _stream_fields = "host,container_name" +```yaml +sinks: + vlogs: + inputs: + - your_input + type: elasticsearch + endpoints: + - http://localhost:9428/insert/elasticsearch/ + mode: bulk + api_version: v8 + healthcheck: + enabled: false + compression: gzip + query: + _msg_field: message + _time_field: timestamp + _stream_fields: host,container_name ``` By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/victorialogs/keyconcepts/#multitenancy). -If you need storing logs in other tenant, then specify the needed tenant via `[sinks.vlogs.request.headers]` section. -For example, the following `vector.toml` config instructs Vector to store the data to `(AccountID=12, ProjectID=34)` tenant: +If you need storing logs in other tenant, then specify the needed tenant via `sinks.vlogs.request.headers` section. +For example, the following `vector.yaml` config instructs Vector to store the data to `(AccountID=12, ProjectID=34)` tenant: -```toml -[sinks.vlogs] - inputs = [ "your_input" ] - type = "elasticsearch" - endpoints = [ "http://localhost:9428/insert/elasticsearch/" ] - mode = "bulk" - api_version = "v8" - healthcheck.enabled = false - - [sinks.vlogs.query] - _msg_field = "message" - _time_field = "timestamp" - _stream_fields = "host,container_name" - - [sinks.vlogs.request.headers] - AccountID = "12" - ProjectID = "34" +```yaml +sinks: + vlogs: + inputs: + - your_input + type: elasticsearch + endpoints: + - http://localhost:9428/insert/elasticsearch/ + mode: bulk + api_version: v8 + healthcheck: + enabled: false + query: + _msg_field: message + _time_field: timestamp + _stream_fields: host,container_name + request: + headers: + AccountID: "12" + ProjectID: "34" ``` -## HTTP sink +## HTTP Vector can be configured with [HTTP](https://vector.dev/docs/reference/configuration/sinks/http/) sink type for sending data to [JSON stream API](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api): -```toml -[sinks.vlogs] - inputs = [ "your_input" ] - type = "http" - uri = "http://localhost:9428/insert/jsonline?_stream_fields=host,container_name&_msg_field=message&_time_field=timestamp" - encoding.codec = "json" - framing.method = "newline_delimited" - healthcheck.enabled = false - - [sinks.vlogs.request.headers] - AccountID = "12" - ProjectID = "34" +```yaml +sinks: + vlogs: + inputs: + - your_input + type: http + uri: http://localhost:9428/insert/jsonline?_stream_fields=host,container_name&_msg_field=message&_time_field=timestamp + encoding: + codec: json + framing: + method: newline_delimited + healthcheck: + enabled: false + request: + headers: + AccountID: "12" + ProjectID: "34" ``` See also: