diff --git a/deployment/docker/victorialogs/filebeat-docker/README.md b/deployment/docker/victorialogs/filebeat-docker/README.md index 4fefc5a5d..9d00353ea 100644 --- a/deployment/docker/victorialogs/filebeat-docker/README.md +++ b/deployment/docker/victorialogs/filebeat-docker/README.md @@ -1,8 +1,7 @@ -# Docker compose Filebeat integration with VictoriaLogs +# Docker compose Filebeat integration with VictoriaLogs for docker The folder contains the example of integration of [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html) with Victorialogs - To spin-up environment run the following command: ``` docker compose up -d @@ -10,13 +9,42 @@ docker compose up -d To shut down the docker-compose environment run the following command: ``` -docker compose down -v +docker compose down +docker compose rm -f ``` -The docker compose file contains the following component +The docker compose file contains the following components: -* filebeat - fileabeat is configured to collect logs from the docker, you can find configuration in the `filebeat.yml`. It writes data in VictoriaLogs +* filebeat - fileabeat is configured to collect logs from the `docker`, you can find configuration in the `filebeat.yml`. It writes data in VictoriaLogs * filebeat-exporter - it export metrics about the filebeat -* VictoriaLogs - the log database, it accept data from `filebeat` by elastic protocol -* VictoriaMetrics - collect metrics from `filebeat` via `filebeat-exporter`, `VictoriaLogs` and `VictoriaMetrics`(itself) -* grafana - it comes with two predefined dashboards for `VictoriaLogs` and `VictoriaMetrics` \ No newline at end of file +* VictoriaLogs - the log database, it accepts the data from `filebeat` by elastic protocol +* VictoriaMetrics - collects metrics from `filebeat` via `filebeat-exporter`, `VictoriaLogs` and `VictoriaMetrics`(itself) +* grafana - it comes with two predefined dashboards for `VictoriaLogs` and `VictoriaMetrics` + + +the example of filebeat configuration(`filebeat.yml`) + +```yaml +filebeat.autodiscover: + providers: + - type: docker + hints.enabled: true + +processors: + - add_docker_metadata: ~ + +output.elasticsearch: + hosts: [ "http://victorialogs:9428/insert/elasticsearch/" ] + worker: 5 + parameters: + _msg_field: "message" + _time_field: "@timestamp" + _stream_fields: "container.name" + +http: + enabled: true + host: 0.0.0.0 + port: 5066 +``` + +Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance. \ No newline at end of file diff --git a/deployment/docker/victorialogs/filebeat-syslog/README.md b/deployment/docker/victorialogs/filebeat-syslog/README.md new file mode 100644 index 000000000..3761c53fa --- /dev/null +++ b/deployment/docker/victorialogs/filebeat-syslog/README.md @@ -0,0 +1,40 @@ +# Docker compose Filebeat integration with VictoriaLogs for syslog + +The folder contains the example of integration of [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html) with Victorialogs + +To spin-up environment run the following command: +``` +docker compose up -d +``` + +To shut down the docker-compose environment run the following command: +``` +docker compose down +docker compose rm -f +``` + +The docker compose file contains the following components: + +* filebeat - fileabeat is configured to accept `syslog` logs in `rfc3164` format on `5140` port, you can find configuration in the `filebeat.yml`. It writes data in VictoriaLogs +* VictoriaLogs - the log database, it accepts the data from `filebeat` by elastic protocol + +the example of filebeat configuration(`filebeat.yml`) + +```yaml +filebeat.inputs: + - type: syslog + format: rfc3164 + protocol.tcp: + host: "0.0.0.0:5140" + +output.elasticsearch: + hosts: [ "http://victorialogs:9428/insert/elasticsearch/" ] + worker: 5 + bulk_max_size: 1000 + parameters: + _msg_field: "message" + _time_field: "@timestamp" + _stream_fields: "host.name,process.program,process.pid,container.name" +``` + +Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance. \ No newline at end of file diff --git a/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml b/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml index 355ba04ef..db79dfeee 100644 --- a/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml +++ b/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml @@ -14,7 +14,7 @@ services: # Run `make package-victoria-logs` to build victoria-logs image victorialogs: - image: docker.io/victoriametrics/victoria-logs:heads-logs-examples-0-g66b42a677-dirty-b95f1d4b + image: docker.io/victoriametrics/victoria-logs:heads-master-0-g88993f312-dirty-e01fb71f volumes: - victorialogs-filebeat-syslog-vl:/vlogs ports: diff --git a/deployment/docker/victorialogs/fluentbit-docker/README.md b/deployment/docker/victorialogs/fluentbit-docker/README.md new file mode 100644 index 000000000..05755295c --- /dev/null +++ b/deployment/docker/victorialogs/fluentbit-docker/README.md @@ -0,0 +1,55 @@ +# Docker compose Fluentbit integration with VictoriaLogs for docker + +The folder contains the example of integration of [fluentbit](https://docs.fluentbit.io/manual) with Victorialogs + +To spin-up environment run the following command: +``` +docker compose up -d +``` + +To shut down the docker-compose environment run the following command: +``` +docker compose down +docker compose rm -f +``` + +The docker compose file contains the following components: + +* fluentbit - fluentbit is configured to collect logs from the `docker`, you can find configuration in the `fluent-bit.conf`. It writes data in VictoriaLogs +* VictoriaLogs - the log database, it accepts the data from `fluentbit` by json line protocol + +the example of fluentbit configuration(`filebeat.yml`) + +```shell +[INPUT] + name tail + path /var/lib/docker/containers/**/*.log + multiline.parser docker, cri + Parser docker + Docker_Mode On + +[INPUT] + Name syslog + Listen 0.0.0.0 + Port 5140 + Parser syslog-rfc3164 + Mode tcp + +[SERVICE] + Flush 1 + Parsers_File parsers.conf + +[Output] + Name http + Match * + host victorialogs + port 9428 + compress gzip + uri /insert/jsonline/?_stream_fields=stream&_msg_field=log&_time_field=date + format json_lines + json_date_format iso8601 + header AccountID 0 + header ProjectID 0 +``` + +Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance. \ No newline at end of file diff --git a/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml b/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml index bf53a6fd3..55022f848 100644 --- a/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml @@ -12,7 +12,7 @@ services: # Run `make package-victoria-logs` to build victoria-logs image victorialogs: - image: docker.io/victoriametrics/victoria-logs:heads-logs-examples-0-g66b42a677-dirty-b95f1d4b + image: docker.io/victoriametrics/victoria-logs:heads-master-0-g88993f312-dirty-e01fb71f volumes: - victorialogs-fluentbit-vl:/vlogs ports: diff --git a/deployment/docker/victorialogs/logstash/README.md b/deployment/docker/victorialogs/logstash/README.md index f3263bf97..d4e19bdaa 100644 --- a/deployment/docker/victorialogs/logstash/README.md +++ b/deployment/docker/victorialogs/logstash/README.md @@ -1,4 +1,4 @@ -# How to set up sending logs to VictoriaLogs from logstash +# Docker compose Logstash integration with VictoriaLogs for syslog It is required to use [OpenSearch plugin](https://github.com/opensearch-project/logstash-output-opensearch) for output configuration. Plugin can be installed by using the following command: @@ -7,9 +7,32 @@ bin/logstash-plugin install logstash-output-opensearch ``` OpenSearch plugin is required because elasticsearch output plugin performs various checks for Elasticsearch version and license which are not applicable for VictoriaLogs. -Here is an example of logstash configuration: +To spin-up environment run the following command: +``` +docker compose up -d +``` + +To shut down the docker-compose environment run the following command: +``` +docker compose down +docker compose rm -f +``` + +The docker compose file contains the following components: + +* logstash - logstash is configured to accept `syslog` on `5140` port, you can find configuration in the `pipeline.conf`. It writes data in VictoriaLogs +* VictoriaLogs - the log database, it accepts the data from `logstash` by elastic protocol + + +Here is an example of logstash configuration(`pipeline.conf`): ``` +input { + syslog { + port => 5140 + } +} +output { opensearch { hosts => ["http://victorialogs:9428/insert/elasticsearch"] custom_headers => { @@ -22,6 +45,7 @@ Here is an example of logstash configuration: "_time_field" => "@timestamp" } } +} ``` Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance. \ No newline at end of file diff --git a/deployment/docker/victorialogs/logstash/docker-compose.yml b/deployment/docker/victorialogs/logstash/docker-compose.yml index 25c32ee40..1ff406901 100644 --- a/deployment/docker/victorialogs/logstash/docker-compose.yml +++ b/deployment/docker/victorialogs/logstash/docker-compose.yml @@ -15,7 +15,7 @@ services: # Run `make package-victoria-logs` to build victoria-logs image victorialogs: - image: docker.io/victoriametrics/victoria-logs:heads-logs-examples-0-g66b42a677-dirty-b95f1d4b + image: docker.io/victoriametrics/victoria-logs:heads-master-0-g88993f312-dirty-e01fb71f volumes: - victorialogs-logstash-vl:/vlogs ports: diff --git a/deployment/docker/victorialogs/vector-docker/README.md b/deployment/docker/victorialogs/vector-docker/README.md new file mode 100644 index 000000000..d5dc63e2d --- /dev/null +++ b/deployment/docker/victorialogs/vector-docker/README.md @@ -0,0 +1,57 @@ +# Docker compose Vector integration with VictoriaLogs for docker + +The folder contains the example of integration of [vector](https://vector.dev/docs/) with Victorialogs + +To spin-up environment run the following command: +``` +docker compose up -d +``` + +To shut down the docker-compose environment run the following command: +``` +docker compose down +docker compose rm -f +``` + +The docker compose file contains the following components: + +* vector - vector is configured to collect logs from the `docker`, you can find configuration in the `vector.toml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics. +* VictoriaLogs - the log database, it accepts the data from `vector` by elastic protocol +* VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics`(itself) +* grafana - it comes with two predefined dashboards for `VictoriaLogs` and `VictoriaMetrics` + + +the example of vector configuration(`vector.toml`) + +``` +[api] + enabled = true + address = "0.0.0.0:8686" + + [sources.docker] + type = "docker_logs" + + [sinks.vlogs] + type = "elasticsearch" + inputs = [ "docker" ] + endpoints = [ "http://victorialogs:9428/insert/elasticsearch/" ] + id_key = "id" + mode = "bulk" + healthcheck.enabled = false + + [sinks.vlogs.query] + _msg_field = "message" + _time_field = "timestamp" + _stream_fields = "host,container_name" + + [sources.vector_metrics] + type = "internal_metrics" + + [sinks.victoriametrics] + type = "prometheus_remote_write" + endpoint = "http://victoriametrics:8428/api/v1/write" + inputs = ["vector_metrics"] + healthcheck.enabled = false +``` + +Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance. \ No newline at end of file diff --git a/deployment/docker/victorialogs/vector-docker/docker-compose.yml b/deployment/docker/victorialogs/vector-docker/docker-compose.yml index 5a16963ce..30ff3a708 100644 --- a/deployment/docker/victorialogs/vector-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/vector-docker/docker-compose.yml @@ -28,7 +28,7 @@ services: # Run `make package-victoria-logs` to build victoria-logs image victorialogs: - image: docker.io/victoriametrics/victoria-logs:heads-docker-logs-0-g4dbdb3039 + image: docker.io/victoriametrics/victoria-logs:heads-master-0-g88993f312-dirty-e01fb71f volumes: - victorialogs-vector-docker-vl:/vlogs ports: