From f0b4dd74269f7700139b4bb49a7f43711144b3e7 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 26 Feb 2024 10:33:04 +0100 Subject: [PATCH] deployment: create a separate env for VictoriaLogs (#5857) * deployment: create a separate env for VictoriaLogs The new environment consists of the following components: * VictoriaLogs * fluentbit for collecting logs and sending to VictoriaLogs * VictoriaMetrics for scraping and storing metrics from fluentbit and VictoriaLogs * Grafana with VictoriaLogs datasource for monitoring ----------------- The motivation for creating a separate environment is to simplify existing environments and make it easier to update or modify them in future. Signed-off-by: hagen1778 --- deployment/docker/Makefile | 6 ++ deployment/docker/README.md | 23 ++++-- .../docker/docker-compose-victorialogs.yml | 80 +++++++++++++++++++ deployment/docker/docker-compose.yml | 30 ------- deployment/docker/fluent-bit.conf | 3 + deployment/docker/prometheus-victorialogs.yml | 13 +++ deployment/docker/prometheus.yml | 3 - .../victorialogs-datasource.yml | 12 +++ deployment/docker/victorialogs/download.sh | 22 +++++ deployment/docker/vm-datasource/download.sh | 4 +- docs/CHANGELOG.md | 1 + 11 files changed, 157 insertions(+), 40 deletions(-) create mode 100644 deployment/docker/docker-compose-victorialogs.yml create mode 100644 deployment/docker/prometheus-victorialogs.yml create mode 100644 deployment/docker/provisioning/datasources/victorialogs-datasource/victorialogs-datasource.yml create mode 100755 deployment/docker/victorialogs/download.sh diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index 8ea1c416b..6b44bab75 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -204,3 +204,9 @@ docker-cluster-vm-datasource-up: docker-cluster-vm-datasource-down: $(DOCKER_COMPOSE) -f deployment/docker/docker-compose-cluster.yml -f deployment/docker/vm-datasource/docker-compose-cluster.yml down -v + +docker-victorialogs-up: + $(DOCKER_COMPOSE) -f deployment/docker/docker-compose-victorialogs.yml up -d + +docker-victorialogs-down: + $(DOCKER_COMPOSE) -f deployment/docker/docker-compose-victorialogs.yml down -v \ No newline at end of file diff --git a/deployment/docker/README.md b/deployment/docker/README.md index c186cf09c..06d3c493b 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -169,13 +169,26 @@ VictoriaMetrics installations. ## VictoriaLogs server -VictoriaLogs will be accessible on the following port: `--httpListenAddr=:9428` +To spin-up environment with VictoriaLogs run the following command: +``` +make docker-victorialogs-up +``` -[Fluent Bit](https://docs.fluentbit.io/manual) is used to send logs to VictoriaLogs instance. -Fluent Bit is configured to send logs from running containers to VictoriaLogs instance. -Additionally, it is configured to listen for syslog logs on port `5140` and send them to VictoriaLogs instance. +VictoriaLogs will be accessible on the `--httpListenAddr=:9428` port. +In addition to VictoriaLogs server, the docker compose contains the following componetns: +* [fluentbit](https://docs.fluentbit.io/manual) service for collecting docker logs and sending them to VictoriaLogs; +* VictoriaMetrics single server to collect metrics from `VictoriaLogs` and `fluentbit`; +* [grafana](#grafana) is configured with [VictoriaLogs datasource](https://github.com/VictoriaMetrics/victorialogs-datasource). -To access VictoriaLogs UI use link [http://localhost:9428/select/vmui/](http://localhost:9428/select/vmui/). +To access Grafana use link [http://localhost:3000](http://localhost:3000). + +To access [VictoriaLogs UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui) +use link [http://localhost:9428/select/vmui](http://localhost:9428/select/vmui). Please, also see [how to monitor](https://docs.victoriametrics.com/VictoriaLogs/#monitoring) VictoriaLogs installations. + +To shutdown environment execute the following command: +``` +make docker-victorialogs-down +``` \ No newline at end of file diff --git a/deployment/docker/docker-compose-victorialogs.yml b/deployment/docker/docker-compose-victorialogs.yml new file mode 100644 index 000000000..b64245f5f --- /dev/null +++ b/deployment/docker/docker-compose-victorialogs.yml @@ -0,0 +1,80 @@ +version: "3.5" +services: + # Grafana instance configured with VictoriaLogs as datasource + grafana: + container_name: grafana + image: grafana/grafana:10.3.1 + depends_on: + - "victoriametrics" + - "victorialogs" + ports: + - 3000:3000 + entrypoint: [ "/bin/bash", "-c" ] + command: [ "chmod +x /download.sh && /download.sh && /run.sh" ] + volumes: + - grafanadata:/var/lib/grafana + - ./provisioning/datasources/victorialogs-datasource:/etc/grafana/provisioning/datasources + - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ./provisioning/plugins/:/var/lib/grafana/plugins + - ./../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json + - ./../../dashboards/victorialogs.json:/var/lib/grafana/dashboards/vl.json + - ./victorialogs/download.sh:/download.sh + environment: + - "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victorialogs-datasource" + networks: + - vm_net + restart: always + + # fluentbit is logs collector. It collects logs according to fluent-bit.conf + # and forwards them to VictoriaLogs + fluentbit: + container_name: fluentbit + image: cr.fluentbit.io/fluent/fluent-bit:2.1.4 + volumes: + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf + depends_on: [victorialogs] + ports: + - "5140:5140" + networks: + - vm_net + +# VictoriaLogs instance, a single process responsible for +# storing logs and serving read queries. + victorialogs: + container_name: victorialogs + image: docker.io/victoriametrics/victoria-logs:v0.4.2-victorialogs + command: + - "--storageDataPath=/vlogs" + - "--httpListenAddr=:9428" + volumes: + - vldata:/vlogs + ports: + - "9428:9428" + networks: + - vm_net + + # VictoriaMetrics instance, a single process responsible for + # scraping, storing metrics and serve read requests. + victoriametrics: + container_name: victoriametrics + image: victoriametrics/victoria-metrics:v1.98.0 + ports: + - 8428:8428 + volumes: + - vmdata:/storage + - ./prometheus-victorialogs.yml:/etc/prometheus/prometheus.yml + command: + - "--storageDataPath=/storage" + - "--httpListenAddr=:8428" + - "--promscrape.config=/etc/prometheus/prometheus.yml" + networks: + - vm_net + restart: always + +volumes: + vmdata: {} + vldata: {} + grafanadata: {} +networks: + vm_net: diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index 6c63446b2..2bffe1b93 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -57,7 +57,6 @@ services: - ./provisioning/datasources/prometheus-datasource:/etc/grafana/provisioning/datasources - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards - ./../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json - - ./../../dashboards/victorialogs.json:/var/lib/grafana/dashboards/vl.json - ./../../dashboards/vmagent.json:/var/lib/grafana/dashboards/vmagent.json - ./../../dashboards/vmalert.json:/var/lib/grafana/dashboards/vmalert.json networks: @@ -107,38 +106,9 @@ services: - vm_net restart: always - # fluentbit is logs collector. It collects logs according to fluent-bit.conf - # and forwards them to VictoriaLogs - fluentbit: - container_name: fluentbit - image: cr.fluentbit.io/fluent/fluent-bit:2.1.4 - volumes: - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf - depends_on: [victorialogs] - ports: - - "5140:5140" - networks: - - vm_net - -# VictoriaLogs instance, a single process responsible for -# storing logs and serving read queries. - victorialogs: - container_name: victorialogs - image: docker.io/victoriametrics/victoria-logs:v0.4.2-victorialogs - command: - - "--storageDataPath=/vlogs" - - "--httpListenAddr=:9428" - volumes: - - victorialogs-fluentbit:/vlogs - ports: - - "9428:9428" - networks: - - vm_net volumes: vmagentdata: {} vmdata: {} grafanadata: {} - victorialogs-fluentbit: {} networks: vm_net: diff --git a/deployment/docker/fluent-bit.conf b/deployment/docker/fluent-bit.conf index 42d349338..44e3d9429 100644 --- a/deployment/docker/fluent-bit.conf +++ b/deployment/docker/fluent-bit.conf @@ -16,6 +16,9 @@ [SERVICE] Flush 1 Parsers_File parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_PORT 2020 [Output] Name http diff --git a/deployment/docker/prometheus-victorialogs.yml b/deployment/docker/prometheus-victorialogs.yml new file mode 100644 index 000000000..0bbcafe8d --- /dev/null +++ b/deployment/docker/prometheus-victorialogs.yml @@ -0,0 +1,13 @@ +global: + scrape_interval: 10s + +scrape_configs: + - job_name: 'victoriametrics' + static_configs: + - targets: ['victoriametrics:8428'] + - job_name: 'victorialogs' + static_configs: + - targets: ['victorialogs:9428'] + - job_name: 'fluentbit' + static_configs: + - targets: ['fluentbit:2020/api/v1/metrics/prometheus'] \ No newline at end of file diff --git a/deployment/docker/prometheus.yml b/deployment/docker/prometheus.yml index 9c034f200..3dbc659fe 100644 --- a/deployment/docker/prometheus.yml +++ b/deployment/docker/prometheus.yml @@ -11,6 +11,3 @@ scrape_configs: - job_name: 'victoriametrics' static_configs: - targets: ['victoriametrics:8428'] - - job_name: 'victorialogs' - static_configs: - - targets: ['victorialogs:9428'] diff --git a/deployment/docker/provisioning/datasources/victorialogs-datasource/victorialogs-datasource.yml b/deployment/docker/provisioning/datasources/victorialogs-datasource/victorialogs-datasource.yml new file mode 100644 index 000000000..8daca757f --- /dev/null +++ b/deployment/docker/provisioning/datasources/victorialogs-datasource/victorialogs-datasource.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +datasources: + - name: VictoriaLogs + type: victorialogs-datasource + access: proxy + url: http://victorialogs:9428 + + - name: VictoriaMetrics + type: prometheus + access: proxy + url: http://victoriametrics:8428 \ No newline at end of file diff --git a/deployment/docker/victorialogs/download.sh b/deployment/docker/victorialogs/download.sh new file mode 100755 index 000000000..5c4715426 --- /dev/null +++ b/deployment/docker/victorialogs/download.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -ex + +VM_DS_PATH='/var/lib/grafana/plugins/victorialogs-datasource' +PLUGIN_PATH='/var/lib/grafana/plugins' + +if [[ -f ${VM_DS_PATH}/plugin.json ]]; then + ver=$(cat ${VM_DS_PATH}/plugin.json) + if [[ ! -z "$ver" ]]; then + exit + fi +fi + +echo "VictoriaLogs datasource is not installed. Installing datasource..." +rm -rf ${VM_DS_PATH}/* || true +mkdir -p ${VM_DS_PATH} + +export LATEST_VERSION=$(curl https://api.github.com/repos/VictoriaMetrics/victorialogs-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ +curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/${LATEST_VERSION}/victorialogs-datasource-${LATEST_VERSION}.tar.gz -o ${PLUGIN_PATH}/plugin.tar.gz && \ +tar -xzf ${PLUGIN_PATH}/plugin.tar.gz -C ${PLUGIN_PATH} +echo "VictoriaLogs datasource has been installed." +rm ${PLUGIN_PATH}/plugin.tar.gz diff --git a/deployment/docker/vm-datasource/download.sh b/deployment/docker/vm-datasource/download.sh index 21f9f1ae6..ea31964ef 100755 --- a/deployment/docker/vm-datasource/download.sh +++ b/deployment/docker/vm-datasource/download.sh @@ -11,12 +11,12 @@ if [[ -f ${VM_DS_PATH}/plugin.json ]]; then fi fi -echo "Victoriametrics datasource is not installed. Installing datasource..." +echo "VictoriaMetrics datasource is not installed. Installing datasource..." rm -rf ${VM_DS_PATH}/* || true mkdir -p ${VM_DS_PATH} export LATEST_VERSION=$(curl https://api.github.com/repos/VictoriaMetrics/grafana-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ curl -L https://github.com/VictoriaMetrics/grafana-datasource/releases/download/${LATEST_VERSION}/victoriametrics-datasource-${LATEST_VERSION}.tar.gz -o ${PLUGIN_PATH}/plugin.tar.gz && \ tar -xzf ${PLUGIN_PATH}/plugin.tar.gz -C ${PLUGIN_PATH} -echo "Victoriametrics datasource has been installed." +echo "VictoriaMetrics datasource has been installed." rm ${PLUGIN_PATH}/plugin.tar.gz diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 14f46c8cc..04ab8379b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -40,6 +40,7 @@ See also [LTS releases](https://docs.victoriametrics.com/LTS-releases.html). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for [InfluxDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-influxdb-1x), [Remote Read protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-by-remote-read-protocol) and [OpenTSDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-opentsdb). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for pull requests [1](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5783), [2](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5798), [3](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5797). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): preserve [`WITH` templates](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/expand-with-exprs) when clicking the `prettify query` button at the right side of query input field. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5383). * FEATURE: [vmalert](https://docs.victoriametrics.com/#vmalert): support filtering by group, rule or labels in [vmalert's UI](https://docs.victoriametrics.com/vmalert/#web) for `/groups` and `/alerts` pages. See [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5791) by @victoramsantos. +* FEATURE: [docker-compose](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#docker-compose-environment-for-victoriametrics): create a separate [docker-compose environment](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/docker-compose-victorialogs.yml) for VictoriaLogs installation, including fluentbit and [VictoriaLogs Grafana datasource](https://github.com/VictoriaMetrics/victorialogs-datasource). * BUGFIX: all VictoriaMetrics components: return back periodic closing of incoming connections to `-httpListenAddr` every 2 minutes, which was disabled in [v1.98.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.98.0) when addressing [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1304#issuecomment-1636997037). See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1304#issuecomment-1961891450) for details on why the periodic closing of incoming connections has been returned back. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fix possible deadlock when [sharding among remote storages](https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages) is enabled with `-remoteWrite.shardByURL` command-line flag. Thanks to @penguinlav for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5834) for [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5833).