diff --git a/deployment/logs-benchmark/Makefile b/deployment/logs-benchmark/Makefile new file mode 100644 index 000000000..ace542971 --- /dev/null +++ b/deployment/logs-benchmark/Makefile @@ -0,0 +1,17 @@ +prepare-logs: + cd ./source_logs && bash download.sh + +docker-up-elk: + docker-compose -f docker-compose.yml -f docker-compose-elk.yml up -d + +docker-stop-elk: + docker-compose -f docker-compose.yml -f docker-compose-elk.yml stop + +docker-up-loki: + docker-compose -f docker-compose.yml -f docker-compose-loki.yml up -d + +docker-stop-loki: + docker-compose -f docker-compose.yml -f docker-compose-loki.yml stop + +docker-cleanup: + docker-compose -f docker-compose.yml -f docker-compose-elk.yml -f docker-compose-loki.yml down -v --remove-orphans diff --git a/deployment/logs-benchmark/docker-compose-elk.yml b/deployment/logs-benchmark/docker-compose-elk.yml new file mode 100644 index 000000000..0b96af053 --- /dev/null +++ b/deployment/logs-benchmark/docker-compose-elk.yml @@ -0,0 +1,69 @@ +version: '3' + +services: + filebeat-elastic: + image: docker.elastic.co/beats/filebeat:8.8.0 + restart: on-failure + volumes: + - ./elk/filebeat/filebeat-elastic.yml:/usr/share/filebeat/filebeat.yml:ro + depends_on: + - elastic + + filebeat-vlogs: + image: docker.elastic.co/beats/filebeat:8.8.0 + restart: on-failure + volumes: + - ./elk/filebeat/filebeat-vlogs.yml:/usr/share/filebeat/filebeat.yml:ro + depends_on: + - vlogs + + generator: + image: golang:1.21.3-alpine + restart: always + working_dir: /go/src/app + volumes: + - ./generator:/go/src/app + - ./source_logs:/go/src/source_logs + command: + - go + - run + - main.go + - -logsPath=/go/src/source_logs/logs + - -outputRateLimitItems=10000 + - -syslog.addr=filebeat-elastic:12345 + - -syslog.addr2=filebeat-vlogs:12345 + - -logs.randomSuffix=false + depends_on: [ filebeat-elastic, filebeat-vlogs ] + + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0 + volumes: + - ./elk/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml + - elastic:/usr/share/elasticsearch/data + environment: + ES_JAVA_OPTS: "-Xmx2048m" + + kibana: + image: docker.elastic.co/kibana/kibana:8.8.0 + volumes: + - ./elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml + ports: + - '5601:5601' + depends_on: [ elastic ] + + beat-exporter-elastic: + image: trustpilot/beat-exporter:0.4.0 + command: + - -beat.uri=http://filebeat-elastic:5066 + depends_on: + - filebeat-elastic + + beat-exporter-vlogs: + image: trustpilot/beat-exporter:0.4.0 + command: + - -beat.uri=http://filebeat-vlogs:5066 + depends_on: + - filebeat-vlogs + +volumes: + elastic: diff --git a/deployment/logs-benchmark/docker-compose-loki.yml b/deployment/logs-benchmark/docker-compose-loki.yml new file mode 100644 index 000000000..231884052 --- /dev/null +++ b/deployment/logs-benchmark/docker-compose-loki.yml @@ -0,0 +1,52 @@ +version: '3' + +services: + generator: + image: golang:1.21.3-alpine + restart: always + working_dir: /go/src/app + volumes: + - ./generator:/go/src/app + - ./source_logs:/go/src/source_logs + command: + - go + - run + - main.go + - -logsPath=/go/src/source_logs/logs + - -outputRateLimitItems=10000 + - -outputRateLimitPeriod=1s + - -syslog.addr=rsyslog:514 + - -syslog.addr2=rsyslog:514 + - -logs.randomSuffix=false + depends_on: [ rsyslog ] + + loki: + image: grafana/loki:2.9.0 + user: 0:0 + ports: + - "3100:3100" + command: -config.file=/etc/loki/loki-config.yaml + volumes: + - loki:/tmp/loki + - ./loki/:/etc/loki/ + + promtail: + image: grafana/promtail:2.9.0 + command: -config.file=/etc/promtail/promtail-config.yaml + volumes: + - ./loki/:/etc/promtail/ + depends_on: + - loki + - vlogs + + rsyslog: + build: + dockerfile: Dockerfile + context: rsyslog + volumes: + - ./rsyslog/rsyslog.conf:/etc/rsyslog.conf + depends_on: [ promtail ] + + +volumes: + loki: diff --git a/deployment/logs-benchmark/docker-compose.yml b/deployment/logs-benchmark/docker-compose.yml index 3a9bb6301..2281cc682 100644 --- a/deployment/logs-benchmark/docker-compose.yml +++ b/deployment/logs-benchmark/docker-compose.yml @@ -1,47 +1,6 @@ version: '3' services: - filebeat-elastic: - image: docker.elastic.co/beats/filebeat:8.8.0 - restart: on-failure - volumes: - - ./elk/filebeat/filebeat-elastic.yml:/usr/share/filebeat/filebeat.yml:ro - depends_on: [ elastic ] - - filebeat-vlogs: - image: docker.elastic.co/beats/filebeat:8.8.0 - restart: on-failure - volumes: - - ./elk/filebeat/filebeat-vlogs.yml:/usr/share/filebeat/filebeat.yml:ro - depends_on: [ vlogs ] - - generator: - image: golang:1.20-alpine - restart: always - working_dir: /go/src/app - volumes: - - ./generator:/go/src/app - - ./source_logs:/go/src/source_logs - command: - - go - - run - - main.go - - -logsPath=/go/src/source_logs/logs - - -outputRateLimitItems=10000 - - -outputRateLimitPeriod=1s - - -syslog.addr=filebeat-elastic:12345 - - -syslog.addr2=filebeat-vlogs:12345 - - -logs.randomSuffix=false - depends_on: [ filebeat-vlogs, filebeat-elastic ] - - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0 - volumes: - - ./elk/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - - elastic:/usr/share/elasticsearch/data - environment: - ES_JAVA_OPTS: "-Xmx2048m" - # Run `make package-victoria-logs` to build victoria-logs image vlogs: image: docker.io/victoriametrics/victoria-logs:v0.4.1-victorialogs @@ -52,24 +11,6 @@ services: command: - -storageDataPath=/vlogs - kibana: - image: docker.elastic.co/kibana/kibana:8.8.0 - volumes: - - ./elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml - ports: - - '5601:5601' - depends_on: [ elastic ] - - beat-exporter-elastic: - image: trustpilot/beat-exporter:0.4.0 - command: - - -beat.uri=http://filebeat-elastic:5066 - - beat-exporter-vlogs: - image: trustpilot/beat-exporter:0.4.0 - command: - - -beat.uri=http://filebeat-vlogs:5066 - cadvisor: image: gcr.io/cadvisor/cadvisor:v0.47.0 restart: unless-stopped @@ -128,7 +69,6 @@ services: restart: always volumes: - elastic: vlogs: vmsingle: grafanadata: {} diff --git a/deployment/logs-benchmark/generator/main.go b/deployment/logs-benchmark/generator/main.go index 25f778d14..22f9d8a4f 100644 --- a/deployment/logs-benchmark/generator/main.go +++ b/deployment/logs-benchmark/generator/main.go @@ -64,6 +64,13 @@ func main() { } syslogTag := "logs-benchmark-" + sourceFile + "-" + strconv.FormatInt(startedAt, 10) + + // Loki uses RFC5424 syslog format, which has a 48 character limit on the tag. + tagLen := len(syslogTag) + if tagLen > 48 { + truncate := tagLen - 48 + syslogTag = syslogTag[truncate:] + } logger, err := syslog.Dial("tcp", *syslogAddr, syslog.LOG_INFO, syslogTag) if err != nil { panic(fmt.Errorf("error dialing syslog: %w", err)) diff --git a/deployment/logs-benchmark/grafana/dashboards/stats.json b/deployment/logs-benchmark/grafana/dashboards/elk-stats.json similarity index 100% rename from deployment/logs-benchmark/grafana/dashboards/stats.json rename to deployment/logs-benchmark/grafana/dashboards/elk-stats.json diff --git a/deployment/logs-benchmark/grafana/dashboards/comparison.json b/deployment/logs-benchmark/grafana/dashboards/elk.json similarity index 98% rename from deployment/logs-benchmark/grafana/dashboards/comparison.json rename to deployment/logs-benchmark/grafana/dashboards/elk.json index 0afd9efd8..ba896c8e9 100644 --- a/deployment/logs-benchmark/grafana/dashboards/comparison.json +++ b/deployment/logs-benchmark/grafana/dashboards/elk.json @@ -512,12 +512,10 @@ "current": { "selected": true, "text": [ - "logs-benchmark-elastic-1", - "logs-benchmark-vlogs-1" + "All" ], "value": [ - "logs-benchmark-elastic-1", - "logs-benchmark-vlogs-1" + "$__all" ] }, "datasource": { @@ -536,7 +534,7 @@ "refId": "StandardVariableQuery" }, "refresh": 1, - "regex": ".*vlogs|elastic.*", + "regex": ".*benchmark-vlogs|benchmark-elastic.*", "skipUrlSync": false, "sort": 1, "type": "query" @@ -551,6 +549,6 @@ "timezone": "", "title": "Elastic vs VLogs", "uid": "hkm6P6_4z", - "version": 2, + "version": 3, "weekStart": "" } diff --git a/deployment/logs-benchmark/grafana/dashboards/loki.json b/deployment/logs-benchmark/grafana/dashboards/loki.json new file mode 100644 index 000000000..fbe1ed4ad --- /dev/null +++ b/deployment/logs-benchmark/grafana/dashboards/loki.json @@ -0,0 +1,597 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*VictoriaLogs.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Loki.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(rate(vl_rows_ingested_total))", + "legendFormat": "VictoriaLogs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "rate(loki_distributor_lines_received_total)", + "hide": false, + "legendFormat": "Loki", + "range": true, + "refId": "B" + } + ], + "title": "Ingestion rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*vlogs.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*loki.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$containers\"})) by (name)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU cores usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*vlogs.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*loki.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(container_memory_rss{name=~\"$containers\"}) by (name)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*vlogs.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*loki.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(node_disk_usage_bytes{path=~\"$containers_selector\"}) by (path)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk space used", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "hide": 2, + "name": "containers_selector", + "query": ".*(vlogs|loki).*", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "definition": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "containers", + "options": [], + "query": { + "query": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": ".*benchmark-vlogs|benchmark-loki.*", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Loki vs VLogs", + "uid": "hkm6P6_4y", + "version": 1, + "weekStart": "" +} diff --git a/deployment/logs-benchmark/loki/loki-config.yaml b/deployment/logs-benchmark/loki/loki-config.yaml new file mode 100644 index 000000000..2d9649c06 --- /dev/null +++ b/deployment/logs-benchmark/loki/loki-config.yaml @@ -0,0 +1,63 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 0.0.0.0 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +analytics: + reporting_enabled: false + + +limits_config: + ingestion_rate_mb: 10240 + ingestion_burst_size_mb: 10240 + max_streams_per_user: 10000000 + max_global_streams_per_user: 10000000 + + retention_period: 30d + + per_stream_rate_limit: 10240M + per_stream_rate_limit_burst: 10240M + cardinality_limit: 20000000 diff --git a/deployment/logs-benchmark/loki/promtail-config.yaml b/deployment/logs-benchmark/loki/promtail-config.yaml new file mode 100644 index 000000000..c83d50513 --- /dev/null +++ b/deployment/logs-benchmark/loki/promtail-config.yaml @@ -0,0 +1,26 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + - url: http://vlogs:9428/insert/loki/api/v1/push?_stream_fields=hostname,application +# batchwait: 5s +# batchsize: 5242880 + +scrape_configs: +- job_name: syslog + syslog: + listen_address: 0.0.0.0:5140 + idle_timeout: 12h + use_incoming_timestamp: true + labels: + job: syslog + relabel_configs: + - source_labels: [__syslog_message_hostname] + target_label: hostname + - source_labels: [__syslog_message_app_name] + target_label: application diff --git a/deployment/logs-benchmark/readme.md b/deployment/logs-benchmark/readme.md index b5afa9685..72e4f45cc 100644 --- a/deployment/logs-benchmark/readme.md +++ b/deployment/logs-benchmark/readme.md @@ -1,23 +1,44 @@ # Benchmark for VictoriaLogs +Benchmark compares VictoriaLogs with ELK stack and Grafana Loki. + Benchmark is based on: - Logs from this repository - https://github.com/logpai/loghub +- [logs generator](./generator) + +For ELK suite it uses: + - filebeat - https://www.elastic.co/beats/filebeat - elastic + kibana -- [logs generator](./generator) + +For Grafana Loki suite it uses: + +- [Promtail](https://grafana.com/docs/loki/latest/send-data/promtail/) +- rsyslog - required to push logs in RFC5424 format to Promtail +- [Loki](https://grafana.com/oss/loki/) ## How it works -docker-compose.yml contains: +[docker-compose.yml](./docker-compose.yml) contains common configurations for all suites: +- VictoriaLogs instance +- vmsingle - port forwarded to `localhost:8428` to see UI +- exporters for system metris + +ELK suite uses [docker-compose-elk.yml](./docker-compose-elk.yml) with the following services: + +- [logs generator](./generator) which generates logs and sends them to filebeat instances via syslog - 2 filebeat instances - one for elastic and one for VictoriaLogs. - elastic instance -- VictoriaLogs instance - kibana instance - port forwarded to `localhost:5601` to see UI -- vmsingle - port forwarded to `localhost:8428` to see UI -- [logs generator](./generator) -- exporters for filebeat/system + +Loki suite uses [docker-compose-loki.yml](./docker-compose-loki.yml) with the following services: + +- [logs generator](./generator) which generates logs and sends them rsyslog +- rsyslog instance - sends logs to Promtail +- Promtail instance - sends logs to Loki and VictoriaLogs +- Loki instance [Logs generator](./generator) generates logs based on logs located at `./source_logs/logs` and sends them to filebeat instances via syslog. @@ -44,6 +65,7 @@ Note that with logs listed in `download.sh` it will require 49GB of free space: If it is needed to minimize disk footprint, you can download only some of them by commenting out lines in `download.sh`. Unarchived logs size per file for reference: + ```shell 2.3M Linux.log 73M SSH.log @@ -52,12 +74,11 @@ Unarchived logs size per file for reference: 13G hadoop-*.log ``` - 2. (optional) If needed, adjust amount of logs sent by generator by modifying `-outputRateLimitItems` and `outputRateLimitPeriod` parameters in [docker-compose.yml](./docker-compose.yml). By default, it is configured to send 10000 logs per second. -3. Build victoria-logs image and adjust `image` parameter in [docker-compose.yml](./docker-compose.yml): +3. (optional) Build victoria-logs image and adjust `image` parameter in [docker-compose.yml](./docker-compose.yml): ```shell make package-victoria-logs @@ -74,8 +95,30 @@ output.elasticsearch: hosts: [ "http://vlogs:9428/insert/elasticsearch/" ] ``` -4. Run `docker-compose up -d` to start benchmark. +4. Choose a suite to run. -5. Navigate to `http://localhost:3000/d/hkm6P6_4z/elastic-vs-vlogs` to see Grafana dashboard with resource usage comparison. + In order to run ELK suite use the following command: + ``` + make docker-up-elk + ``` + In order to run Loki suite use the following command: + ``` + make docker-up-loki + ``` -![grafana-dashboard.png](grafana-dashboard.png) + +5. Navigate to `http://localhost:3000/` to see Grafana dashboards with resource usage + comparison. + + Navigate to `http://localhost:3000/d/hkm6P6_4z/elastic-vs-vlogs` to see ELK suite results. + + Navigate to `http://localhost:3000/d/hkm6P6_4y/loki-vs-vlogs` to see Loki suite results. + + +Example results vs ELK: + +![elk-grafana-dashboard.png](results/elk-grafana-dashboard.png) + +Example results vs Loki: + +![loki-grafana-dashboard.png](results/loki-grafana-dashboard.png) diff --git a/deployment/logs-benchmark/grafana-dashboard.png b/deployment/logs-benchmark/results/elk-grafana-dashboard.png similarity index 100% rename from deployment/logs-benchmark/grafana-dashboard.png rename to deployment/logs-benchmark/results/elk-grafana-dashboard.png diff --git a/deployment/logs-benchmark/results/loki-grafana-dashboard.png b/deployment/logs-benchmark/results/loki-grafana-dashboard.png new file mode 100644 index 000000000..49c0542df Binary files /dev/null and b/deployment/logs-benchmark/results/loki-grafana-dashboard.png differ diff --git a/deployment/logs-benchmark/rsyslog/Dockerfile b/deployment/logs-benchmark/rsyslog/Dockerfile new file mode 100644 index 000000000..e8cf6effd --- /dev/null +++ b/deployment/logs-benchmark/rsyslog/Dockerfile @@ -0,0 +1,9 @@ +FROM alpine:3 +RUN apk add --no-cache rsyslog rsyslog-tls tzdata + +COPY rsyslog.conf /etc/rsyslog.conf + +VOLUME /var/run/rsyslog/dev +EXPOSE 514 10514 + +CMD ["/usr/sbin/rsyslogd", "-n"] diff --git a/deployment/logs-benchmark/rsyslog/rsyslog.conf b/deployment/logs-benchmark/rsyslog/rsyslog.conf new file mode 100644 index 000000000..d30655f2a --- /dev/null +++ b/deployment/logs-benchmark/rsyslog/rsyslog.conf @@ -0,0 +1,12 @@ +module(load="imudp") +input(type="imudp" port="514") + +module(load="imtcp") +input(type="imtcp" port="514") + +*.* action(type="omfwd" + protocol="tcp" target="promtail" port="5140" + Template="RSYSLOG_SyslogProtocol23Format" + TCP_Framing="octet-counted" KeepAlive="on" + action.resumeRetryCount="-1" + queue.type="linkedlist" queue.size="1000000") diff --git a/deployment/logs-benchmark/vmsingle/promscrape.yml b/deployment/logs-benchmark/vmsingle/promscrape.yml index 6f289cd93..1d54e1e42 100644 --- a/deployment/logs-benchmark/vmsingle/promscrape.yml +++ b/deployment/logs-benchmark/vmsingle/promscrape.yml @@ -22,3 +22,9 @@ scrape_configs: static_configs: - targets: ['node-exporter:9100'] - targets: ['du-exporter:9995'] + - job_name: 'loki' + static_configs: + - targets: ['loki:3100'] + - job_name: 'promtail' + static_configs: + - targets: ['promtail:9080']