github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

deployment/logs-benchmark: add suite for Loki (#5165)

Browse source 
* deployment/logs-benchmark: add suite for Loki

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

* deployment/logs-benchmark: update go image to 1.21.3

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

* deployment/logs-benchmark: split command to run elk and loki

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

---------

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
This commit is contained in:
Zakhar Bessarab 2023-10-13 17:13:41 +04:00 committed by GitHub
parent d1638db849
commit b246b0e55f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 916 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
deployment/logs-benchmark/Makefile Normal file
View file

@ -0,0 +1,17 @@
prepare-logs:
cd ./source_logs && bash download.sh
docker-up-elk:
docker-compose -f docker-compose.yml -f docker-compose-elk.yml up -d
docker-stop-elk:
docker-compose -f docker-compose.yml -f docker-compose-elk.yml stop
docker-up-loki:
docker-compose -f docker-compose.yml -f docker-compose-loki.yml up -d
docker-stop-loki:
docker-compose -f docker-compose.yml -f docker-compose-loki.yml stop
docker-cleanup:
docker-compose -f docker-compose.yml -f docker-compose-elk.yml -f docker-compose-loki.yml down -v --remove-orphans

69
deployment/logs-benchmark/docker-compose-elk.yml Normal file
View file

@ -0,0 +1,69 @@
version: '3'
services:
filebeat-elastic:
image: docker.elastic.co/beats/filebeat:8.8.0
restart: on-failure
volumes:
- ./elk/filebeat/filebeat-elastic.yml:/usr/share/filebeat/filebeat.yml:ro
depends_on:
- elastic
filebeat-vlogs:
image: docker.elastic.co/beats/filebeat:8.8.0
restart: on-failure
volumes:
- ./elk/filebeat/filebeat-vlogs.yml:/usr/share/filebeat/filebeat.yml:ro
depends_on:
- vlogs
generator:
image: golang:1.21.3-alpine
restart: always
working_dir: /go/src/app
volumes:
- ./generator:/go/src/app
- ./source_logs:/go/src/source_logs
command:
- go
- run
- main.go
- -logsPath=/go/src/source_logs/logs
- -outputRateLimitItems=10000
- -syslog.addr=filebeat-elastic:12345
- -syslog.addr2=filebeat-vlogs:12345
- -logs.randomSuffix=false
depends_on: [ filebeat-elastic, filebeat-vlogs ]
elastic:
image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
volumes:
- ./elk/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- elastic:/usr/share/elasticsearch/data
environment:
ES_JAVA_OPTS: "-Xmx2048m"
kibana:
image: docker.elastic.co/kibana/kibana:8.8.0
volumes:
- ./elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
ports:
- '5601:5601'
depends_on: [ elastic ]
beat-exporter-elastic:
image: trustpilot/beat-exporter:0.4.0
command:
- -beat.uri=http://filebeat-elastic:5066
depends_on:
- filebeat-elastic
beat-exporter-vlogs:
image: trustpilot/beat-exporter:0.4.0
command:
- -beat.uri=http://filebeat-vlogs:5066
depends_on:
- filebeat-vlogs
volumes:
elastic:

52
deployment/logs-benchmark/docker-compose-loki.yml Normal file
View file

@ -0,0 +1,52 @@
version: '3'
services:
generator:
image: golang:1.21.3-alpine
restart: always
working_dir: /go/src/app
volumes:
- ./generator:/go/src/app
- ./source_logs:/go/src/source_logs
command:
- go
- run
- main.go
- -logsPath=/go/src/source_logs/logs
- -outputRateLimitItems=10000
- -outputRateLimitPeriod=1s
- -syslog.addr=rsyslog:514
- -syslog.addr2=rsyslog:514
- -logs.randomSuffix=false
depends_on: [ rsyslog ]
loki:
image: grafana/loki:2.9.0
user: 0:0
ports:
- "3100:3100"
command: -config.file=/etc/loki/loki-config.yaml
volumes:
- loki:/tmp/loki
- ./loki/:/etc/loki/
promtail:
image: grafana/promtail:2.9.0
command: -config.file=/etc/promtail/promtail-config.yaml
volumes:
- ./loki/:/etc/promtail/
depends_on:
- loki
- vlogs
rsyslog:
build:
dockerfile: Dockerfile
context: rsyslog
volumes:
- ./rsyslog/rsyslog.conf:/etc/rsyslog.conf
depends_on: [ promtail ]
volumes:
loki:

60
deployment/logs-benchmark/docker-compose.yml
View file

@ -1,47 +1,6 @@
version: '3'
services:
filebeat-elastic:
image: docker.elastic.co/beats/filebeat:8.8.0
restart: on-failure
volumes:
- ./elk/filebeat/filebeat-elastic.yml:/usr/share/filebeat/filebeat.yml:ro
depends_on: [ elastic ]
filebeat-vlogs:
image: docker.elastic.co/beats/filebeat:8.8.0
restart: on-failure
volumes:
- ./elk/filebeat/filebeat-vlogs.yml:/usr/share/filebeat/filebeat.yml:ro
depends_on: [ vlogs ]
generator:
image: golang:1.20-alpine
restart: always
working_dir: /go/src/app
volumes:
- ./generator:/go/src/app
- ./source_logs:/go/src/source_logs
command:
- go
- run
- main.go
- -logsPath=/go/src/source_logs/logs
- -outputRateLimitItems=10000
- -outputRateLimitPeriod=1s
- -syslog.addr=filebeat-elastic:12345
- -syslog.addr2=filebeat-vlogs:12345
- -logs.randomSuffix=false
depends_on: [ filebeat-vlogs, filebeat-elastic ]
elastic:
image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
volumes:
- ./elk/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- elastic:/usr/share/elasticsearch/data
environment:
ES_JAVA_OPTS: "-Xmx2048m"
# Run `make package-victoria-logs` to build victoria-logs image
vlogs:
image: docker.io/victoriametrics/victoria-logs:v0.4.1-victorialogs
@ -52,24 +11,6 @@ services:
command:
- -storageDataPath=/vlogs
kibana:
image: docker.elastic.co/kibana/kibana:8.8.0
volumes:
- ./elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
ports:
- '5601:5601'
depends_on: [ elastic ]
beat-exporter-elastic:
image: trustpilot/beat-exporter:0.4.0
command:
- -beat.uri=http://filebeat-elastic:5066
beat-exporter-vlogs:
image: trustpilot/beat-exporter:0.4.0
command:
- -beat.uri=http://filebeat-vlogs:5066
cadvisor:
image: gcr.io/cadvisor/cadvisor:v0.47.0
restart: unless-stopped
@ -128,7 +69,6 @@ services:
restart: always
volumes:
elastic:
vlogs:
vmsingle:
grafanadata: {}

7
deployment/logs-benchmark/generator/main.go
View file

@ -64,6 +64,13 @@ func main() {
}
syslogTag := "logs-benchmark-" + sourceFile + "-" + strconv.FormatInt(startedAt, 10)
// Loki uses RFC5424 syslog format, which has a 48 character limit on the tag.
tagLen := len(syslogTag)
if tagLen > 48 {
truncate := tagLen - 48
syslogTag = syslogTag[truncate:]
}
logger, err := syslog.Dial("tcp", *syslogAddr, syslog.LOG_INFO, syslogTag)
if err != nil {
panic(fmt.Errorf("error dialing syslog: %w", err))

0
deployment/logs-benchmark/grafana/dashboards/stats.json → deployment/logs-benchmark/grafana/dashboards/elk-stats.json
View file

10
deployment/logs-benchmark/grafana/dashboards/comparison.json → deployment/logs-benchmark/grafana/dashboards/elk.json
View file

@ -512,12 +512,10 @@
"current": {
"selected": true,
"text": [
"logs-benchmark-elastic-1",
"logs-benchmark-vlogs-1"
"All"
],
"value": [
"logs-benchmark-elastic-1",
"logs-benchmark-vlogs-1"
"$__all"
]
},
"datasource": {
@ -536,7 +534,7 @@
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": ".*vlogs|elastic.*",
"regex": ".*benchmark-vlogs|benchmark-elastic.*",
"skipUrlSync": false,
"sort": 1,
"type": "query"
@ -551,6 +549,6 @@
"timezone": "",
"title": "Elastic vs VLogs",
"uid": "hkm6P6_4z",
"version": 2,
"version": 3,
"weekStart": ""
}

597
deployment/logs-benchmark/grafana/dashboards/loki.json Normal file
View file

@ -0,0 +1,597 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*VictoriaLogs.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*Loki.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(rate(vl_rows_ingested_total))",
"legendFormat": "VictoriaLogs",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "rate(loki_distributor_lines_received_total)",
"hide": false,
"legendFormat": "Loki",
"range": true,
"refId": "B"
}
],
"title": "Ingestion rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*vlogs.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*loki.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 7
},
"id": 2,
"options": {
"legend": {
"calcs": [
"mean",
"min",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$containers\"})) by (name)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "CPU cores usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*vlogs.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*loki.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 12,
"x": 12,
"y": 7
},
"id": 3,
"options": {
"legend": {
"calcs": [
"min",
"max",
"mean"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(container_memory_rss{name=~\"$containers\"}) by (name)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Memory Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "decbytes"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*vlogs.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*loki.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 18
},
"id": 5,
"options": {
"legend": {
"calcs": [
"min",
"max",
"mean"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(node_disk_usage_bytes{path=~\"$containers_selector\"}) by (path)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Disk space used",
"type": "timeseries"
}
],
"refresh": "10s",
"schemaVersion": 37,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"hide": 2,
"name": "containers_selector",
"query": ".*(vlogs|loki).*",
"skipUrlSync": false,
"type": "constant"
},
{
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"definition": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
"hide": 0,
"includeAll": true,
"label": "",
"multi": true,
"name": "containers",
"options": [],
"query": {
"query": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": ".*benchmark-vlogs|benchmark-loki.*",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Loki vs VLogs",
"uid": "hkm6P6_4y",
"version": 1,
"weekStart": ""
}

63
deployment/logs-benchmark/loki/loki-config.yaml Normal file
View file

@ -0,0 +1,63 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
common:
instance_addr: 0.0.0.0
path_prefix: /tmp/loki
storage:
filesystem:
chunks_directory: /tmp/loki/chunks
rules_directory: /tmp/loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
query_range:
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 100
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
ruler:
alertmanager_url: http://localhost:9093
# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
#
# Statistics help us better understand how Loki is used, and they show us performance
# levels for most users. This helps us prioritize features and documentation.
# For more information on what's sent, look at
# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
# Refer to the buildReport method to see what goes into a report.
#
# If you would like to disable reporting, uncomment the following lines:
analytics:
reporting_enabled: false
limits_config:
ingestion_rate_mb: 10240
ingestion_burst_size_mb: 10240
max_streams_per_user: 10000000
max_global_streams_per_user: 10000000
retention_period: 30d
per_stream_rate_limit: 10240M
per_stream_rate_limit_burst: 10240M
cardinality_limit: 20000000

26
deployment/logs-benchmark/loki/promtail-config.yaml Normal file
View file

@ -0,0 +1,26 @@
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://loki:3100/loki/api/v1/push
- url: http://vlogs:9428/insert/loki/api/v1/push?_stream_fields=hostname,application
# batchwait: 5s
# batchsize: 5242880
scrape_configs:
- job_name: syslog
syslog:
listen_address: 0.0.0.0:5140
idle_timeout: 12h
use_incoming_timestamp: true
labels:
job: syslog
relabel_configs:
- source_labels: [__syslog_message_hostname]
target_label: hostname
- source_labels: [__syslog_message_app_name]
target_label: application

65
deployment/logs-benchmark/readme.md
View file

@ -1,23 +1,44 @@
# Benchmark for VictoriaLogs
Benchmark compares VictoriaLogs with ELK stack and Grafana Loki.
Benchmark is based on:
- Logs from this repository - https://github.com/logpai/loghub
- [logs generator](./generator)
For ELK suite it uses:
- filebeat - https://www.elastic.co/beats/filebeat
- elastic + kibana
- [logs generator](./generator)
For Grafana Loki suite it uses:
- [Promtail](https://grafana.com/docs/loki/latest/send-data/promtail/)
- rsyslog - required to push logs in RFC5424 format to Promtail
- [Loki](https://grafana.com/oss/loki/)
## How it works
docker-compose.yml contains:
[docker-compose.yml](./docker-compose.yml) contains common configurations for all suites:
- VictoriaLogs instance
- vmsingle - port forwarded to `localhost:8428` to see UI
- exporters for system metris
ELK suite uses [docker-compose-elk.yml](./docker-compose-elk.yml) with the following services:
- [logs generator](./generator) which generates logs and sends them to filebeat instances via syslog
- 2 filebeat instances - one for elastic and one for VictoriaLogs.
- elastic instance
- VictoriaLogs instance
- kibana instance - port forwarded to `localhost:5601` to see UI
- vmsingle - port forwarded to `localhost:8428` to see UI
- [logs generator](./generator)
- exporters for filebeat/system
Loki suite uses [docker-compose-loki.yml](./docker-compose-loki.yml) with the following services:
- [logs generator](./generator) which generates logs and sends them rsyslog
- rsyslog instance - sends logs to Promtail
- Promtail instance - sends logs to Loki and VictoriaLogs
- Loki instance
[Logs generator](./generator) generates logs based on logs located at `./source_logs/logs` and sends them to filebeat
instances via syslog.
@ -44,6 +65,7 @@ Note that with logs listed in `download.sh` it will require 49GB of free space:
If it is needed to minimize disk footprint, you can download only some of them by commenting out lines in `download.sh`.
Unarchived logs size per file for reference:
```shell
2.3M Linux.log
73M SSH.log
@ -52,12 +74,11 @@ Unarchived logs size per file for reference:
13G hadoop-*.log
```
2. (optional) If needed, adjust amount of logs sent by generator by modifying `-outputRateLimitItems` and
`outputRateLimitPeriod` parameters in [docker-compose.yml](./docker-compose.yml). By default, it is configured to
send 10000 logs per second.
3. Build victoria-logs image and adjust `image` parameter in [docker-compose.yml](./docker-compose.yml):
3. (optional) Build victoria-logs image and adjust `image` parameter in [docker-compose.yml](./docker-compose.yml):
```shell
make package-victoria-logs
@ -74,8 +95,30 @@ output.elasticsearch:
hosts: [ "http://vlogs:9428/insert/elasticsearch/" ]
```
4. Run `docker-compose up -d` to start benchmark.
4. Choose a suite to run.
5. Navigate to `http://localhost:3000/d/hkm6P6_4z/elastic-vs-vlogs` to see Grafana dashboard with resource usage comparison.
In order to run ELK suite use the following command:
```
make docker-up-elk
```
In order to run Loki suite use the following command:
```
make docker-up-loki
```
![grafana-dashboard.png](grafana-dashboard.png)
5. Navigate to `http://localhost:3000/` to see Grafana dashboards with resource usage
comparison.
Navigate to `http://localhost:3000/d/hkm6P6_4z/elastic-vs-vlogs` to see ELK suite results.
Navigate to `http://localhost:3000/d/hkm6P6_4y/loki-vs-vlogs` to see Loki suite results.
Example results vs ELK:
![elk-grafana-dashboard.png](results/elk-grafana-dashboard.png)
Example results vs Loki:
![loki-grafana-dashboard.png](results/loki-grafana-dashboard.png)

0
deployment/logs-benchmark/grafana-dashboard.png → deployment/logs-benchmark/results/elk-grafana-dashboard.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 157 KiB

After

Width:  |  Height:  |  Size: 157 KiB

BIN
deployment/logs-benchmark/results/loki-grafana-dashboard.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 176 KiB

9
deployment/logs-benchmark/rsyslog/Dockerfile Normal file
View file

@ -0,0 +1,9 @@
FROM alpine:3
RUN apk add --no-cache rsyslog rsyslog-tls tzdata
COPY rsyslog.conf /etc/rsyslog.conf
VOLUME /var/run/rsyslog/dev
EXPOSE 514 10514
CMD ["/usr/sbin/rsyslogd", "-n"]

12
deployment/logs-benchmark/rsyslog/rsyslog.conf Normal file
View file

@ -0,0 +1,12 @@
module(load="imudp")
input(type="imudp" port="514")
module(load="imtcp")
input(type="imtcp" port="514")
*.* action(type="omfwd"
protocol="tcp" target="promtail" port="5140"
Template="RSYSLOG_SyslogProtocol23Format"
TCP_Framing="octet-counted" KeepAlive="on"
action.resumeRetryCount="-1"
queue.type="linkedlist" queue.size="1000000")

6
deployment/logs-benchmark/vmsingle/promscrape.yml
View file

@ -22,3 +22,9 @@ scrape_configs:
static_configs:
- targets: ['node-exporter:9100']
- targets: ['du-exporter:9995']
- job_name: 'loki'
static_configs:
- targets: ['loki:3100']
- job_name: 'promtail'
static_configs:
- targets: ['promtail:9080']