github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

deployment/logs-benchmark: add benchmark setup (#4470)

Browse source 
* deployment/logs-benchmark: add benchmark setup

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

* deployment/logs-benchmark: update readme info

- add sizes of uncompressed logs
- add more info about dashboard

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

---------

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
This commit is contained in:
Zakhar Bessarab 2023-06-21 11:04:03 +04:00 committed by GitHub
parent 95b54bb178
commit 1ebfea34d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 1515 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

134
deployment/logs-benchmark/docker-compose.yml Normal file
View file

@ -0,0 +1,134 @@
version: '3'
services:
filebeat-elastic:
image: docker.elastic.co/beats/filebeat:8.8.0
restart: on-failure
volumes:
- ./elk/filebeat/filebeat-elastic.yml:/usr/share/filebeat/filebeat.yml:ro
depends_on: [ elastic ]
filebeat-vlogs:
image: docker.elastic.co/beats/filebeat:8.8.0
restart: on-failure
volumes:
- ./elk/filebeat/filebeat-vlogs.yml:/usr/share/filebeat/filebeat.yml:ro
depends_on: [ vlogs ]
generator:
image: golang:1.20-alpine
restart: always
working_dir: /go/src/app
volumes:
- ./generator:/go/src/app
- ./source_logs:/go/src/source_logs
command:
- go
- run
- main.go
- -logsPath=/go/src/source_logs/logs
- -outputRateLimitItems=10000
- -outputRateLimitPeriod=1s
- -syslog.addr=filebeat-elastic:12345
- -syslog.addr2=filebeat-vlogs:12345
- -logs.randomSuffix=false
depends_on: [ filebeat-vlogs, filebeat-elastic ]
elastic:
image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
volumes:
- ./elk/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- elastic:/usr/share/elasticsearch/data
environment:
ES_JAVA_OPTS: "-Xmx2048m"
# Run `make package-victoria-logs` to build victoria-logs image
vlogs:
image: docker.io/victoriametrics/victoria-logs:heads-logs-0-gcc3fa9cd3
volumes:
- vlogs:/vlogs
ports:
- '9428:9428'
command:
- -storageDataPath=/vlogs
kibana:
image: docker.elastic.co/kibana/kibana:8.8.0
volumes:
- ./elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
ports:
- '5601:5601'
depends_on: [ elastic ]
beat-exporter-elastic:
image: trustpilot/beat-exporter:0.4.0
command:
- -beat.uri=http://filebeat-elastic:5066
beat-exporter-vlogs:
image: trustpilot/beat-exporter:0.4.0
command:
- -beat.uri=http://filebeat-vlogs:5066
cadvisor:
image: gcr.io/cadvisor/cadvisor:v0.47.0
restart: unless-stopped
privileged: true
volumes:
- /:/rootfs:ro
- /var/run:/var/run:ro
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
- /dev/disk/:/dev/disk:ro
node-exporter:
image: prom/node-exporter:latest
restart: unless-stopped
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.rootfs=/rootfs'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
du-exporter:
image: ghcr.io/dundee/disk_usage_exporter/disk_usage_exporter-c4084307c537335c2ddb6f4b9b527422:latest
restart: unless-stopped
user: "root"
volumes:
- /var/lib/docker/volumes:/var/lib/docker/volumes:ro
- ./du/config.yml:/config.yml:ro
command:
- '--config=/config.yml'
vmsingle:
image: victoriametrics/victoria-metrics:v1.91.2
ports:
- '8428:8428'
command:
- -storageDataPath=/vmsingle
- -promscrape.config=/promscrape.yml
- -promscrape.maxScrapeSize=1Gb
volumes:
- vmsingle:/vmsingle
- ./vmsingle/promscrape.yml:/promscrape.yml
grafana:
image: grafana/grafana:9.2.7
depends_on: [vmsingle]
ports:
- 3000:3000
volumes:
- grafanadata:/var/lib/grafana
- ./grafana/provisioning/:/etc/grafana/provisioning/
- ./grafana/dashboards:/var/lib/grafana/dashboards/
restart: always
volumes:
elastic:
vlogs:
vmsingle:
grafanadata: {}

3
deployment/logs-benchmark/du/config.yml Normal file
View file

@ -0,0 +1,3 @@
analyzed-path: /var/lib/docker/volumes
bind-address: 0.0.0.0:9995
dir-level: 1

4
deployment/logs-benchmark/elk/elastic/elasticsearch.yml Normal file
View file

@ -0,0 +1,4 @@
cluster.name: "bench"
network.host: 0.0.0.0
xpack.security.enabled: false
discovery.type: single-node

15
deployment/logs-benchmark/elk/filebeat/filebeat-elastic.yml Normal file
View file

@ -0,0 +1,15 @@
filebeat.inputs:
- type: syslog
format: rfc3164
protocol.tcp:
host: "0.0.0.0:12345"
output.elasticsearch:
hosts: [ "http://elastic:9200" ]
worker: 5
bulk_max_size: 1000
http:
enabled: true
host: 0.0.0.0
port: 5066

19
deployment/logs-benchmark/elk/filebeat/filebeat-vlogs.yml Normal file
View file

@ -0,0 +1,19 @@
filebeat.inputs:
- type: syslog
format: rfc3164
protocol.tcp:
host: "0.0.0.0:12345"
output.elasticsearch:
hosts: [ "http://vlogs:9428/insert/elasticsearch/" ]
worker: 5
bulk_max_size: 1000
parameters:
_msg_field: "message"
_time_field: "@timestamp"
_stream_fields: "host.name,process.program,process.pid"
http:
enabled: true
host: 0.0.0.0
port: 5066

3
deployment/logs-benchmark/elk/kibana/kibana.yml Normal file
View file

@ -0,0 +1,3 @@
server.name: kibana
server.host: "0.0.0.0"
elasticsearch.hosts: [ "http://elastic:9200" ]

101
deployment/logs-benchmark/generator/main.go Normal file
View file

@ -0,0 +1,101 @@
package main
import (
"bufio"
"encoding/binary"
"flag"
"fmt"
"log"
"log/syslog"
"math/rand"
"net"
"os"
"strconv"
"strings"
"time"
)
var (
logsPath = flag.String("logsPath", "", "Path to logs directory")
syslogAddr = flag.String("syslog.addr", "logstash:12345", "Addr to send logs to")
syslogAddr2 = flag.String("syslog.addr2", "logstash:12345", "Addr to send logs to")
randomSuffix = flag.Bool("logs.randomSuffix", false, "Whether to add a random suffix to a log line")
outputRateLimitItems = flag.Int("outputRateLimitItems", 100, "Number of items to send per second")
outputRateLimitPeriod = flag.Duration("outputRateLimitPeriod", time.Second, "Period of time to send items")
)
func main() {
flag.Parse()
startedAt := time.Now().Unix()
logFiles, err := os.ReadDir(*logsPath)
if err != nil {
panic(fmt.Errorf("error reading directory %s:%w", *logsPath, err))
}
sourceFiles := make([]string, 0)
for _, logFile := range logFiles {
if strings.HasSuffix(logFile.Name(), ".log") {
sourceFiles = append(sourceFiles, logFile.Name())
}
}
log.Printf("sourceFiles: %v", sourceFiles)
log.Printf("running with rate limit: %d items per %s", *outputRateLimitItems, *outputRateLimitPeriod)
limitTicker := time.NewTicker(*outputRateLimitPeriod)
limitItems := *outputRateLimitItems
limitter := make(chan struct{}, limitItems)
go func() {
for {
select {
case <-limitTicker.C:
for i := 0; i < limitItems; i++ {
limitter <- struct{}{}
}
}
}
}()
for _, sourceFile := range sourceFiles {
log.Printf("sourceFile: %s", sourceFile)
f, err := os.Open(*logsPath + "/" + sourceFile)
if err != nil {
panic(err)
}
syslogTag := "logs-benchmark-" + sourceFile + "-" + strconv.FormatInt(startedAt, 10)
logger, err := syslog.Dial("tcp", *syslogAddr, syslog.LOG_INFO, syslogTag)
if err != nil {
panic(fmt.Errorf("error dialing syslog: %w", err))
}
logger2, err := syslog.Dial("tcp", *syslogAddr2, syslog.LOG_INFO, syslogTag)
if err != nil {
panic(fmt.Errorf("error dialing syslog: %w", err))
}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
<-limitter
line := scanner.Text()
if *randomSuffix {
line = line + " " + randomString()
}
_ = logger.Info(line)
_ = logger2.Info(line)
}
logger.Close()
logger2.Close()
}
}
func randomString() string {
buf := make([]byte, 4)
ip := rand.Uint32()
binary.LittleEndian.PutUint32(buf, ip)
return net.IP(buf).String()
}

BIN
deployment/logs-benchmark/grafana-dashboard.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 153 KiB

680
deployment/logs-benchmark/grafana/dashboards/comparison.json Normal file
View file

@ -0,0 +1,680 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(rate(filebeat_libbeat_output_events{type=\"acked\"})) by (instance, type) *100",
"legendFormat": "{{instance}} - {{type}}",
"range": true,
"refId": "A"
}
],
"title": "Filebeat items",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "percent"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*vlogs.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*elastic.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 7
},
"id": 2,
"options": {
"legend": {
"calcs": [
"mean",
"min",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$containers\"})) by (name) *100",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*vlogs.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*elastic.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 12,
"x": 12,
"y": 7
},
"id": 3,
"options": {
"legend": {
"calcs": [
"min",
"max",
"mean"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(container_memory_rss{name=~\"$containers\"}) by (name)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Memory Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "decbytes"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*vlogs.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*elastic.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 18
},
"id": 5,
"options": {
"legend": {
"calcs": [
"min",
"max",
"mean"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(node_disk_usage_bytes{path=~\"$containers_selector\"}) by (path)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Disk space used",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "decbytes"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*vlogs.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*elastic.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 12,
"x": 12,
"y": 18
},
"id": 6,
"options": {
"legend": {
"calcs": [
"min",
"max",
"mean"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(delta(node_disk_usage_bytes{path=~\"$containers_selector\"}[1m])) by (path)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Disk space used",
"type": "timeseries"
}
],
"refresh": "10s",
"schemaVersion": 37,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"hide": 2,
"name": "containers_selector",
"query": ".*(vlogs|elastic).*",
"skipUrlSync": false,
"type": "constant"
},
{
"current": {
"selected": true,
"text": [
"logs-benchmark-elastic-1",
"logs-benchmark-vlogs-1"
],
"value": [
"logs-benchmark-elastic-1",
"logs-benchmark-vlogs-1"
]
},
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"definition": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
"hide": 0,
"includeAll": true,
"label": "",
"multi": true,
"name": "containers",
"options": [],
"query": {
"query": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": ".*vlogs|elastic.*",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-15m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Elastic vs VLogs",
"uid": "hkm6P6_4z",
"version": 1,
"weekStart": ""
}

393
deployment/logs-benchmark/grafana/dashboards/stats.json Normal file
View file

@ -0,0 +1,393 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "percent"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*vlogs.*"
},
"properties": [
{
"id": "displayName",
"value": "victoria-logs"
},
{
"id": "color",
"value": {
"fixedColor": "purple",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*elastic.*"
},
"properties": [
{
"id": "displayName",
"value": "elasticsearch"
},
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 5,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"displayMode": "gradient",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true
},
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$containers\"}[5m])) by (name) * 100",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "CPU Usage",
"type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*vlogs.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "purple",
"mode": "fixed"
}
},
{
"id": "displayName",
"value": "victoria-logs"
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*elastic.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
},
{
"id": "displayName",
"value": "elasticsearch"
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 5,
"x": 5,
"y": 0
},
"id": 3,
"options": {
"displayMode": "gradient",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true
},
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(container_memory_rss{name=~\"$containers\"}[5m]) by (name)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Memory Usage",
"type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "decbytes"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*vlogs.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "purple",
"mode": "fixed"
}
},
{
"id": "displayName",
"value": "victoria-logs"
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*elastic.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
},
{
"id": "displayName",
"value": "elasticsearch"
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 5,
"x": 10,
"y": 0
},
"id": 5,
"options": {
"displayMode": "gradient",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true
},
"pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"editorMode": "code",
"expr": "sum(node_disk_usage_bytes{path=~\"$containers_selector\"}[5m]) by (path)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Disk space used",
"type": "bargauge"
}
],
"refresh": false,
"schemaVersion": 37,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"hide": 2,
"name": "containers_selector",
"query": ".*(vlogs|elastic).*",
"skipUrlSync": false,
"type": "constant"
},
{
"current": {
"selected": true,
"text": [
"logs-benchmark-elastic-1",
"logs-benchmark-vlogs-1"
],
"value": [
"logs-benchmark-elastic-1",
"logs-benchmark-vlogs-1"
]
},
"datasource": {
"type": "prometheus",
"uid": "P4169E866C3094E38"
},
"definition": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
"hide": 0,
"includeAll": true,
"label": "",
"multi": true,
"name": "containers",
"options": [],
"query": {
"query": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": ".*vlogs|elastic.*",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Elastic vs VLogs - stats only",
"uid": "hkm6P6_4J",
"version": 2,
"weekStart": ""
}

9
deployment/logs-benchmark/grafana/provisioning/dashboards/dashboard.yml Normal file
View file

@ -0,0 +1,9 @@
apiVersion: 1
providers:
- name: Prometheus
orgId: 1
folder: ''
type: file
options:
path: /var/lib/grafana/dashboards

8
deployment/logs-benchmark/grafana/provisioning/datasources/datasource.yml Normal file
View file

@ -0,0 +1,8 @@
apiVersion: 1
datasources:
- name: VictoriaMetrics
type: prometheus
access: proxy
url: http://vmsingle:8428
isDefault: true

86
deployment/logs-benchmark/readme.md Normal file
View file

@ -0,0 +1,86 @@
# Benchmark for VictoriaLogs
Benchmark is based on:
- Logs from this repository - https://github.com/logpai/loghub
- filebeat - https://www.elastic.co/beats/filebeat
- elastic + kibana
- [logs generator](./generator)
## How it works
docker-compose.yml contains:
- 2 filebeat instances - one for elastic and one for VictoriaLogs.
- elastic instance
- VictoriaLogs instance
- kibana instance - port forwarded to `localhost:5601` to see UI
- vmsingle - port forwarded to `localhost:8428` to see UI
- [logs generator](./generator)
- exporters for filebeat/system
[Logs generator](./generator) generates logs based on logs located at `./source_logs/logs` and sends them to filebeat
instances via syslog.
Logs are generated by reading files line by line, adding randomized suffix to each line and sending them to filebeat via
syslog.
By default, generator will exit once all files are read. `docker-compose` will restart it and files will be read again
generating new logs.
Each filebeat than writes logs to elastic and VictoriaLogs via elasticsearch-compatible API.
## How to run
1. Download and unarchive logs by running:
```shell
cd source_logs
bash download.sh
```
Note that with logs listed in `download.sh` it will require 49GB of free space:
- 3GB for archives
- 46GB for unarchived logs
If it is needed to minimize disk footprint, you can download only some of them by commenting out lines in `download.sh`.
Unarchived logs size per file for reference:
```shell
2.3M Linux.log
73M SSH.log
32G Thunderbird.log
5.1M Apache.log
13G hadoop-*.log
```
2. (optional) If needed, adjust amount of logs sent by generator by modifying `-outputRateLimitItems` and
`outputRateLimitPeriod` parameters in [docker-compose.yml](./docker-compose.yml). By default, it is configured to
send 10000 logs per second.
3. Build victoria-logs image and adjust `image` parameter in [docker-compose.yml](./docker-compose.yml):
```shell
make package-victoria-logs
=> exporting to image 0.5s
=> => exporting layers 0.5s
=> => writing image sha256:3ef5d4d5dfc767353d897abba25314ae7820b6f4d8422b5b2a1342e7be5dd579 0.0s
=> => naming to docker.io/victoriametrics/victoria-logs:heads-logs-0-gcc3fa9cd3-dirty-cbfad271
```
Image name should be replaced at `vlogs` service in [docker-compose.yml](./docker-compose.yml).
It is also possible to configure filebeat to send logs to VictoriaLogs running on local machine.
To do this modify [filebeat config for vlogs](./elk/filebeat/filebeat-vlogs.yml) and replace `vlogs` address
with address of local VictoriaLogs instance:
```yaml
output.elasticsearch:
hosts: [ "http://vlogs:9428/insert/elasticsearch/" ]
```
4. Run `docker-compose up -d` to start benchmark.
5. Navigate to `http://localhost:3000/d/hkm6P6_4z/elastic-vs-vlogs` to see Grafana dashboard with resource usage comparison.
![grafana-dashboard.png](grafana-dashboard.png)

2
deployment/logs-benchmark/source_logs/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*.log
*.tar.gz

34
deployment/logs-benchmark/source_logs/download.sh Executable file
View file

@ -0,0 +1,34 @@
#!/bin/bash
set -ex
# Unarchived size: 5.1M Apache.log
if [ ! -f Apache.tar.gz ]; then
curl -o Apache.tar.gz -C - https://zenodo.org/record/3227177/files/Apache.tar.gz?download=1
fi
# Unarchived size: 13G hadoop-*.log
if [ ! -f HDFS_2.tar.gz ]; then
curl -o HDFS_2.tar.gz -C - https://zenodo.org/record/3227177/files/HDFS_2.tar.gz?download=1
fi
# Unarchived size: 2.3M Linux.log
if [ ! -f Linux.tar.gz ]; then
curl -o Linux.tar.gz -C - https://zenodo.org/record/3227177/files/Linux.tar.gz?download=1
fi
# Unarchived size: 32G Thunderbird.log
if [ ! -f Thunderbird.tar.gz ]; then
curl -o Thunderbird.tar.gz -C - https://zenodo.org/record/3227177/files/Thunderbird.tar.gz?download=1
fi
# Unarchived size: 73M SSH.log
if [ ! -f SSH.tar.gz ]; then
curl -o SSH.tar.gz -C - https://zenodo.org/record/3227177/files/SSH.tar.gz?download=1
fi
mkdir -p logs
for file in *.tar.gz; do
tar -xzf $file -C logs
done

24
deployment/logs-benchmark/vmsingle/promscrape.yml Normal file
View file

@ -0,0 +1,24 @@
scrape_configs:
- job_name: "filebeat"
scrape_interval: 30s
static_configs:
- targets:
- beat-exporter-elastic:9479
- beat-exporter-vlogs:9479
- job_name: "victoria-logs"
scrape_interval: 30s
static_configs:
- targets:
- vlogs:9428
- job_name: "cadvisor"
scrape_interval: 30s
metric_relabel_configs:
- action: labeldrop
regex: "container_label_.*"
static_configs:
- targets:
- cadvisor:8080
- job_name: 'node'
static_configs:
- targets: ['node-exporter:9100']
- targets: ['du-exporter:9995']