diff --git a/deployment/logs-benchmark/docker-compose.yml b/deployment/logs-benchmark/docker-compose.yml new file mode 100644 index 000000000..0d2763b75 --- /dev/null +++ b/deployment/logs-benchmark/docker-compose.yml @@ -0,0 +1,134 @@ +version: '3' + +services: + filebeat-elastic: + image: docker.elastic.co/beats/filebeat:8.8.0 + restart: on-failure + volumes: + - ./elk/filebeat/filebeat-elastic.yml:/usr/share/filebeat/filebeat.yml:ro + depends_on: [ elastic ] + + filebeat-vlogs: + image: docker.elastic.co/beats/filebeat:8.8.0 + restart: on-failure + volumes: + - ./elk/filebeat/filebeat-vlogs.yml:/usr/share/filebeat/filebeat.yml:ro + depends_on: [ vlogs ] + + generator: + image: golang:1.20-alpine + restart: always + working_dir: /go/src/app + volumes: + - ./generator:/go/src/app + - ./source_logs:/go/src/source_logs + command: + - go + - run + - main.go + - -logsPath=/go/src/source_logs/logs + - -outputRateLimitItems=10000 + - -outputRateLimitPeriod=1s + - -syslog.addr=filebeat-elastic:12345 + - -syslog.addr2=filebeat-vlogs:12345 + - -logs.randomSuffix=false + depends_on: [ filebeat-vlogs, filebeat-elastic ] + + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0 + volumes: + - ./elk/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml + - elastic:/usr/share/elasticsearch/data + environment: + ES_JAVA_OPTS: "-Xmx2048m" + + # Run `make package-victoria-logs` to build victoria-logs image + vlogs: + image: docker.io/victoriametrics/victoria-logs:heads-logs-0-gcc3fa9cd3 + volumes: + - vlogs:/vlogs + ports: + - '9428:9428' + command: + - -storageDataPath=/vlogs + + kibana: + image: docker.elastic.co/kibana/kibana:8.8.0 + volumes: + - ./elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml + ports: + - '5601:5601' + depends_on: [ elastic ] + + beat-exporter-elastic: + image: trustpilot/beat-exporter:0.4.0 + command: + - -beat.uri=http://filebeat-elastic:5066 + + beat-exporter-vlogs: + image: trustpilot/beat-exporter:0.4.0 + command: + - -beat.uri=http://filebeat-vlogs:5066 + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.47.0 + restart: unless-stopped + privileged: true + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + + node-exporter: + image: prom/node-exporter:latest + restart: unless-stopped + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + + du-exporter: + image: ghcr.io/dundee/disk_usage_exporter/disk_usage_exporter-c4084307c537335c2ddb6f4b9b527422:latest + restart: unless-stopped + user: "root" + volumes: + - /var/lib/docker/volumes:/var/lib/docker/volumes:ro + - ./du/config.yml:/config.yml:ro + command: + - '--config=/config.yml' + + vmsingle: + image: victoriametrics/victoria-metrics:v1.91.2 + ports: + - '8428:8428' + command: + - -storageDataPath=/vmsingle + - -promscrape.config=/promscrape.yml + - -promscrape.maxScrapeSize=1Gb + volumes: + - vmsingle:/vmsingle + - ./vmsingle/promscrape.yml:/promscrape.yml + + grafana: + image: grafana/grafana:9.2.7 + depends_on: [vmsingle] + ports: + - 3000:3000 + volumes: + - grafanadata:/var/lib/grafana + - ./grafana/provisioning/:/etc/grafana/provisioning/ + - ./grafana/dashboards:/var/lib/grafana/dashboards/ + restart: always + +volumes: + elastic: + vlogs: + vmsingle: + grafanadata: {} diff --git a/deployment/logs-benchmark/du/config.yml b/deployment/logs-benchmark/du/config.yml new file mode 100644 index 000000000..04549df1b --- /dev/null +++ b/deployment/logs-benchmark/du/config.yml @@ -0,0 +1,3 @@ +analyzed-path: /var/lib/docker/volumes +bind-address: 0.0.0.0:9995 +dir-level: 1 diff --git a/deployment/logs-benchmark/elk/elastic/elasticsearch.yml b/deployment/logs-benchmark/elk/elastic/elasticsearch.yml new file mode 100644 index 000000000..4f2914833 --- /dev/null +++ b/deployment/logs-benchmark/elk/elastic/elasticsearch.yml @@ -0,0 +1,4 @@ +cluster.name: "bench" +network.host: 0.0.0.0 +xpack.security.enabled: false +discovery.type: single-node diff --git a/deployment/logs-benchmark/elk/filebeat/filebeat-elastic.yml b/deployment/logs-benchmark/elk/filebeat/filebeat-elastic.yml new file mode 100644 index 000000000..a8efb45f8 --- /dev/null +++ b/deployment/logs-benchmark/elk/filebeat/filebeat-elastic.yml @@ -0,0 +1,15 @@ +filebeat.inputs: + - type: syslog + format: rfc3164 + protocol.tcp: + host: "0.0.0.0:12345" + +output.elasticsearch: + hosts: [ "http://elastic:9200" ] + worker: 5 + bulk_max_size: 1000 + +http: + enabled: true + host: 0.0.0.0 + port: 5066 diff --git a/deployment/logs-benchmark/elk/filebeat/filebeat-vlogs.yml b/deployment/logs-benchmark/elk/filebeat/filebeat-vlogs.yml new file mode 100644 index 000000000..7bc4d755f --- /dev/null +++ b/deployment/logs-benchmark/elk/filebeat/filebeat-vlogs.yml @@ -0,0 +1,19 @@ +filebeat.inputs: + - type: syslog + format: rfc3164 + protocol.tcp: + host: "0.0.0.0:12345" + +output.elasticsearch: + hosts: [ "http://vlogs:9428/insert/elasticsearch/" ] + worker: 5 + bulk_max_size: 1000 + parameters: + _msg_field: "message" + _time_field: "@timestamp" + _stream_fields: "host.name,process.program,process.pid" + +http: + enabled: true + host: 0.0.0.0 + port: 5066 diff --git a/deployment/logs-benchmark/elk/kibana/kibana.yml b/deployment/logs-benchmark/elk/kibana/kibana.yml new file mode 100644 index 000000000..dbbdc4c6e --- /dev/null +++ b/deployment/logs-benchmark/elk/kibana/kibana.yml @@ -0,0 +1,3 @@ +server.name: kibana +server.host: "0.0.0.0" +elasticsearch.hosts: [ "http://elastic:9200" ] diff --git a/deployment/logs-benchmark/generator/main.go b/deployment/logs-benchmark/generator/main.go new file mode 100644 index 000000000..44401c74b --- /dev/null +++ b/deployment/logs-benchmark/generator/main.go @@ -0,0 +1,101 @@ +package main + +import ( + "bufio" + "encoding/binary" + "flag" + "fmt" + "log" + "log/syslog" + "math/rand" + "net" + "os" + "strconv" + "strings" + "time" +) + +var ( + logsPath = flag.String("logsPath", "", "Path to logs directory") + syslogAddr = flag.String("syslog.addr", "logstash:12345", "Addr to send logs to") + syslogAddr2 = flag.String("syslog.addr2", "logstash:12345", "Addr to send logs to") + randomSuffix = flag.Bool("logs.randomSuffix", false, "Whether to add a random suffix to a log line") + + outputRateLimitItems = flag.Int("outputRateLimitItems", 100, "Number of items to send per second") + outputRateLimitPeriod = flag.Duration("outputRateLimitPeriod", time.Second, "Period of time to send items") +) + +func main() { + flag.Parse() + startedAt := time.Now().Unix() + + logFiles, err := os.ReadDir(*logsPath) + if err != nil { + panic(fmt.Errorf("error reading directory %s:%w", *logsPath, err)) + } + + sourceFiles := make([]string, 0) + + for _, logFile := range logFiles { + if strings.HasSuffix(logFile.Name(), ".log") { + sourceFiles = append(sourceFiles, logFile.Name()) + } + } + log.Printf("sourceFiles: %v", sourceFiles) + log.Printf("running with rate limit: %d items per %s", *outputRateLimitItems, *outputRateLimitPeriod) + + limitTicker := time.NewTicker(*outputRateLimitPeriod) + limitItems := *outputRateLimitItems + limitter := make(chan struct{}, limitItems) + go func() { + for { + select { + case <-limitTicker.C: + for i := 0; i < limitItems; i++ { + limitter <- struct{}{} + } + } + } + }() + + for _, sourceFile := range sourceFiles { + log.Printf("sourceFile: %s", sourceFile) + f, err := os.Open(*logsPath + "/" + sourceFile) + if err != nil { + panic(err) + } + + syslogTag := "logs-benchmark-" + sourceFile + "-" + strconv.FormatInt(startedAt, 10) + logger, err := syslog.Dial("tcp", *syslogAddr, syslog.LOG_INFO, syslogTag) + if err != nil { + panic(fmt.Errorf("error dialing syslog: %w", err)) + } + logger2, err := syslog.Dial("tcp", *syslogAddr2, syslog.LOG_INFO, syslogTag) + if err != nil { + panic(fmt.Errorf("error dialing syslog: %w", err)) + } + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + <-limitter + line := scanner.Text() + if *randomSuffix { + line = line + " " + randomString() + } + _ = logger.Info(line) + _ = logger2.Info(line) + } + + logger.Close() + logger2.Close() + } + +} + +func randomString() string { + buf := make([]byte, 4) + ip := rand.Uint32() + + binary.LittleEndian.PutUint32(buf, ip) + return net.IP(buf).String() +} diff --git a/deployment/logs-benchmark/grafana-dashboard.png b/deployment/logs-benchmark/grafana-dashboard.png new file mode 100644 index 000000000..659e83c23 Binary files /dev/null and b/deployment/logs-benchmark/grafana-dashboard.png differ diff --git a/deployment/logs-benchmark/grafana/dashboards/comparison.json b/deployment/logs-benchmark/grafana/dashboards/comparison.json new file mode 100644 index 000000000..2ccd3815a --- /dev/null +++ b/deployment/logs-benchmark/grafana/dashboards/comparison.json @@ -0,0 +1,680 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(rate(filebeat_libbeat_output_events{type=\"acked\"})) by (instance, type) *100", + "legendFormat": "{{instance}} - {{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Filebeat items", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*vlogs.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*elastic.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$containers\"})) by (name) *100", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*vlogs.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*elastic.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(container_memory_rss{name=~\"$containers\"}) by (name)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*vlogs.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*elastic.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(node_disk_usage_bytes{path=~\"$containers_selector\"}) by (path)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk space used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*vlogs.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*elastic.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(delta(node_disk_usage_bytes{path=~\"$containers_selector\"}[1m])) by (path)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk space used", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "hide": 2, + "name": "containers_selector", + "query": ".*(vlogs|elastic).*", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": { + "selected": true, + "text": [ + "logs-benchmark-elastic-1", + "logs-benchmark-vlogs-1" + ], + "value": [ + "logs-benchmark-elastic-1", + "logs-benchmark-vlogs-1" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "definition": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "containers", + "options": [], + "query": { + "query": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": ".*vlogs|elastic.*", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Elastic vs VLogs", + "uid": "hkm6P6_4z", + "version": 1, + "weekStart": "" +} diff --git a/deployment/logs-benchmark/grafana/dashboards/stats.json b/deployment/logs-benchmark/grafana/dashboards/stats.json new file mode 100644 index 000000000..147fca8fe --- /dev/null +++ b/deployment/logs-benchmark/grafana/dashboards/stats.json @@ -0,0 +1,393 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*vlogs.*" + }, + "properties": [ + { + "id": "displayName", + "value": "victoria-logs" + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*elastic.*" + }, + "properties": [ + { + "id": "displayName", + "value": "elasticsearch" + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.2.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$containers\"}[5m])) by (name) * 100", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*vlogs.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "displayName", + "value": "victoria-logs" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*elastic.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "displayName", + "value": "elasticsearch" + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 5, + "x": 5, + "y": 0 + }, + "id": 3, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.2.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(container_memory_rss{name=~\"$containers\"}[5m]) by (name)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*vlogs.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "displayName", + "value": "victoria-logs" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*elastic.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "displayName", + "value": "elasticsearch" + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 5, + "x": 10, + "y": 0 + }, + "id": 5, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.2.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "editorMode": "code", + "expr": "sum(node_disk_usage_bytes{path=~\"$containers_selector\"}[5m]) by (path)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk space used", + "type": "bargauge" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "hide": 2, + "name": "containers_selector", + "query": ".*(vlogs|elastic).*", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": { + "selected": true, + "text": [ + "logs-benchmark-elastic-1", + "logs-benchmark-vlogs-1" + ], + "value": [ + "logs-benchmark-elastic-1", + "logs-benchmark-vlogs-1" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "P4169E866C3094E38" + }, + "definition": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "containers", + "options": [], + "query": { + "query": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": ".*vlogs|elastic.*", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Elastic vs VLogs - stats only", + "uid": "hkm6P6_4J", + "version": 2, + "weekStart": "" +} diff --git a/deployment/logs-benchmark/grafana/provisioning/dashboards/dashboard.yml b/deployment/logs-benchmark/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 000000000..c071995d0 --- /dev/null +++ b/deployment/logs-benchmark/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: +- name: Prometheus + orgId: 1 + folder: '' + type: file + options: + path: /var/lib/grafana/dashboards diff --git a/deployment/logs-benchmark/grafana/provisioning/datasources/datasource.yml b/deployment/logs-benchmark/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 000000000..11560e6f8 --- /dev/null +++ b/deployment/logs-benchmark/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: VictoriaMetrics + type: prometheus + access: proxy + url: http://vmsingle:8428 + isDefault: true diff --git a/deployment/logs-benchmark/readme.md b/deployment/logs-benchmark/readme.md new file mode 100644 index 000000000..63c48ce22 --- /dev/null +++ b/deployment/logs-benchmark/readme.md @@ -0,0 +1,86 @@ +# Benchmark for VictoriaLogs + +Benchmark is based on: + +- Logs from this repository - https://github.com/logpai/loghub +- filebeat - https://www.elastic.co/beats/filebeat +- elastic + kibana +- [logs generator](./generator) + +## How it works + +docker-compose.yml contains: + +- 2 filebeat instances - one for elastic and one for VictoriaLogs. +- elastic instance +- VictoriaLogs instance +- kibana instance - port forwarded to `localhost:5601` to see UI +- vmsingle - port forwarded to `localhost:8428` to see UI +- [logs generator](./generator) +- exporters for filebeat/system + +[Logs generator](./generator) generates logs based on logs located at `./source_logs/logs` and sends them to filebeat +instances via syslog. +Logs are generated by reading files line by line, adding randomized suffix to each line and sending them to filebeat via +syslog. +By default, generator will exit once all files are read. `docker-compose` will restart it and files will be read again +generating new logs. + +Each filebeat than writes logs to elastic and VictoriaLogs via elasticsearch-compatible API. + +## How to run + +1. Download and unarchive logs by running: + +```shell +cd source_logs +bash download.sh +``` + +Note that with logs listed in `download.sh` it will require 49GB of free space: + +- 3GB for archives +- 46GB for unarchived logs + +If it is needed to minimize disk footprint, you can download only some of them by commenting out lines in `download.sh`. +Unarchived logs size per file for reference: +```shell +2.3M Linux.log + 73M SSH.log + 32G Thunderbird.log +5.1M Apache.log + 13G hadoop-*.log +``` + + +2. (optional) If needed, adjust amount of logs sent by generator by modifying `-outputRateLimitItems` and + `outputRateLimitPeriod` parameters in [docker-compose.yml](./docker-compose.yml). By default, it is configured to + send 10000 logs per second. + +3. Build victoria-logs image and adjust `image` parameter in [docker-compose.yml](./docker-compose.yml): + +```shell +make package-victoria-logs + +=> exporting to image 0.5s +=> => exporting layers 0.5s +=> => writing image sha256:3ef5d4d5dfc767353d897abba25314ae7820b6f4d8422b5b2a1342e7be5dd579 0.0s +=> => naming to docker.io/victoriametrics/victoria-logs:heads-logs-0-gcc3fa9cd3-dirty-cbfad271 +``` + +Image name should be replaced at `vlogs` service in [docker-compose.yml](./docker-compose.yml). + +It is also possible to configure filebeat to send logs to VictoriaLogs running on local machine. +To do this modify [filebeat config for vlogs](./elk/filebeat/filebeat-vlogs.yml) and replace `vlogs` address +with address of local VictoriaLogs instance: + +```yaml +output.elasticsearch: + hosts: [ "http://vlogs:9428/insert/elasticsearch/" ] +``` + +4. Run `docker-compose up -d` to start benchmark. + +5. Navigate to `http://localhost:3000/d/hkm6P6_4z/elastic-vs-vlogs` to see Grafana dashboard with resource usage comparison. + +![grafana-dashboard.png](grafana-dashboard.png) diff --git a/deployment/logs-benchmark/source_logs/.gitignore b/deployment/logs-benchmark/source_logs/.gitignore new file mode 100644 index 000000000..730022a6c --- /dev/null +++ b/deployment/logs-benchmark/source_logs/.gitignore @@ -0,0 +1,2 @@ +*.log +*.tar.gz diff --git a/deployment/logs-benchmark/source_logs/download.sh b/deployment/logs-benchmark/source_logs/download.sh new file mode 100755 index 000000000..b89015112 --- /dev/null +++ b/deployment/logs-benchmark/source_logs/download.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +# Unarchived size: 5.1M Apache.log +if [ ! -f Apache.tar.gz ]; then + curl -o Apache.tar.gz -C - https://zenodo.org/record/3227177/files/Apache.tar.gz?download=1 +fi + +# Unarchived size: 13G hadoop-*.log +if [ ! -f HDFS_2.tar.gz ]; then + curl -o HDFS_2.tar.gz -C - https://zenodo.org/record/3227177/files/HDFS_2.tar.gz?download=1 +fi + +# Unarchived size: 2.3M Linux.log +if [ ! -f Linux.tar.gz ]; then + curl -o Linux.tar.gz -C - https://zenodo.org/record/3227177/files/Linux.tar.gz?download=1 +fi + +# Unarchived size: 32G Thunderbird.log +if [ ! -f Thunderbird.tar.gz ]; then + curl -o Thunderbird.tar.gz -C - https://zenodo.org/record/3227177/files/Thunderbird.tar.gz?download=1 +fi + +# Unarchived size: 73M SSH.log +if [ ! -f SSH.tar.gz ]; then + curl -o SSH.tar.gz -C - https://zenodo.org/record/3227177/files/SSH.tar.gz?download=1 +fi + +mkdir -p logs + +for file in *.tar.gz; do + tar -xzf $file -C logs +done diff --git a/deployment/logs-benchmark/vmsingle/promscrape.yml b/deployment/logs-benchmark/vmsingle/promscrape.yml new file mode 100644 index 000000000..6f289cd93 --- /dev/null +++ b/deployment/logs-benchmark/vmsingle/promscrape.yml @@ -0,0 +1,24 @@ +scrape_configs: + - job_name: "filebeat" + scrape_interval: 30s + static_configs: + - targets: + - beat-exporter-elastic:9479 + - beat-exporter-vlogs:9479 + - job_name: "victoria-logs" + scrape_interval: 30s + static_configs: + - targets: + - vlogs:9428 + - job_name: "cadvisor" + scrape_interval: 30s + metric_relabel_configs: + - action: labeldrop + regex: "container_label_.*" + static_configs: + - targets: + - cadvisor:8080 + - job_name: 'node' + static_configs: + - targets: ['node-exporter:9100'] + - targets: ['du-exporter:9995']