mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
Add docker compose examples: filebeat(docker, syslog), fluentbit(docker), logstash, vector(docker)
Signed-off-by: Artem Navoiev <tenmozes@gmail.com>
This commit is contained in:
Artem Navoiev
parent
7346bb4f03
commit
b024e46284
17 changed files with 1183 additions and 0 deletions
779
dashboards/victorialogs.json
Normal file
779
dashboards/victorialogs.json
Normal file
|
|
@ -0,0 +1,779 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 16,
|
||||
"panels": [],
|
||||
"title": "Stats",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"description": "How many log entries are in storage",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 2,
|
||||
"w": 5,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 10,
|
||||
"links": [],
|
||||
"maxDataPoints": 100,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"text": {},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "9.2.7",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": false,
|
||||
"expr": "sum(vl_rows)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Total log entries",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"description": "Shows the datapoints ingestion rate.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 2,
|
||||
"w": 5,
|
||||
"x": 5,
|
||||
"y": 1
|
||||
},
|
||||
"id": 22,
|
||||
"links": [],
|
||||
"maxDataPoints": 100,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"text": {},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "9.2.7",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": false,
|
||||
"expr": "sum(rate(vl_rows_ingested_total))",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Ingestion rate",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"description": "Total amount of used disk space",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 2,
|
||||
"w": 5,
|
||||
"x": 10,
|
||||
"y": 1
|
||||
},
|
||||
"id": 24,
|
||||
"links": [],
|
||||
"maxDataPoints": 100,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"text": {},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "9.2.7",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": false,
|
||||
"expr": "sum(vl_compressed_data_size_bytes)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Disk space usage",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"description": "The ratio of original data size and compressed data stored on disk",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 2,
|
||||
"w": 5,
|
||||
"x": 15,
|
||||
"y": 1
|
||||
},
|
||||
"id": 25,
|
||||
"links": [],
|
||||
"maxDataPoints": 100,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"text": {},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "9.2.7",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": false,
|
||||
"expr": " sum(vl_uncompressed_data_size_bytes) / sum(vl_compressed_data_size_bytes)",
|
||||
"format": "time_series",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Comprasion ratio",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 3
|
||||
},
|
||||
"id": 18,
|
||||
"panels": [],
|
||||
"title": "Overview",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"description": "How many datapoints are inserted into storage per second",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"links": [],
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"id": 2,
|
||||
"links": [],
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean",
|
||||
"lastNotNull"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"sortBy": "Last *",
|
||||
"sortDesc": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "9.1.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vl_rows_ingested_total)) by (type)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "{{type}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Datapoints ingestion rate ",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"description": "* `*` - unsupported query path\n* `/insert` - insert into VM\n* `/metrics` - query VL system metrics\n* `/query` - read the data",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"links": [],
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 4
|
||||
},
|
||||
"id": 14,
|
||||
"links": [],
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean",
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"sortBy": "Last *",
|
||||
"sortDesc": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "9.1.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vl_http_requests_total)) by (path) > 0",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "{{path}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Requests rate ",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"description": "Shows the amount of on-disk space occupied by data before and after compressiom",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"links": [],
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"id": 6,
|
||||
"links": [],
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean",
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"sortBy": "Last *",
|
||||
"sortDesc": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "9.1.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(vl_compressed_data_size_bytes)",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "disk usage",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Disk space usage ",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"description": "The number of the new log streams created over the last 24h",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"links": [],
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 12
|
||||
},
|
||||
"id": 26,
|
||||
"links": [],
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean",
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"sortBy": "Last *",
|
||||
"sortDesc": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "9.1.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "increase(vl_streams_created_total[1d])",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "new log streams over 24h",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Log stream churn rate",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 37,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-30m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "VictoriaLogs",
|
||||
"uid": "OqPIZTX4z",
|
||||
"version": 4,
|
||||
"weekStart": ""
|
||||
}
|
||||
22
deployment/docker/victorialogs/filebeat-docker/README.md
Normal file
22
deployment/docker/victorialogs/filebeat-docker/README.md
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# Docker compose Filebeat integration with VictoriaLogs
|
||||
|
||||
The folder contains the example of integration of [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html) with Victorialogs
|
||||
|
||||
|
||||
To spin-up environment run the following command:
|
||||
```
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
To shut down the docker-compose environment run the following command:
|
||||
```
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
The docker compose file contains the following component
|
||||
|
||||
* filebeat - fileabeat is configured to collect logs from the docker, you can find configuration in the `filebeat.yml`. It writes data in VictoriaLogs
|
||||
* filebeat-exporter - it export metrics about the filebeat
|
||||
* VictoriaLogs - the log database, it accept data from `filebeat` by elastic protocol
|
||||
* VictoriaMetrics - collect metrics from `filebeat` via `filebeat-exporter`, `VictoriaLogs` and `VictoriaMetrics`(itself)
|
||||
* grafana - it comes with two predefined dashboards for `VictoriaLogs` and `VictoriaMetrics`
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
version: '3'
|
||||
|
||||
services:
|
||||
filebeat-victorialogs:
|
||||
image: docker.elastic.co/beats/filebeat:8.8.1
|
||||
restart: on-failure
|
||||
volumes:
|
||||
- type: bind
|
||||
source: /var/run/docker.sock
|
||||
target: /var/run/docker.sock
|
||||
- type: bind
|
||||
source: /var/lib/docker/containers/
|
||||
target: /var/lib/docker/containers/
|
||||
- ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
|
||||
user: root
|
||||
command:
|
||||
- "--strict.perms=false"
|
||||
depends_on: [ victorialogs ]
|
||||
beat-exporter-victorialogs:
|
||||
image: trustpilot/beat-exporter:0.4.0
|
||||
command:
|
||||
- -beat.uri=http://filebeat-victorialogs:5066
|
||||
|
||||
# Run `make package-victoria-logs` to build victoria-logs image
|
||||
victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:heads-master-0-g88993f312-dirty-e01fb71f
|
||||
volumes:
|
||||
- victorialogs-filebeat-docker-vl:/vlogs
|
||||
ports:
|
||||
- '9428:9428'
|
||||
command:
|
||||
- -storageDataPath=/vlogs
|
||||
- -loggerFormat=json
|
||||
grafana:
|
||||
container_name: grafana
|
||||
image: grafana/grafana:9.2.7
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
- 3000:3000
|
||||
volumes:
|
||||
- victorialogs-filebeat-docker-grafana:/var/lib/grafana
|
||||
- ./../../provisioning/:/etc/grafana/provisioning/
|
||||
- ./../../../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json
|
||||
- ./../../../../dashboards/victorialogs.json:/var/lib/grafana/dashboards/vl.json
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.91.2
|
||||
ports:
|
||||
- '8428:8428'
|
||||
command:
|
||||
- -storageDataPath=/vmsingle
|
||||
- -promscrape.config=/promscrape.yml
|
||||
- -loggerFormat=json
|
||||
volumes:
|
||||
- victorialogs-filebeat-docker-vm:/vmsingle
|
||||
- ./scrape.yml:/promscrape.yml
|
||||
|
||||
volumes:
|
||||
victorialogs-filebeat-docker-vl:
|
||||
victorialogs-filebeat-docker-vm:
|
||||
victorialogs-filebeat-docker-grafana:
|
||||
20
deployment/docker/victorialogs/filebeat-docker/filebeat.yml
Normal file
20
deployment/docker/victorialogs/filebeat-docker/filebeat.yml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
filebeat.autodiscover:
|
||||
providers:
|
||||
- type: docker
|
||||
hints.enabled: true
|
||||
|
||||
processors:
|
||||
- add_docker_metadata: ~
|
||||
|
||||
output.elasticsearch:
|
||||
hosts: [ "http://victorialogs:9428/insert/elasticsearch/" ]
|
||||
worker: 5
|
||||
parameters:
|
||||
_msg_field: "message"
|
||||
_time_field: "@timestamp"
|
||||
_stream_fields: "container.name"
|
||||
|
||||
http:
|
||||
enabled: true
|
||||
host: 0.0.0.0
|
||||
port: 5066
|
||||
16
deployment/docker/victorialogs/filebeat-docker/scrape.yml
Normal file
16
deployment/docker/victorialogs/filebeat-docker/scrape.yml
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
scrape_configs:
|
||||
- job_name: "filebeat"
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets:
|
||||
- beat-exporter-victorialogs:9479
|
||||
- job_name: "victoriametrics"
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets:
|
||||
- victoriametrics:8428
|
||||
- job_name: "victorialogs"
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets:
|
||||
- victorialogs:9428
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
version: "3"
|
||||
|
||||
services:
|
||||
filebeat-vlogs:
|
||||
image: docker.elastic.co/beats/filebeat:8.8.1
|
||||
user: root
|
||||
command:
|
||||
- "--strict.perms=false"
|
||||
volumes:
|
||||
- ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- /var/lib/docker:/var/lib/docker
|
||||
depends_on: [victorialogs]
|
||||
ports:
|
||||
- "5140:5140"
|
||||
|
||||
# Run `make package-victoria-logs` to build victoria-logs image
|
||||
victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:heads-logs-examples-0-g66b42a677-dirty-b95f1d4b
|
||||
volumes:
|
||||
- victorialogs-filebeat-syslog-vl:/vlogs
|
||||
ports:
|
||||
- "9428:9428"
|
||||
command:
|
||||
- -storageDataPath=/vlogs
|
||||
|
||||
volumes:
|
||||
victorialogs-filebeat-syslog-vl:
|
||||
22
deployment/docker/victorialogs/filebeat-syslog/filebeat.yml
Normal file
22
deployment/docker/victorialogs/filebeat-syslog/filebeat.yml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
filebeat.inputs:
|
||||
- type: syslog
|
||||
format: rfc3164
|
||||
protocol.tcp:
|
||||
host: "0.0.0.0:5140"
|
||||
|
||||
filebeat.autodiscover:
|
||||
providers:
|
||||
- type: docker
|
||||
hints.enabled: true
|
||||
|
||||
processors:
|
||||
- add_docker_metadata: ~
|
||||
|
||||
output.elasticsearch:
|
||||
hosts: [ "http://victorialogs:9428/insert/elasticsearch/" ]
|
||||
worker: 5
|
||||
bulk_max_size: 1000
|
||||
parameters:
|
||||
_msg_field: "message"
|
||||
_time_field: "@timestamp"
|
||||
_stream_fields: "host.name,process.program,process.pid,container.name"
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
version: "3"
|
||||
|
||||
services:
|
||||
fluentbit:
|
||||
image: cr.fluentbit.io/fluent/fluent-bit:2.1.4
|
||||
volumes:
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
- ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
|
||||
depends_on: [victorialogs]
|
||||
ports:
|
||||
- "5140:5140"
|
||||
|
||||
# Run `make package-victoria-logs` to build victoria-logs image
|
||||
victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:heads-logs-examples-0-g66b42a677-dirty-b95f1d4b
|
||||
volumes:
|
||||
- victorialogs-fluentbit-vl:/vlogs
|
||||
ports:
|
||||
- "9428:9428"
|
||||
command:
|
||||
- -storageDataPath=/vlogs
|
||||
|
||||
volumes:
|
||||
victorialogs-fluentbit-vl:
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
[INPUT]
|
||||
name tail
|
||||
path /var/lib/docker/containers/**/*.log
|
||||
multiline.parser docker, cri
|
||||
Parser docker
|
||||
Docker_Mode On
|
||||
|
||||
[INPUT]
|
||||
Name syslog
|
||||
Listen 0.0.0.0
|
||||
Port 5140
|
||||
Parser syslog-rfc3164
|
||||
Mode tcp
|
||||
|
||||
[SERVICE]
|
||||
Flush 1
|
||||
Parsers_File parsers.conf
|
||||
|
||||
[Output]
|
||||
Name http
|
||||
Match *
|
||||
host victorialogs
|
||||
port 9428
|
||||
compress gzip
|
||||
uri /insert/jsonline/?_stream_fields=stream&_msg_field=log&_time_field=date
|
||||
format json_lines
|
||||
json_date_format iso8601
|
||||
header AccountID 0
|
||||
header ProjectID 0
|
||||
3
deployment/docker/victorialogs/logstash/Dockerfile
Normal file
3
deployment/docker/victorialogs/logstash/Dockerfile
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
FROM docker.elastic.co/logstash/logstash:8.8.1
|
||||
|
||||
RUN bin/logstash-plugin install logstash-output-opensearch
|
||||
27
deployment/docker/victorialogs/logstash/README.md
Normal file
27
deployment/docker/victorialogs/logstash/README.md
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
# How to set up sending logs to VictoriaLogs from logstash
|
||||
|
||||
It is required to use [OpenSearch plugin](https://github.com/opensearch-project/logstash-output-opensearch) for output configuration.
|
||||
Plugin can be installed by using the following command:
|
||||
```
|
||||
bin/logstash-plugin install logstash-output-opensearch
|
||||
```
|
||||
OpenSearch plugin is required because elasticsearch output plugin performs various checks for Elasticsearch version and license which are not applicable for VictoriaLogs.
|
||||
|
||||
Here is an example of logstash configuration:
|
||||
|
||||
```
|
||||
opensearch {
|
||||
hosts => ["http://victorialogs:9428/insert/elasticsearch"]
|
||||
custom_headers => {
|
||||
"AccountID" => "0"
|
||||
"ProjectID" => "0"
|
||||
}
|
||||
parameters => {
|
||||
"_stream_fields" => "host.ip,process.name"
|
||||
"_msg_field" => "message"
|
||||
"_time_field" => "@timestamp"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) to achieve better performance.
|
||||
27
deployment/docker/victorialogs/logstash/docker-compose.yml
Normal file
27
deployment/docker/victorialogs/logstash/docker-compose.yml
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
version: '3'
|
||||
|
||||
services:
|
||||
logstash:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
restart: on-failure
|
||||
volumes:
|
||||
- ./pipeline.conf:/usr/share/logstash/pipeline/logstash.conf:ro
|
||||
- ./logstash.yml:/usr/share/logstash/config/logstash.yml:ro
|
||||
depends_on: [victorialogs]
|
||||
ports:
|
||||
- "5140:5140"
|
||||
|
||||
# Run `make package-victoria-logs` to build victoria-logs image
|
||||
victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:heads-logs-examples-0-g66b42a677-dirty-b95f1d4b
|
||||
volumes:
|
||||
- victorialogs-logstash-vl:/vlogs
|
||||
ports:
|
||||
- "9428:9428"
|
||||
command:
|
||||
- -storageDataPath=/vlogs
|
||||
|
||||
volumes:
|
||||
victorialogs-logstash-vl:
|
||||
2
deployment/docker/victorialogs/logstash/logstash.yml
Normal file
2
deployment/docker/victorialogs/logstash/logstash.yml
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
http.host: 0.0.0.0
|
||||
xpack.monitoring.enabled: false
|
||||
20
deployment/docker/victorialogs/logstash/pipeline.conf
Normal file
20
deployment/docker/victorialogs/logstash/pipeline.conf
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
input {
|
||||
syslog {
|
||||
port => 5140
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
opensearch {
|
||||
hosts => ["http://victorialogs:9428/insert/elasticsearch"]
|
||||
custom_headers => {
|
||||
"AccountID" => "0"
|
||||
"ProjectID" => "0"
|
||||
}
|
||||
parameters => {
|
||||
"_stream_fields" => "host.ip,process.name"
|
||||
"_msg_field" => "message"
|
||||
"_time_field" => "@timestamp"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
version: '3'
|
||||
|
||||
services:
|
||||
vector:
|
||||
image: docker.io/timberio/vector:0.30.0-distroless-static
|
||||
restart: on-failure
|
||||
volumes:
|
||||
- type: bind
|
||||
source: /var/run/docker.sock
|
||||
target: /var/run/docker.sock
|
||||
- type: bind
|
||||
source: /var/lib/docker
|
||||
target: /var/lib/docker
|
||||
- ./vector.toml:/etc/vector/vector.toml:ro
|
||||
user: root
|
||||
ports:
|
||||
- '8686:8686'
|
||||
depends_on:
|
||||
victorialogs:
|
||||
condition: service_healthy
|
||||
victoriametrics:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8686/health"]
|
||||
interval: 1s
|
||||
timeout: 1s
|
||||
retries: 10
|
||||
|
||||
# Run `make package-victoria-logs` to build victoria-logs image
|
||||
victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:heads-docker-logs-0-g4dbdb3039
|
||||
volumes:
|
||||
- victorialogs-vector-docker-vl:/vlogs
|
||||
ports:
|
||||
- '9428:9428'
|
||||
command:
|
||||
- -storageDataPath=/vlogs
|
||||
- -loggerFormat=json
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:9428/health"]
|
||||
interval: 1s
|
||||
timeout: 1s
|
||||
retries: 10
|
||||
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.91.2
|
||||
ports:
|
||||
- '8428:8428'
|
||||
command:
|
||||
- -storageDataPath=/vmsingle
|
||||
- -promscrape.config=/promscrape.yml
|
||||
- -loggerFormat=json
|
||||
volumes:
|
||||
- victorialogs-vector-docker-vm:/vmsingle
|
||||
- ./scrape.yml:/promscrape.yml
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:8428/health"]
|
||||
interval: 1s
|
||||
timeout: 1s
|
||||
retries: 10
|
||||
|
||||
volumes:
|
||||
victorialogs-vector-docker-vl:
|
||||
victorialogs-vector-docker-vm:
|
||||
11
deployment/docker/victorialogs/vector-docker/scrape.yml
Normal file
11
deployment/docker/victorialogs/vector-docker/scrape.yml
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
scrape_configs:
|
||||
- job_name: "victoriametrics"
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets:
|
||||
- victoriametrics:8428
|
||||
- job_name: "victorialogs"
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets:
|
||||
- victorialogs:9428
|
||||
28
deployment/docker/victorialogs/vector-docker/vector.toml
Normal file
28
deployment/docker/victorialogs/vector-docker/vector.toml
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
[api]
|
||||
enabled = true
|
||||
address = "0.0.0.0:8686"
|
||||
|
||||
[sources.docker]
|
||||
type = "docker_logs"
|
||||
|
||||
[sinks.vlogs]
|
||||
type = "elasticsearch"
|
||||
inputs = [ "docker" ]
|
||||
endpoints = [ "http://victorialogs:9428/insert/elasticsearch/" ]
|
||||
id_key = "id"
|
||||
mode = "bulk"
|
||||
healthcheck.enabled = false
|
||||
|
||||
[sinks.vlogs.query]
|
||||
_msg_field = "message"
|
||||
_time_field = "timestamp"
|
||||
_stream_fields = "host,container_name"
|
||||
|
||||
[sources.vector_metrics]
|
||||
type = "internal_metrics"
|
||||
|
||||
[sinks.victoriametrics]
|
||||
type = "prometheus_remote_write"
|
||||
endpoint = "http://victoriametrics:8428/api/v1/write"
|
||||
inputs = ["vector_metrics"]
|
||||
healthcheck.enabled = false
|
||||
Loading…
Reference in a new issue