mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
docs: vmanomaly update vmanomaly + vmalert guide (#5636)
* docs: vmanomaly update vmanomaly + vmalert guide Signed-off-by: Artem Navoiev <tenmozes@gmail.com> * docs: vmanomaly update vmanomaly + vmalert guide. Update docker compose and monitoring section Signed-off-by: Artem Navoiev <tenmozes@gmail.com> * typos and fixes Signed-off-by: Artem Navoiev <tenmozes@gmail.com> --------- Signed-off-by: Artem Navoiev <tenmozes@gmail.com>
This commit is contained in:
Artem Navoiev
GitHub
parent
ff33e60a3d
commit
f89d16fc4c
10 changed files with 261 additions and 31 deletions
|
|
@ -0,0 +1,23 @@
|
|||
# Docker Compose file for "Getting Started with vmanomaly" guide
|
||||
|
||||
Please read the "Getting Started with vmanomaly" guide first - [https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert.html](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert.html)
|
||||
|
||||
To make this Docker compose file work, you MUST replace the content of [vmanomaly_license.txt](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmanomaly_license.txt) with valid license.
|
||||
|
||||
You can issue the [trial license here](https://victoriametrics.com/products/enterprise/trial/)
|
||||
|
||||
|
||||
## How to run
|
||||
|
||||
1. Replace content of `vmanomaly_license.txt` with your license
|
||||
1. Run
|
||||
|
||||
```sh
|
||||
docker compose up -d
|
||||
```
|
||||
1. Open Grafana on `http://127.0.0.1:3000/`
|
||||
```sh
|
||||
open http://127.0.0.1:3000/
|
||||
```
|
||||
|
||||
If you don't see any data, please wait a few minutes.
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
route:
|
||||
receiver: blackhole
|
||||
|
||||
receivers:
|
||||
- name: blackhole
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: VictoriaMetrics
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://victoriametrics:8428
|
||||
isDefault: true
|
||||
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
services:
|
||||
vmagent:
|
||||
container_name: vmagent
|
||||
image: victoriametrics/vmagent:v1.96.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
- 8429:8429
|
||||
volumes:
|
||||
- vmagentdata-guide-vmanomaly-vmalert:/vmagentdata
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
command:
|
||||
- "--promscrape.config=/etc/prometheus/prometheus.yml"
|
||||
- "--remoteWrite.url=http://victoriametrics:8428/api/v1/write"
|
||||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
|
||||
victoriametrics:
|
||||
container_name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:v1.96.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
- vmdata-guide-vmanomaly-vmalert:/storage
|
||||
command:
|
||||
- "--storageDataPath=/storage"
|
||||
- "--httpListenAddr=:8428"
|
||||
- "--vmalert.proxyURL=http://vmalert:8880"
|
||||
- "-search.disableCache=1" # for guide only, do not use in production
|
||||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
|
||||
grafana:
|
||||
container_name: grafana
|
||||
image: grafana/grafana-oss:10.2.1
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
- 3000:3000
|
||||
volumes:
|
||||
- grafanadata-guide-vmanomaly-vmalert:/var/lib/grafana
|
||||
- ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
|
||||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
|
||||
|
||||
vmalert:
|
||||
container_name: vmalert
|
||||
image: victoriametrics/vmalert:v1.96.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
- 8880:8880
|
||||
volumes:
|
||||
- ./vmalert_config.yml:/etc/alerts/alerts.yml
|
||||
command:
|
||||
- "--datasource.url=http://victoriametrics:8428/"
|
||||
- "--remoteRead.url=http://victoriametrics:8428/"
|
||||
- "--remoteWrite.url=http://victoriametrics:8428/"
|
||||
- "--notifier.url=http://alertmanager:9093/"
|
||||
- "--rule=/etc/alerts/*.yml"
|
||||
# display source of alerts in grafana
|
||||
- "--external.url=http://127.0.0.1:3000" #grafana outside container
|
||||
# when copypaste the line be aware of '$$' for escaping in '$expr'
|
||||
- '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
|
||||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
vmanomaly:
|
||||
container_name: vmanomaly
|
||||
image: victoriametrics/vmanomaly:v1.7.2
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
- "8500:8500"
|
||||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
volumes:
|
||||
- ./vmanomaly_config.yml:/config.yaml
|
||||
- ./vmanomaly_license.txt:/license.txt
|
||||
platform: "linux/amd64"
|
||||
command:
|
||||
- "/config.yaml"
|
||||
- "--license-file=/license.txt"
|
||||
alertmanager:
|
||||
container_name: alertmanager
|
||||
image: prom/alertmanager:v0.25.0
|
||||
volumes:
|
||||
- ./alertmanager.yml:/config/alertmanager.yml
|
||||
command:
|
||||
- "--config.file=/config/alertmanager.yml"
|
||||
ports:
|
||||
- 9093:9093
|
||||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
|
||||
node-exporter:
|
||||
image: quay.io/prometheus/node-exporter:v1.7.0
|
||||
container_name: node-exporter
|
||||
ports:
|
||||
- 9100:9100
|
||||
pid: host
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- vm_net
|
||||
|
||||
volumes:
|
||||
vmagentdata-guide-vmanomaly-vmalert: {}
|
||||
vmdata-guide-vmanomaly-vmalert: {}
|
||||
grafanadata-guide-vmanomaly-vmalert: {}
|
||||
networks:
|
||||
vm_net:
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
global:
|
||||
scrape_interval: 10s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'vmagent'
|
||||
static_configs:
|
||||
- targets: ['vmagent:8429']
|
||||
- job_name: 'vmalert'
|
||||
static_configs:
|
||||
- targets: ['vmalert:8880']
|
||||
- job_name: 'victoriametrics'
|
||||
static_configs:
|
||||
- targets: ['victoriametrics:8428']
|
||||
- job_name: 'node-exporter'
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
- job_name: 'vmanomaly'
|
||||
static_configs:
|
||||
- targets: [ 'vmanomaly:8500' ]
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
groups:
|
||||
- name: AnomalyExample
|
||||
rules:
|
||||
- alert: HighAnomalyScore
|
||||
expr: 'anomaly_score > 1.0'
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Anomaly Score exceeded 1.0. `rate(node_cpu_seconds_total)` is showing abnormal behavior.
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
scheduler:
|
||||
infer_every: "1m"
|
||||
fit_every: "2h"
|
||||
fit_window: "14d"
|
||||
|
||||
model:
|
||||
class: "model.prophet.ProphetModel"
|
||||
args:
|
||||
interval_width: 0.98
|
||||
|
||||
reader:
|
||||
datasource_url: "http://victoriametrics:8428/"
|
||||
queries:
|
||||
node_cpu_rate: "rate(node_cpu_seconds_total)"
|
||||
|
||||
writer:
|
||||
datasource_url: "http://victoriametrics:8428/"
|
||||
|
||||
|
||||
monitoring:
|
||||
pull: # Enable /metrics endpoint.
|
||||
addr: "0.0.0.0"
|
||||
port: 8500
|
||||
|
|
@ -0,0 +1 @@
|
|||
INSERT_YOU_LICENSE_HERE
|
||||
|
|
@ -22,10 +22,12 @@ aliases:
|
|||
- [vmagent](https://docs.victoriametrics.com/vmagent.html) (v.1.96.0)
|
||||
- [Grafana](https://grafana.com/)(v.10.2.1)
|
||||
- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
|
||||
- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter)
|
||||
- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter)(v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)(v0.25.0)
|
||||
|
||||
<img width="800" alt="vmanomaly typical setup diagramm" src="guide-vmanomaly-vmalert_overview.webp">
|
||||
|
||||
Example of configuration can be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/)
|
||||
|
||||
## 1. What is vmanomaly?
|
||||
|
||||
*VictoriaMetrics Anomaly Detection* ([vmanomaly](https://docs.victoriametrics.com/vmanomaly.html)) is a service that continuously scans time series stored in VictoriaMetrics and detects unexpected changes within data patterns in real-time. It does so by utilizing user-configurable machine learning models.
|
||||
|
|
@ -156,6 +158,10 @@ reader:
|
|||
writer:
|
||||
datasource_url: "http://victoriametrics:8428/"
|
||||
|
||||
monitoring:
|
||||
pull: # Enable /metrics endpoint.
|
||||
addr: "0.0.0.0"
|
||||
port: 8500
|
||||
```
|
||||
|
||||
</div>
|
||||
|
|
@ -196,23 +202,25 @@ In the query expression we need to put a condition on the generated anomaly scor
|
|||
You can choose your threshold value that you consider reasonable based on the anomaly score metric, generated by vmanomaly. One of the best ways is to estimate it visually, by plotting the `anomaly_score` metric, along with predicted "expected" range of `yhat_lower` and `yhat_upper`. Later in this tutorial we will show an example
|
||||
|
||||
## 8. Docker Compose configuration
|
||||
|
||||
You can find the `docker-compose.yml` and all configs in this [folder](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/)
|
||||
|
||||
Now we are going to configure the `docker-compose.yml` file to run all needed services.
|
||||
Here are all services we are going to run:
|
||||
|
||||
<p align="center">
|
||||
<img src="guide-vmanomaly-vmalert_docker-compose.webp" width="800" alt="Docker compose services">
|
||||
</p>
|
||||
|
||||
* vmanomaly - VictoriaMetrics Anomaly Detection service.
|
||||
* victoriametrics - VictoriaMetrics Time Series Database
|
||||
* vmagent - is an agent which helps you collect metrics from various sources, relabel and filter the collected metrics and store them in VictoriaMetrics or any other storage systems via Prometheus remote_write protocol.
|
||||
* [grafana](https://grafana.com/) - visualization tool.
|
||||
* node-exporter - Prometheus [Node Exporter](https://prometheus.io/docs/guides/node-exporter/) exposes a wide variety of hardware- and kernel-related metrics.
|
||||
* vmalert - VictoriaMetrics Alerting service.
|
||||
* vmanomaly - VictoriaMetrics Anomaly Detection service.
|
||||
* alertmanager - Notification services that handles alerts from vmalert.
|
||||
|
||||
### Grafana setup
|
||||
To enable VictoriaMetrics datasource as the default in Grafana we need to create a file `datasource.yml`
|
||||
|
||||
The default username/password pair is `admin:admin`
|
||||
|
||||
<div class="with-copy" markdown="1">
|
||||
|
||||
``` yaml
|
||||
|
|
@ -261,9 +269,20 @@ scrape_configs:
|
|||
|
||||
### vmanomaly licencing
|
||||
|
||||
We are going to use license stored locally in file `vmanomaly_licence.txt` with key in it.
|
||||
We are going to use license stored locally in file `vmanomaly_license.txt` with key in it.
|
||||
You can explore other license options [here](https://docs.victoriametrics.com/vmanomaly.html#licensing)
|
||||
|
||||
### Alertmanager setup
|
||||
|
||||
Let's create `alertmanager.yml` file for `alertmanager` configuration.
|
||||
|
||||
```yml
|
||||
route:
|
||||
receiver: blackhole
|
||||
|
||||
receivers:
|
||||
- name: blackhole
|
||||
```
|
||||
|
||||
### Docker-compose
|
||||
|
||||
|
|
@ -275,13 +294,13 @@ Let's wrap it all up together into the `docker-compose.yml` file.
|
|||
services:
|
||||
vmagent:
|
||||
container_name: vmagent
|
||||
image: victoriametrics/vmagent:latest
|
||||
image: victoriametrics/vmagent:v1.96.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
- 8429:8429
|
||||
volumes:
|
||||
- vmagentdata:/vmagentdata
|
||||
- vmagentdata-guide-vmanomaly-vmalert:/vmagentdata
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
command:
|
||||
- "--promscrape.config=/etc/prometheus/prometheus.yml"
|
||||
|
|
@ -289,30 +308,23 @@ services:
|
|||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
|
||||
|
||||
victoriametrics:
|
||||
container_name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:v1.96.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
- 8089:8089
|
||||
- 8089:8089/udp
|
||||
- 2003:2003
|
||||
- 2003:2003/udp
|
||||
- 4242:4242
|
||||
volumes:
|
||||
- vmdata:/storage
|
||||
- vmdata-guide-vmanomaly-vmalert:/storage
|
||||
command:
|
||||
- "--storageDataPath=/storage"
|
||||
- "--graphiteListenAddr=:2003"
|
||||
- "--opentsdbListenAddr=:4242"
|
||||
- "--httpListenAddr=:8428"
|
||||
- "--influxListenAddr=:8089"
|
||||
- "--vmalert.proxyURL=http://vmalert:8880"
|
||||
- "-search.disableCache=1" # for guide only, do not use in production
|
||||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
|
||||
|
||||
grafana:
|
||||
container_name: grafana
|
||||
image: grafana/grafana-oss:10.2.1
|
||||
|
|
@ -321,16 +333,16 @@ services:
|
|||
ports:
|
||||
- 3000:3000
|
||||
volumes:
|
||||
- grafanadata:/var/lib/grafana
|
||||
- grafanadata-guide-vmanomaly-vmalert:/var/lib/grafana
|
||||
- ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
|
||||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
|
||||
|
||||
|
||||
vmalert:
|
||||
container_name: vmalert
|
||||
image: victoriametrics/vmalert:latest
|
||||
image: victoriametrics/vmalert:v1.96.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
|
@ -346,13 +358,13 @@ services:
|
|||
# display source of alerts in grafana
|
||||
- "--external.url=http://127.0.0.1:3000" #grafana outside container
|
||||
# when copypaste the line be aware of '$$' for escaping in '$expr'
|
||||
- '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
|
||||
- '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
|
||||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
vmanomaly:
|
||||
container_name: vmanomaly
|
||||
image: us-docker.pkg.dev/victoriametrics-test/public/vmanomaly-trial:v1.7.2
|
||||
image: victoriametrics/vmanomaly:v1.7.2
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
|
@ -364,12 +376,24 @@ services:
|
|||
- ./vmanomaly_config.yml:/config.yaml
|
||||
- ./vmanomaly_license.txt:/license.txt
|
||||
platform: "linux/amd64"
|
||||
command:
|
||||
command:
|
||||
- "/config.yaml"
|
||||
- "--license-file=/license.txt"
|
||||
alertmanager:
|
||||
container_name: alertmanager
|
||||
image: prom/alertmanager:v0.25.0
|
||||
volumes:
|
||||
- ./alertmanager.yml:/config/alertmanager.yml
|
||||
command:
|
||||
- "--config.file=/config/alertmanager.yml"
|
||||
ports:
|
||||
- 9093:9093
|
||||
networks:
|
||||
- vm_net
|
||||
restart: always
|
||||
|
||||
node-exporter:
|
||||
image: quay.io/prometheus/node-exporter:latest
|
||||
image: quay.io/prometheus/node-exporter:v1.7.0
|
||||
container_name: node-exporter
|
||||
ports:
|
||||
- 9100:9100
|
||||
|
|
@ -377,11 +401,11 @@ services:
|
|||
restart: unless-stopped
|
||||
networks:
|
||||
- vm_net
|
||||
|
||||
|
||||
volumes:
|
||||
vmagentdata: {}
|
||||
vmdata: {}
|
||||
grafanadata: {}
|
||||
vmagentdata-guide-vmanomaly-vmalert: {}
|
||||
vmdata-guide-vmanomaly-vmalert: {}
|
||||
grafanadata-guide-vmanomaly-vmalert: {}
|
||||
networks:
|
||||
vm_net:
|
||||
```
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 36 KiB After Width: | Height: | Size: 42 KiB |
Loading…
Reference in a new issue