diff --git a/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/README.md b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/README.md new file mode 100644 index 000000000..c42fe147a --- /dev/null +++ b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/README.md @@ -0,0 +1,23 @@ +# Docker Compose file for "Getting Started with vmanomaly" guide + +Please read the "Getting Started with vmanomaly" guide first - [https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert.html](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert.html) + +To make this Docker compose file work, you MUST replace the content of [vmanomaly_license.txt](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmanomaly_license.txt) with valid license. + +You can issue the [trial license here](https://victoriametrics.com/products/enterprise/trial/) + + +## How to run + +1. Replace content of `vmanomaly_license.txt` with your license +1. Run + +```sh +docker compose up -d +``` +1. Open Grafana on `http://127.0.0.1:3000/` +```sh +open http://127.0.0.1:3000/ +``` + +If you don't see any data, please wait a few minutes. \ No newline at end of file diff --git a/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/alertmanager.yml b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/alertmanager.yml new file mode 100644 index 000000000..4b68f7863 --- /dev/null +++ b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/alertmanager.yml @@ -0,0 +1,5 @@ +route: + receiver: blackhole + +receivers: + - name: blackhole \ No newline at end of file diff --git a/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/datasource.yml b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/datasource.yml new file mode 100644 index 000000000..7d7c1eecd --- /dev/null +++ b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/datasource.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: VictoriaMetrics + type: prometheus + access: proxy + url: http://victoriametrics:8428 + isDefault: true + diff --git a/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/docker-compose.yml b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/docker-compose.yml new file mode 100644 index 000000000..4482da1a2 --- /dev/null +++ b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/docker-compose.yml @@ -0,0 +1,117 @@ +services: + vmagent: + container_name: vmagent + image: victoriametrics/vmagent:v1.96.0 + depends_on: + - "victoriametrics" + ports: + - 8429:8429 + volumes: + - vmagentdata-guide-vmanomaly-vmalert:/vmagentdata + - ./prometheus.yml:/etc/prometheus/prometheus.yml + command: + - "--promscrape.config=/etc/prometheus/prometheus.yml" + - "--remoteWrite.url=http://victoriametrics:8428/api/v1/write" + networks: + - vm_net + restart: always + + victoriametrics: + container_name: victoriametrics + image: victoriametrics/victoria-metrics:v1.96.0 + ports: + - 8428:8428 + volumes: + - vmdata-guide-vmanomaly-vmalert:/storage + command: + - "--storageDataPath=/storage" + - "--httpListenAddr=:8428" + - "--vmalert.proxyURL=http://vmalert:8880" + - "-search.disableCache=1" # for guide only, do not use in production + networks: + - vm_net + restart: always + + grafana: + container_name: grafana + image: grafana/grafana-oss:10.2.1 + depends_on: + - "victoriametrics" + ports: + - 3000:3000 + volumes: + - grafanadata-guide-vmanomaly-vmalert:/var/lib/grafana + - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml + networks: + - vm_net + restart: always + + + vmalert: + container_name: vmalert + image: victoriametrics/vmalert:v1.96.0 + depends_on: + - "victoriametrics" + ports: + - 8880:8880 + volumes: + - ./vmalert_config.yml:/etc/alerts/alerts.yml + command: + - "--datasource.url=http://victoriametrics:8428/" + - "--remoteRead.url=http://victoriametrics:8428/" + - "--remoteWrite.url=http://victoriametrics:8428/" + - "--notifier.url=http://alertmanager:9093/" + - "--rule=/etc/alerts/*.yml" + # display source of alerts in grafana + - "--external.url=http://127.0.0.1:3000" #grafana outside container + # when copypaste the line be aware of '$$' for escaping in '$expr' + - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' + networks: + - vm_net + restart: always + vmanomaly: + container_name: vmanomaly + image: victoriametrics/vmanomaly:v1.7.2 + depends_on: + - "victoriametrics" + ports: + - "8500:8500" + networks: + - vm_net + restart: always + volumes: + - ./vmanomaly_config.yml:/config.yaml + - ./vmanomaly_license.txt:/license.txt + platform: "linux/amd64" + command: + - "/config.yaml" + - "--license-file=/license.txt" + alertmanager: + container_name: alertmanager + image: prom/alertmanager:v0.25.0 + volumes: + - ./alertmanager.yml:/config/alertmanager.yml + command: + - "--config.file=/config/alertmanager.yml" + ports: + - 9093:9093 + networks: + - vm_net + restart: always + + node-exporter: + image: quay.io/prometheus/node-exporter:v1.7.0 + container_name: node-exporter + ports: + - 9100:9100 + pid: host + restart: unless-stopped + networks: + - vm_net + +volumes: + vmagentdata-guide-vmanomaly-vmalert: {} + vmdata-guide-vmanomaly-vmalert: {} + grafanadata-guide-vmanomaly-vmalert: {} +networks: + vm_net: diff --git a/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/prometheus.yml b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/prometheus.yml new file mode 100644 index 000000000..e8f51e64a --- /dev/null +++ b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 10s + +scrape_configs: + - job_name: 'vmagent' + static_configs: + - targets: ['vmagent:8429'] + - job_name: 'vmalert' + static_configs: + - targets: ['vmalert:8880'] + - job_name: 'victoriametrics' + static_configs: + - targets: ['victoriametrics:8428'] + - job_name: 'node-exporter' + static_configs: + - targets: ['node-exporter:9100'] + - job_name: 'vmanomaly' + static_configs: + - targets: [ 'vmanomaly:8500' ] diff --git a/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmalert_config.yml b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmalert_config.yml new file mode 100644 index 000000000..ce1c5515c --- /dev/null +++ b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmalert_config.yml @@ -0,0 +1,9 @@ +groups: + - name: AnomalyExample + rules: + - alert: HighAnomalyScore + expr: 'anomaly_score > 1.0' + labels: + severity: warning + annotations: + summary: Anomaly Score exceeded 1.0. `rate(node_cpu_seconds_total)` is showing abnormal behavior. diff --git a/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmanomaly_config.yml b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmanomaly_config.yml new file mode 100644 index 000000000..75f1d9757 --- /dev/null +++ b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmanomaly_config.yml @@ -0,0 +1,23 @@ +scheduler: + infer_every: "1m" + fit_every: "2h" + fit_window: "14d" + +model: + class: "model.prophet.ProphetModel" + args: + interval_width: 0.98 + +reader: + datasource_url: "http://victoriametrics:8428/" + queries: + node_cpu_rate: "rate(node_cpu_seconds_total)" + +writer: + datasource_url: "http://victoriametrics:8428/" + + +monitoring: + pull: # Enable /metrics endpoint. + addr: "0.0.0.0" + port: 8500 diff --git a/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmanomaly_license.txt b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmanomaly_license.txt new file mode 100644 index 000000000..193243777 --- /dev/null +++ b/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/vmanomaly_license.txt @@ -0,0 +1 @@ +INSERT_YOU_LICENSE_HERE \ No newline at end of file diff --git a/docs/anomaly-detection/guides/guide-vmanomaly-vmalert.md b/docs/anomaly-detection/guides/guide-vmanomaly-vmalert.md index 5d826c11e..389940a10 100644 --- a/docs/anomaly-detection/guides/guide-vmanomaly-vmalert.md +++ b/docs/anomaly-detection/guides/guide-vmanomaly-vmalert.md @@ -22,10 +22,12 @@ aliases: - [vmagent](https://docs.victoriametrics.com/vmagent.html) (v.1.96.0) - [Grafana](https://grafana.com/)(v.10.2.1) - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/) -- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) +- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter)(v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)(v0.25.0) vmanomaly typical setup diagramm +Example of configuration can be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/) + ## 1. What is vmanomaly? *VictoriaMetrics Anomaly Detection* ([vmanomaly](https://docs.victoriametrics.com/vmanomaly.html)) is a service that continuously scans time series stored in VictoriaMetrics and detects unexpected changes within data patterns in real-time. It does so by utilizing user-configurable machine learning models. @@ -156,6 +158,10 @@ reader: writer: datasource_url: "http://victoriametrics:8428/" +monitoring: + pull: # Enable /metrics endpoint. + addr: "0.0.0.0" + port: 8500 ``` @@ -196,23 +202,25 @@ In the query expression we need to put a condition on the generated anomaly scor You can choose your threshold value that you consider reasonable based on the anomaly score metric, generated by vmanomaly. One of the best ways is to estimate it visually, by plotting the `anomaly_score` metric, along with predicted "expected" range of `yhat_lower` and `yhat_upper`. Later in this tutorial we will show an example ## 8. Docker Compose configuration + +You can find the `docker-compose.yml` and all configs in this [folder](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-vmalert-guide/) + Now we are going to configure the `docker-compose.yml` file to run all needed services. Here are all services we are going to run: -

- Docker compose services -

- +* vmanomaly - VictoriaMetrics Anomaly Detection service. * victoriametrics - VictoriaMetrics Time Series Database * vmagent - is an agent which helps you collect metrics from various sources, relabel and filter the collected metrics and store them in VictoriaMetrics or any other storage systems via Prometheus remote_write protocol. * [grafana](https://grafana.com/) - visualization tool. * node-exporter - Prometheus [Node Exporter](https://prometheus.io/docs/guides/node-exporter/) exposes a wide variety of hardware- and kernel-related metrics. * vmalert - VictoriaMetrics Alerting service. -* vmanomaly - VictoriaMetrics Anomaly Detection service. +* alertmanager - Notification services that handles alerts from vmalert. ### Grafana setup To enable VictoriaMetrics datasource as the default in Grafana we need to create a file `datasource.yml` +The default username/password pair is `admin:admin` +
``` yaml @@ -261,9 +269,20 @@ scrape_configs: ### vmanomaly licencing -We are going to use license stored locally in file `vmanomaly_licence.txt` with key in it. +We are going to use license stored locally in file `vmanomaly_license.txt` with key in it. You can explore other license options [here](https://docs.victoriametrics.com/vmanomaly.html#licensing) +### Alertmanager setup + +Let's create `alertmanager.yml` file for `alertmanager` configuration. + +```yml +route: + receiver: blackhole + +receivers: + - name: blackhole +``` ### Docker-compose @@ -275,13 +294,13 @@ Let's wrap it all up together into the `docker-compose.yml` file. services: vmagent: container_name: vmagent - image: victoriametrics/vmagent:latest + image: victoriametrics/vmagent:v1.96.0 depends_on: - "victoriametrics" ports: - 8429:8429 volumes: - - vmagentdata:/vmagentdata + - vmagentdata-guide-vmanomaly-vmalert:/vmagentdata - ./prometheus.yml:/etc/prometheus/prometheus.yml command: - "--promscrape.config=/etc/prometheus/prometheus.yml" @@ -289,30 +308,23 @@ services: networks: - vm_net restart: always - + victoriametrics: container_name: victoriametrics image: victoriametrics/victoria-metrics:v1.96.0 ports: - 8428:8428 - - 8089:8089 - - 8089:8089/udp - - 2003:2003 - - 2003:2003/udp - - 4242:4242 volumes: - - vmdata:/storage + - vmdata-guide-vmanomaly-vmalert:/storage command: - "--storageDataPath=/storage" - - "--graphiteListenAddr=:2003" - - "--opentsdbListenAddr=:4242" - "--httpListenAddr=:8428" - - "--influxListenAddr=:8089" - "--vmalert.proxyURL=http://vmalert:8880" + - "-search.disableCache=1" # for guide only, do not use in production networks: - vm_net restart: always - + grafana: container_name: grafana image: grafana/grafana-oss:10.2.1 @@ -321,16 +333,16 @@ services: ports: - 3000:3000 volumes: - - grafanadata:/var/lib/grafana + - grafanadata-guide-vmanomaly-vmalert:/var/lib/grafana - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml networks: - vm_net restart: always - + vmalert: container_name: vmalert - image: victoriametrics/vmalert:latest + image: victoriametrics/vmalert:v1.96.0 depends_on: - "victoriametrics" ports: @@ -346,13 +358,13 @@ services: # display source of alerts in grafana - "--external.url=http://127.0.0.1:3000" #grafana outside container # when copypaste the line be aware of '$$' for escaping in '$expr' - - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr":{{$$expr|jsonEscape|queryEscape}} },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' + - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]' networks: - vm_net restart: always vmanomaly: container_name: vmanomaly - image: us-docker.pkg.dev/victoriametrics-test/public/vmanomaly-trial:v1.7.2 + image: victoriametrics/vmanomaly:v1.7.2 depends_on: - "victoriametrics" ports: @@ -364,12 +376,24 @@ services: - ./vmanomaly_config.yml:/config.yaml - ./vmanomaly_license.txt:/license.txt platform: "linux/amd64" - command: + command: - "/config.yaml" - "--license-file=/license.txt" + alertmanager: + container_name: alertmanager + image: prom/alertmanager:v0.25.0 + volumes: + - ./alertmanager.yml:/config/alertmanager.yml + command: + - "--config.file=/config/alertmanager.yml" + ports: + - 9093:9093 + networks: + - vm_net + restart: always node-exporter: - image: quay.io/prometheus/node-exporter:latest + image: quay.io/prometheus/node-exporter:v1.7.0 container_name: node-exporter ports: - 9100:9100 @@ -377,11 +401,11 @@ services: restart: unless-stopped networks: - vm_net - + volumes: - vmagentdata: {} - vmdata: {} - grafanadata: {} + vmagentdata-guide-vmanomaly-vmalert: {} + vmdata-guide-vmanomaly-vmalert: {} + grafanadata-guide-vmanomaly-vmalert: {} networks: vm_net: ``` diff --git a/docs/anomaly-detection/guides/guide-vmanomaly-vmalert_overview.webp b/docs/anomaly-detection/guides/guide-vmanomaly-vmalert_overview.webp index 2dfb66a6c..517c1bfcc 100644 Binary files a/docs/anomaly-detection/guides/guide-vmanomaly-vmalert_overview.webp and b/docs/anomaly-detection/guides/guide-vmanomaly-vmalert_overview.webp differ