From 23c615f04673a2321901ca3bcad0e8e859013988 Mon Sep 17 00:00:00 2001 From: Haley Wang Date: Mon, 17 Jun 2024 22:00:45 +0800 Subject: [PATCH] docs: add HA options --- deployment/docker/cluster-ha/auth-cluster.yml | 6 + .../docker/cluster-ha/auth-multi-vmselect.yml | 5 + .../cluster-ha/with-multi-level-vmselect.yml | 224 ++++++++++++++++++ deployment/docker/cluster-ha/with-vmauth.yml | 210 ++++++++++++++++ 4 files changed, 445 insertions(+) create mode 100644 deployment/docker/cluster-ha/auth-cluster.yml create mode 100644 deployment/docker/cluster-ha/auth-multi-vmselect.yml create mode 100644 deployment/docker/cluster-ha/with-multi-level-vmselect.yml create mode 100644 deployment/docker/cluster-ha/with-vmauth.yml diff --git a/deployment/docker/cluster-ha/auth-cluster.yml b/deployment/docker/cluster-ha/auth-cluster.yml new file mode 100644 index 000000000..6a1490b86 --- /dev/null +++ b/deployment/docker/cluster-ha/auth-cluster.yml @@ -0,0 +1,6 @@ +# balance load among vmselects +# see https://docs.victoriametrics.com/vmauth/#load-balancing +unauthorized_user: + url_prefix: + - http://vmselect-1:8481 + - http://vmselect-2:8481 diff --git a/deployment/docker/cluster-ha/auth-multi-vmselect.yml b/deployment/docker/cluster-ha/auth-multi-vmselect.yml new file mode 100644 index 000000000..b0a99fe82 --- /dev/null +++ b/deployment/docker/cluster-ha/auth-multi-vmselect.yml @@ -0,0 +1,5 @@ +# balance load among vmselects +# see https://docs.victoriametrics.com/vmauth/#load-balancing +unauthorized_user: + url_prefix: + - http://vmselect-top:8481 diff --git a/deployment/docker/cluster-ha/with-multi-level-vmselect.yml b/deployment/docker/cluster-ha/with-multi-level-vmselect.yml new file mode 100644 index 000000000..66cc022d8 --- /dev/null +++ b/deployment/docker/cluster-ha/with-multi-level-vmselect.yml @@ -0,0 +1,224 @@ +services: + # Metrics collector. + # It scrapes targets defined in --promscrape.config + # And forward them to --remoteWrite.url + vmagent: + container_name: vmagent + image: victoriametrics/vmagent:v1.101.0 + depends_on: + - "vminsert-1" + - "vminsert-2" + ports: + - 8429:8429 + volumes: + - vmagentdata:/vmagentdata + - ../prometheus-cluster.yml:/etc/prometheus/prometheus.yml + command: + - '--promscrape.config=/etc/prometheus/prometheus.yml' + - '--remoteWrite.url=http://vminsert-1:8480/insert/0/prometheus/' + - '--remoteWrite.url=http://vminsert-2:8480/insert/0/prometheus/' + restart: always + + # Grafana instance configured with VictoriaMetrics as datasource + grafana: + container_name: grafana + image: grafana/grafana:10.4.2 + depends_on: + - "vmselect-top" + ports: + - 3000:3000 + restart: always + volumes: + - grafanadata:/var/lib/grafana + - ../provisioning/datasources/prometheus-datasource:/etc/grafana/provisioning/datasources + - ../provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ../../../dashboards/victoriametrics-cluster.json:/var/lib/grafana/dashboards/vm.json + - ../../../dashboards/vmagent.json:/var/lib/grafana/dashboards/vmagent.json + - ../../../dashboards/vmalert.json:/var/lib/grafana/dashboards/vmalert.json + + # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts, + # where N is number of vmstorages (2 in this case). + vmstorage-1-1: + container_name: vmstorage-1-1 + image: victoriametrics/vmstorage:v1.101.0-cluster + ports: + - 8482 + - 8400 + - 8401 + volumes: + - strgdata-1:/storage + command: + - '--storageDataPath=/storage' + restart: always + vmstorage-1-2: + container_name: vmstorage-1-2 + image: victoriametrics/vmstorage:v1.101.0-cluster + ports: + - 8482 + - 8400 + - 8401 + volumes: + - strgdata-2:/storage + command: + - '--storageDataPath=/storage' + restart: always + + vmstorage-2-1: + container_name: vmstorage-2-1 + image: victoriametrics/vmstorage:v1.101.0-cluster + ports: + - 8482 + - 8400 + - 8401 + volumes: + - strgdata-3:/storage + command: + - '--storageDataPath=/storage' + restart: always + vmstorage-2-2: + container_name: vmstorage-2-2 + image: victoriametrics/vmstorage:v1.101.0-cluster + ports: + - 8482 + - 8400 + - 8401 + volumes: + - strgdata-4:/storage + command: + - '--storageDataPath=/storage' + restart: always + + # vminsert is ingestion frontend. It receives metrics pushed by vmagent, + # pre-process them and distributes across configured vmstorage shards. + vminsert-1: + container_name: vminsert-1 + image: victoriametrics/vminsert:v1.101.0-cluster + depends_on: + - "vmstorage-1-1" + - "vmstorage-1-2" + command: + - '--storageNode=vmstorage-1-1:8400' + - '--storageNode=vmstorage-1-2:8400' + ports: + - 8480 + restart: always + + vminsert-2: + container_name: vminsert-2 + image: victoriametrics/vminsert:v1.101.0-cluster + depends_on: + - "vmstorage-2-1" + - "vmstorage-2-2" + command: + - '--storageNode=vmstorage-2-1:8400' + - '--storageNode=vmstorage-2-2:8400' + ports: + - 8480 + restart: always + + # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL. + # vmselect collects results from configured `--storageNode` shards. + vmselect-local-1: + container_name: vmselect-1 + image: victoriametrics/vmselect:v1.101.0-cluster + depends_on: + - "vmstorage-1-1" + - "vmstorage-1-2" + command: + - '--storageNode=vmstorage-1-1:8401' + - '--storageNode=vmstorage-1-2:8401' + - '-clusternativeListenAddr=:8401' + ports: + - 8481 + restart: always + vmselect-local-2: + container_name: vmselect-2 + image: victoriametrics/vmselect:v1.101.0-cluster + depends_on: + - "vmstorage-2-1" + - "vmstorage-2-2" + command: + - '--storageNode=vmstorage-2-1:8401' + - '--storageNode=vmstorage-2-2:8401' + - '-clusternativeListenAddr=:8401' + ports: + - 8481 + restart: always + + # vmauth is a router and balancer for HTTP requests. + # It is configured via --auth.config and balances + # read requests from Grafana, vmui, vmalert among vmselects. + # It can be used as an authentication proxy. + vmselect-top: + container_name: vmselect-top + image: victoriametrics/vmselect:v1.101.0-cluster + depends_on: + - "vmselect-local-1" + - "vmselect-local-2" + command: + - '--storageNode=c1/vmselect-local-1:8401' + - '--storageNode=c2/vmselect-local-2:8401' + - '--globalReplicationFactor=2' + - '--vmalert.proxyURL=http://vmalert:8880' + ports: + - 8481 + restart: always + + vmauth: + container_name: vmauth + image: victoriametrics/vmauth:v1.101.0 + depends_on: + - "vmselect-top" + volumes: + - ./auth-multi-vmselect.yml:/etc/auth.yml + command: + - '--auth.config=/etc/auth.yml' + ports: + - 8427:8427 + restart: always + + # vmalert executes alerting and recording rules + vmalert: + container_name: vmalert + image: victoriametrics/vmalert:v1.101.0 + depends_on: + - "vmauth" + ports: + - 8880:8880 + volumes: + - ../alerts-cluster.yml:/etc/alerts/alerts.yml + - ../alerts-health.yml:/etc/alerts/alerts-health.yml + - ../alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml + - ../alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml + command: + - '--datasource.url=http://vmauth:8427/select/0/prometheus' + - '--remoteRead.url=http://vmauth:8427/select/0/prometheus' + - '--remoteWrite.url=http://vmagent:8429/api/v1/write' + - '--notifier.url=http://alertmanager:9093/' + - '--rule=/etc/alerts/*.yml' + # display source of alerts in grafana + - '-external.url=http://127.0.0.1:3000' #grafana outside container + # when copypaste the line below be aware of '$$' for escaping in '$expr' + - '--external.alert.source=explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{$$expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}' + restart: always + + # alertmanager receives alerting notifications from vmalert + # and distributes them according to --config.file. + alertmanager: + container_name: alertmanager + image: prom/alertmanager:v0.27.0 + volumes: + - ../alertmanager.yml:/config/alertmanager.yml + command: + - '--config.file=/config/alertmanager.yml' + ports: + - 9093:9093 + restart: always + +volumes: + vmagentdata: {} + strgdata-1: {} + strgdata-2: {} + strgdata-3: {} + strgdata-4: {} + grafanadata: {} diff --git a/deployment/docker/cluster-ha/with-vmauth.yml b/deployment/docker/cluster-ha/with-vmauth.yml new file mode 100644 index 000000000..c72d37c25 --- /dev/null +++ b/deployment/docker/cluster-ha/with-vmauth.yml @@ -0,0 +1,210 @@ +services: + # Metrics collector. + # It scrapes targets defined in --promscrape.config + # And forward them to --remoteWrite.url + vmagent: + container_name: vmagent + image: victoriametrics/vmagent:v1.101.0 + depends_on: + - "vminsert-1" + - "vminsert-2" + ports: + - 8429:8429 + volumes: + - vmagentdata:/vmagentdata + - ./prometheus-cluster.yml:/etc/prometheus/prometheus.yml + command: + - '--promscrape.config=/etc/prometheus/prometheus.yml' + - '--remoteWrite.url=http://vminsert-1:8480/insert/0/prometheus/' + - '--remoteWrite.url=http://vminsert-2:8480/insert/0/prometheus/' + restart: always + + # Grafana instance configured with VictoriaMetrics as datasource + grafana: + container_name: grafana + image: grafana/grafana:10.4.2 + depends_on: + - "vmauth" + ports: + - 3000:3000 + restart: always + volumes: + - grafanadata:/var/lib/grafana + - ../provisioning/datasources/prometheus-datasource:/etc/grafana/provisioning/datasources + - ../provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ../../../dashboards/victoriametrics-cluster.json:/var/lib/grafana/dashboards/vm.json + - ../../../dashboards/vmagent.json:/var/lib/grafana/dashboards/vmagent.json + - ../../../dashboards/vmalert.json:/var/lib/grafana/dashboards/vmalert.json + + # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts, + # where N is number of vmstorages (2 in this case). + vmstorage-1-1: + container_name: vmstorage-1-1 + image: victoriametrics/vmstorage:v1.101.0-cluster + ports: + - 8482 + - 8400 + - 8401 + volumes: + - strgdata-1:/storage + command: + - '--storageDataPath=/storage' + restart: always + vmstorage-1-2: + container_name: vmstorage-1-2 + image: victoriametrics/vmstorage:v1.101.0-cluster + ports: + - 8482 + - 8400 + - 8401 + volumes: + - strgdata-2:/storage + command: + - '--storageDataPath=/storage' + restart: always + + vmstorage-2-1: + container_name: vmstorage-2-1 + image: victoriametrics/vmstorage:v1.101.0-cluster + ports: + - 8482 + - 8400 + - 8401 + volumes: + - strgdata-3:/storage + command: + - '--storageDataPath=/storage' + restart: always + vmstorage-2-2: + container_name: vmstorage-2-2 + image: victoriametrics/vmstorage:v1.101.0-cluster + ports: + - 8482 + - 8400 + - 8401 + volumes: + - strgdata-4:/storage + command: + - '--storageDataPath=/storage' + restart: always + + # vminsert is ingestion frontend. It receives metrics pushed by vmagent, + # pre-process them and distributes across configured vmstorage shards. + vminsert-1: + container_name: vminsert-1 + image: victoriametrics/vminsert:v1.101.0-cluster + depends_on: + - "vmstorage-1-1" + - "vmstorage-1-2" + command: + - '--storageNode=vmstorage-1-1:8400' + - '--storageNode=vmstorage-1-2:8400' + ports: + - 8480 + restart: always + + vminsert-2: + container_name: vminsert-2 + image: victoriametrics/vminsert:v1.101.0-cluster + depends_on: + - "vmstorage-2-1" + - "vmstorage-2-2" + command: + - '--storageNode=vmstorage-2-1:8400' + - '--storageNode=vmstorage-2-2:8400' + ports: + - 8480 + restart: always + + # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL. + # vmselect collects results from configured `--storageNode` shards. + vmselect-1: + container_name: vmselect-1 + image: victoriametrics/vmselect:v1.101.0-cluster + depends_on: + - "vmstorage-1-1" + - "vmstorage-1-2" + command: + - '--storageNode=vmstorage-1-1:8401' + - '--storageNode=vmstorage-1-2:8401' + - '--vmalert.proxyURL=http://vmalert:8880' + ports: + - 8481 + restart: always + vmselect-2: + container_name: vmselect-2 + image: victoriametrics/vmselect:v1.101.0-cluster + depends_on: + - "vmstorage-2-1" + - "vmstorage-2-2" + command: + - '--storageNode=vmstorage-2-1:8401' + - '--storageNode=vmstorage-2-2:8401' + - '--vmalert.proxyURL=http://vmalert:8880' + ports: + - 8481 + restart: always + + # vmauth is a router and balancer for HTTP requests. + # It is configured via --auth.config and balances + # read requests from Grafana, vmui, vmalert among vmselects. + # It can be used as an authentication proxy. + vmauth: + container_name: vmauth + image: victoriametrics/vmauth:v1.101.0 + depends_on: + - "vmselect-1" + - "vmselect-2" + volumes: + - ./auth-cluster.yml:/etc/auth.yml + command: + - '--auth.config=/etc/auth.yml' + ports: + - 8427:8427 + restart: always + + # vmalert executes alerting and recording rules + vmalert: + container_name: vmalert + image: victoriametrics/vmalert:v1.101.0 + depends_on: + - "vmauth" + ports: + - 8880:8880 + volumes: + - ./alerts-cluster.yml:/etc/alerts/alerts.yml + - ./alerts-health.yml:/etc/alerts/alerts-health.yml + - ./alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml + - ./alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml + command: + - '--datasource.url=http://vmauth:8427/select/0/prometheus' + - '--remoteRead.url=http://vmauth:8427/select/0/prometheus' + - '--remoteWrite.url=http://vminsert:8480/insert/0/prometheus' + - '--notifier.url=http://alertmanager:9093/' + - '--rule=/etc/alerts/*.yml' + # display source of alerts in grafana + - '-external.url=http://127.0.0.1:3000' #grafana outside container + # when copypaste the line below be aware of '$$' for escaping in '$expr' + - '--external.alert.source=explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{$$expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}' + restart: always + + # alertmanager receives alerting notifications from vmalert + # and distributes them according to --config.file. + alertmanager: + container_name: alertmanager + image: prom/alertmanager:v0.27.0 + volumes: + - ./alertmanager.yml:/config/alertmanager.yml + command: + - '--config.file=/config/alertmanager.yml' + ports: + - 9093:9093 + restart: always + +volumes: + vmagentdata: {} + strgdata-1: {} + strgdata-2: {} + strgdata-3: {} + strgdata-4: {} + grafanadata: {}