From e64ea0109865465d5fa2d7112d511e5e5eac3bb6 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Wed, 4 Sep 2024 16:59:57 +0200 Subject: [PATCH] wip Signed-off-by: hagen1778 --- deployment/docker/docker-compose.yml | 79 +++++++++--------------- deployment/docker/monitoring.yml | 16 +++++ deployment/docker/prometheus.yml | 79 +++++++++++++++++++++--- deployment/docker/rules.yml | 5 ++ deployment/docker/stream_aggr_config.yml | 5 ++ 5 files changed, 127 insertions(+), 57 deletions(-) create mode 100644 deployment/docker/monitoring.yml create mode 100644 deployment/docker/rules.yml create mode 100644 deployment/docker/stream_aggr_config.yml diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index b9cd2655e..e4f88a59b 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -5,23 +5,25 @@ services: ports: - 9101:9101 command: - - "--metric-count=20" + - "--metric-count=1" - "--series-count=500" - - "--label-count=25" + - "--label-count=20" + - "--series-interval=20000000" + - "--metric-interval=100009999" - "--port=9101" networks: - vm_net restart: always - vmagent-no-aggregation: - container_name: vmagent-no-aggregation + vmagent-monitoring: + container_name: vmagent-monitoring image: victoriametrics/vmagent:v1.103.0 depends_on: - "victoriametrics" ports: - 8429 volumes: - - vmagentdata-no-aggregation:/vmagentdata - - ./prometheus.yml:/etc/prometheus/prometheus.yml + - vmagent-monitoring:/vmagentdata + - ./monitoring.yml:/etc/prometheus/prometheus.yml command: - "--promscrape.config=/etc/prometheus/prometheus.yml" - "--remoteWrite.url=http://victoriametrics:8428/api/v1/write" @@ -43,9 +45,29 @@ services: - "--promscrape.config=/etc/prometheus/prometheus.yml" - "--streamAggr.config=/etc/prometheus/stream_aggr_config.yml" - "--remoteWrite.url=http://victoriametrics:8428/api/v1/write" + - "--promscrape.config.strictParse=false" networks: - vm_net restart: always + + prometheus: + container_name: prometheus + image: prom/prometheus:v2.54.1 + depends_on: + - "victoriametrics" + ports: + - 9090:9090 + volumes: + - prometheus:/prometheus + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./rules.yml:/etc/prometheus/rules.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + networks: + - vm_net + restart: always + # VictoriaMetrics instance, a single process responsible for # storing metrics and serve read requests. victoriametrics: @@ -81,50 +103,9 @@ services: - vm_net restart: always - # vmalert executes alerting and recording rules - vmalert: - container_name: vmalert - image: victoriametrics/vmalert:v1.103.0 - depends_on: - - "victoriametrics" - - "alertmanager" - ports: - - 8880:8880 - volumes: - - ./alerts.yml:/etc/alerts/alerts.yml - - ./alerts-health.yml:/etc/alerts/alerts-health.yml - - ./alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml - - ./alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml - command: - - "--datasource.url=http://victoriametrics:8428/" - - "--remoteRead.url=http://victoriametrics:8428/" - - "--remoteWrite.url=http://victoriametrics:8428/" - - "--notifier.url=http://alertmanager:9093/" - - "--rule=/etc/alerts/*.yml" - # display source of alerts in grafana - - "--external.url=http://127.0.0.1:3000" #grafana outside container - - '--external.alert.source=explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{.Expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"{{ .ActiveAt.UnixMilli }}","to":"now"}}' - networks: - - vm_net - restart: always - - # alertmanager receives alerting notifications from vmalert - # and distributes them according to --config.file. - alertmanager: - container_name: alertmanager - image: prom/alertmanager:v0.27.0 - volumes: - - ./alertmanager.yml:/config/alertmanager.yml - command: - - "--config.file=/config/alertmanager.yml" - ports: - - 9093:9093 - networks: - - vm_net - restart: always - volumes: - vmagentdata-no-aggregation: {} + prometheus: {} + vmagent-monitoring: {} vmagentdata-with-aggregation: {} vmdata: {} grafanadata: {} diff --git a/deployment/docker/monitoring.yml b/deployment/docker/monitoring.yml new file mode 100644 index 000000000..298a496d0 --- /dev/null +++ b/deployment/docker/monitoring.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 5s + +scrape_configs: +- job_name: 'vmagent-monitoring' + static_configs: + - targets: ['vmagent-monitoring:8429'] +- job_name: 'prometheus' + static_configs: + - targets: ['prometheus:9090'] +- job_name: 'vmagent-with-aggregation' + static_configs: + - targets: ['vmagent-with-aggregation:8429'] +- job_name: 'victoriametrics' + static_configs: + - targets: ['victoriametrics:8428'] \ No newline at end of file diff --git a/deployment/docker/prometheus.yml b/deployment/docker/prometheus.yml index b95cf3a4b..5eea0a451 100644 --- a/deployment/docker/prometheus.yml +++ b/deployment/docker/prometheus.yml @@ -1,16 +1,79 @@ global: - scrape_interval: 5s + scrape_interval: 15s + evaluation_interval: 5m + +rule_files: + - "rules.yml" + +remote_write: + - url: http://victoriametrics:8428/api/v1/write + write_relabel_configs: + - source_labels: [ __name__ ] + regex: .*sum_over_time.* + action: keep + queue_config: + max_samples_per_send: 10000 + capacity: 20000 + max_shards: 30 scrape_configs: - - job_name: 'vmagent' + - job_name: 'avalanche1' static_configs: - - targets: ['vmagent-no-aggregation:8429', 'vmagent-with-aggregation:8429'] - - job_name: 'vmalert' + - targets: ['avalanche:9101'] + - job_name: 'avalanche2' static_configs: - - targets: ['vmalert:8880'] - - job_name: 'victoriametrics' + - targets: ['avalanche:9101'] + - job_name: 'avalanche3' static_configs: - - targets: ['victoriametrics:8428'] - - job_name: 'avalanche' + - targets: ['avalanche:9101'] + - job_name: 'avalanche4' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche5' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche6' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche7' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche8' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche9' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche10' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche11' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche12' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche13' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche14' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche15' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche16' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche17' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche18' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche19' + static_configs: + - targets: ['avalanche:9101'] + - job_name: 'avalanche20' static_configs: - targets: ['avalanche:9101'] \ No newline at end of file diff --git a/deployment/docker/rules.yml b/deployment/docker/rules.yml new file mode 100644 index 000000000..a91d681c7 --- /dev/null +++ b/deployment/docker/rules.yml @@ -0,0 +1,5 @@ +groups: + - name: demo + rules: + - record: cycle:avalanche:sum_over_time:5m + expr: sum(sum_over_time(avalanche_metric_mmmmm_0_0[5m])) by(cycle_id) \ No newline at end of file diff --git a/deployment/docker/stream_aggr_config.yml b/deployment/docker/stream_aggr_config.yml new file mode 100644 index 000000000..472de3be7 --- /dev/null +++ b/deployment/docker/stream_aggr_config.yml @@ -0,0 +1,5 @@ +- match: 'avalanche_metric_mmmmm_0_0' + interval: 5m + by: ["cycle_id"] + outputs: + - sum_samples \ No newline at end of file