From 9bb60ab00fdbba758da54f860ad3d5c034f7c35c Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 8 Feb 2022 15:37:38 +0200 Subject: [PATCH] lib/promscrape: set `-promscrape.config.strictParse` to true by default This allows detecting long-living silent errors in -promscrape.config --- README.md | 6 ++-- app/victoria-metrics/main.go | 2 +- app/vmagent/README.md | 32 ++++++++++++++++++-- app/vmagent/main.go | 2 +- app/vmalert/README.md | 42 ++++++++++++++++++--------- app/vmauth/README.md | 2 ++ docs/CHANGELOG.md | 1 + docs/README.md | 6 ++-- docs/Single-server-VictoriaMetrics.md | 6 ++-- docs/vmagent.md | 32 ++++++++++++++++++-- docs/vmalert.md | 42 ++++++++++++++++++--------- docs/vmauth.md | 2 ++ lib/promscrape/config.go | 9 +++--- 13 files changed, 137 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 525e6cca0..415daedac 100644 --- a/README.md +++ b/README.md @@ -1596,7 +1596,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details Supports an array of values separated by comma or specified via multiple flags. -dryRun - Whether to check only -promscrape.config and then exit. Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse + Whether to check only -promscrape.config and then exit. Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag -enableTCP6 Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used -envflag.enable @@ -1604,7 +1604,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -envflag.prefix string Prefix for environment variables if -envflag.enable is set -eula - By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/legal/eula/ + By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf -finalMergeDelay duration The delay before starting final merge for per-month partition after no new data is ingested into it. Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. Zero value disables final merge -forceFlushAuthKey string @@ -1714,7 +1714,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -promscrape.config.dryRun Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output. -promscrape.config.strictParse - Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped + Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true) -promscrape.configCheckInterval duration Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes -promscrape.consul.waitTime duration diff --git a/app/victoria-metrics/main.go b/app/victoria-metrics/main.go index cbed6f0f1..8511f1f5d 100644 --- a/app/victoria-metrics/main.go +++ b/app/victoria-metrics/main.go @@ -27,7 +27,7 @@ var ( minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the first sample in every time series per each discrete interval "+ "equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling") dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+ - "Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse") + "Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag") ) func main() { diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 64bd9d937..86dd8bb12 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -708,13 +708,15 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . The maximum size in bytes of a single DataDog POST request to /api/v1/series Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864) -dryRun - Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse + Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag -enableTCP6 Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used -envflag.enable Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details -envflag.prefix string Prefix for environment variables if -envflag.enable is set + -eula + By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf -fs.disableMmap Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread() -graphiteListenAddr string @@ -760,6 +762,32 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms) -insert.maxQueueDuration duration The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s) + -kafka.consumer.topic array + Kafka topic names for data consumption. + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.basicAuth.password array + Optional basic auth password for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN' + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.basicAuth.username array + Optional basic auth username for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN' + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.brokers array + List of brokers to connect for given topic, e.g. -kafka.consumer.topic.broker=host-1:9092;host-2:9092 + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.defaultFormat string + Expected data format in the topic if -kafka.consumer.topic.format is skipped. (default "promremotewrite") + -kafka.consumer.topic.format array + data format for corresponding kafka topic. Valid formats: influx, prometheus, promremotewrite, graphite, jsonline + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.groupID array + Defines group.id for topic + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.isGzipped array + Enables gzip setting for topic messages payload. Only prometheus, jsonline and influx formats accept gzipped messages. + Supports array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.options array + Optional key=value;key1=value2 settings for topic consumer. See full configuration options at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md. + Supports an array of values separated by comma or specified via multiple flags. -loggerDisableTimestamps Whether to disable writing timestamps in logs -loggerErrorsPerSecondLimit int @@ -810,7 +838,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -promscrape.config.dryRun Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output. -promscrape.config.strictParse - Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped + Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true) -promscrape.configCheckInterval duration Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes -promscrape.consul.waitTime duration diff --git a/app/vmagent/main.go b/app/vmagent/main.go index dcfcf845e..6190d9ed9 100644 --- a/app/vmagent/main.go +++ b/app/vmagent/main.go @@ -53,7 +53,7 @@ var ( configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg") dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+ "-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+ - "Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse") + "Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag") ) var ( diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 39bbb329c..3cc86000e 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -494,6 +494,10 @@ command-line flags with their descriptions. The shortlist of configuration flags is the following: ``` + -clusterMode + If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy + -configCheckInterval duration + Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. -datasource.appendTypePrefix Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL. -datasource.basicAuth.password string @@ -526,8 +530,12 @@ The shortlist of configuration flags is the following: Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used -datasource.url string VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428 + -defaultTenant.graphite string + Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy + -defaultTenant.prometheus string + Default tenant for Prometheus alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy -disableAlertgroupLabel - Whether to disable adding group's name as label to generated alerts and time series. + Whether to disable adding group's Name as label to generated alerts and time series. -dryRun -rule Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified. -enableTCP6 @@ -536,13 +544,15 @@ The shortlist of configuration flags is the following: Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details -envflag.prefix string Prefix for environment variables if -envflag.enable is set + -eula + By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf -evaluationInterval duration How often to evaluate the rules (default 1m0s) -external.alert.source string External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used -external.label array - Optional label in the form 'name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets. + Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets. Supports an array of values separated by comma or specified via multiple flags. -external.url string External URL is used as alert's source for sent alerts to the notifier @@ -591,11 +601,15 @@ The shortlist of configuration flags is the following: Optional basic auth password for -notifier.url Supports an array of values separated by comma or specified via multiple flags. -notifier.basicAuth.passwordFile array - Optional path to basic auth password file for -notifier.url - Supports an array of values separated by comma or specified via multiple flags. + Optional path to basic auth password file for -notifier.url + Supports an array of values separated by comma or specified via multiple flags. -notifier.basicAuth.username array Optional basic auth username for -notifier.url Supports an array of values separated by comma or specified via multiple flags. + -notifier.config string + Path to configuration file for notifiers + -notifier.suppressDuplicateTargetErrors + Whether to suppress 'duplicate target' errors during discovery -notifier.tlsCAFile array Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used Supports an array of values separated by comma or specified via multiple flags. @@ -616,6 +630,14 @@ The shortlist of configuration flags is the following: Supports an array of values separated by comma or specified via multiple flags. -pprofAuthKey string Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings + -promscrape.consul.waitTime duration + Wait time used by Consul service discovery. Default value is used if not set + -promscrape.consulSDCheckInterval duration + Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s) + -promscrape.discovery.concurrency int + The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100) + -promscrape.discovery.concurrentWaitTime duration + The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s) -remoteRead.basicAuth.password string Optional basic auth password for -remoteRead.url -remoteRead.basicAuth.passwordFile string @@ -695,8 +717,8 @@ The shortlist of configuration flags is the following: absolute path to all .yaml files in root. Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. Supports an array of values separated by comma or specified via multiple flags. - -configCheckInterval duration - Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes + -rule.configCheckInterval duration + Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead -rule.maxResolveDuration duration Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group. -rule.validateExpressions @@ -709,14 +731,6 @@ The shortlist of configuration flags is the following: Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower -tlsKeyFile string Path to file with TLS key. Used only if -tls is set - -promscrape.consul.waitTime duration - Wait time used by Consul service discovery. Default value is used if not set - -promscrape.consulSDCheckInterval duration - Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s) - -promscrape.discovery.concurrency int - The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100) - -promscrape.discovery.concurrentWaitTime duration - The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s) -version Show VictoriaMetrics version ``` diff --git a/app/vmauth/README.md b/app/vmauth/README.md index eff0ac7e5..ecb29d23f 100644 --- a/app/vmauth/README.md +++ b/app/vmauth/README.md @@ -223,6 +223,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html . Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details -envflag.prefix string Prefix for environment variables if -envflag.enable is set + -eula + By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf -fs.disableMmap Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread() -http.connTimeout duration diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b32e5a10e..4f885a569 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -20,6 +20,7 @@ sort: 15 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add support for specifying Basic Auth password for notifiers via a file. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1567). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): provide the ability to fetch target responses on behalf of `vmagent` by clicking the `response` link for the needed target at `/targets` page. This feature may be useful for debugging responses from targets located in isolated environments. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): show the total number of scrapes and the total number of scrape errors per target at `/targets` page. This information may be useful when debugging unreliable scrape targets. +* FEATURE: vmagent and single-node VictoriaMetrics: disallow unknown fields at `-promscrape.config` file. Previously unknown fields were allowed. This could lead to long-living silent config errors. The previous behaviour can be returned by passing `-promscrape.config.strictParse=false` command-line flag. * BUGFIX: return proper results from `highestMax()` function at [Graphite render API](https://docs.victoriametrics.com/#graphite-render-api-usage). Previously it was incorrectly returning timeseries with min peaks instead of max peaks. * BUGFIX: properly limit indexdb cache sizes. Previously they could exceed values set via `-memory.allowedPercent` and/or `-memory.allowedBytes` when `indexdb` contained many data parts. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2007). diff --git a/docs/README.md b/docs/README.md index 525e6cca0..415daedac 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1596,7 +1596,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details Supports an array of values separated by comma or specified via multiple flags. -dryRun - Whether to check only -promscrape.config and then exit. Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse + Whether to check only -promscrape.config and then exit. Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag -enableTCP6 Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used -envflag.enable @@ -1604,7 +1604,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -envflag.prefix string Prefix for environment variables if -envflag.enable is set -eula - By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/legal/eula/ + By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf -finalMergeDelay duration The delay before starting final merge for per-month partition after no new data is ingested into it. Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. Zero value disables final merge -forceFlushAuthKey string @@ -1714,7 +1714,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -promscrape.config.dryRun Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output. -promscrape.config.strictParse - Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped + Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true) -promscrape.configCheckInterval duration Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes -promscrape.consul.waitTime duration diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 58c546836..137c26f2a 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1600,7 +1600,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details Supports an array of values separated by comma or specified via multiple flags. -dryRun - Whether to check only -promscrape.config and then exit. Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse + Whether to check only -promscrape.config and then exit. Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag -enableTCP6 Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used -envflag.enable @@ -1608,7 +1608,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -envflag.prefix string Prefix for environment variables if -envflag.enable is set -eula - By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/legal/eula/ + By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf -finalMergeDelay duration The delay before starting final merge for per-month partition after no new data is ingested into it. Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. Zero value disables final merge -forceFlushAuthKey string @@ -1718,7 +1718,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -promscrape.config.dryRun Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output. -promscrape.config.strictParse - Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped + Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true) -promscrape.configCheckInterval duration Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes -promscrape.consul.waitTime duration diff --git a/docs/vmagent.md b/docs/vmagent.md index d70b0279a..563a9fc07 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -712,13 +712,15 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . The maximum size in bytes of a single DataDog POST request to /api/v1/series Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 67108864) -dryRun - Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse + Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag -enableTCP6 Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used -envflag.enable Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details -envflag.prefix string Prefix for environment variables if -envflag.enable is set + -eula + By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf -fs.disableMmap Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread() -graphiteListenAddr string @@ -764,6 +766,32 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms) -insert.maxQueueDuration duration The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s) + -kafka.consumer.topic array + Kafka topic names for data consumption. + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.basicAuth.password array + Optional basic auth password for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN' + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.basicAuth.username array + Optional basic auth username for -kafka.consumer.topic. Must be used in conjunction with any supported auth methods for kafka client, specified by flag -kafka.consumer.topic.options='security.protocol=SASL_SSL;sasl.mechanisms=PLAIN' + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.brokers array + List of brokers to connect for given topic, e.g. -kafka.consumer.topic.broker=host-1:9092;host-2:9092 + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.defaultFormat string + Expected data format in the topic if -kafka.consumer.topic.format is skipped. (default "promremotewrite") + -kafka.consumer.topic.format array + data format for corresponding kafka topic. Valid formats: influx, prometheus, promremotewrite, graphite, jsonline + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.groupID array + Defines group.id for topic + Supports an array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.isGzipped array + Enables gzip setting for topic messages payload. Only prometheus, jsonline and influx formats accept gzipped messages. + Supports array of values separated by comma or specified via multiple flags. + -kafka.consumer.topic.options array + Optional key=value;key1=value2 settings for topic consumer. See full configuration options at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md. + Supports an array of values separated by comma or specified via multiple flags. -loggerDisableTimestamps Whether to disable writing timestamps in logs -loggerErrorsPerSecondLimit int @@ -814,7 +842,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -promscrape.config.dryRun Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output. -promscrape.config.strictParse - Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped + Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true) -promscrape.configCheckInterval duration Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes -promscrape.consul.waitTime duration diff --git a/docs/vmalert.md b/docs/vmalert.md index b07568bf7..060d92009 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -498,6 +498,10 @@ command-line flags with their descriptions. The shortlist of configuration flags is the following: ``` + -clusterMode + If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy + -configCheckInterval duration + Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. -datasource.appendTypePrefix Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL. -datasource.basicAuth.password string @@ -530,8 +534,12 @@ The shortlist of configuration flags is the following: Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used -datasource.url string VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428 + -defaultTenant.graphite string + Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy + -defaultTenant.prometheus string + Default tenant for Prometheus alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy -disableAlertgroupLabel - Whether to disable adding group's name as label to generated alerts and time series. + Whether to disable adding group's Name as label to generated alerts and time series. -dryRun -rule Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified. -enableTCP6 @@ -540,13 +548,15 @@ The shortlist of configuration flags is the following: Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details -envflag.prefix string Prefix for environment variables if -envflag.enable is set + -eula + By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf -evaluationInterval duration How often to evaluate the rules (default 1m0s) -external.alert.source string External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service. eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|crlfEscape|queryEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used -external.label array - Optional label in the form 'name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets. + Optional label in the form 'Name=value' to add to all generated recording rules and alerts. Pass multiple -label flags in order to add multiple label sets. Supports an array of values separated by comma or specified via multiple flags. -external.url string External URL is used as alert's source for sent alerts to the notifier @@ -595,11 +605,15 @@ The shortlist of configuration flags is the following: Optional basic auth password for -notifier.url Supports an array of values separated by comma or specified via multiple flags. -notifier.basicAuth.passwordFile array - Optional path to basic auth password file for -notifier.url - Supports an array of values separated by comma or specified via multiple flags. + Optional path to basic auth password file for -notifier.url + Supports an array of values separated by comma or specified via multiple flags. -notifier.basicAuth.username array Optional basic auth username for -notifier.url Supports an array of values separated by comma or specified via multiple flags. + -notifier.config string + Path to configuration file for notifiers + -notifier.suppressDuplicateTargetErrors + Whether to suppress 'duplicate target' errors during discovery -notifier.tlsCAFile array Optional path to TLS CA file to use for verifying connections to -notifier.url. By default system CA is used Supports an array of values separated by comma or specified via multiple flags. @@ -620,6 +634,14 @@ The shortlist of configuration flags is the following: Supports an array of values separated by comma or specified via multiple flags. -pprofAuthKey string Auth key for /debug/pprof. It must be passed via authKey query arg. It overrides httpAuth.* settings + -promscrape.consul.waitTime duration + Wait time used by Consul service discovery. Default value is used if not set + -promscrape.consulSDCheckInterval duration + Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s) + -promscrape.discovery.concurrency int + The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100) + -promscrape.discovery.concurrentWaitTime duration + The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s) -remoteRead.basicAuth.password string Optional basic auth password for -remoteRead.url -remoteRead.basicAuth.passwordFile string @@ -699,8 +721,8 @@ The shortlist of configuration flags is the following: absolute path to all .yaml files in root. Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars. Supports an array of values separated by comma or specified via multiple flags. - -configCheckInterval duration - Interval for checking for changes in '-rule' or '-notifier.config' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes + -rule.configCheckInterval duration + Interval for checking for changes in '-rule' files. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead -rule.maxResolveDuration duration Limits the maximum duration for automatic alert expiration, which is by default equal to 3 evaluation intervals of the parent group. -rule.validateExpressions @@ -713,14 +735,6 @@ The shortlist of configuration flags is the following: Path to file with TLS certificate. Used only if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower -tlsKeyFile string Path to file with TLS key. Used only if -tls is set - -promscrape.consul.waitTime duration - Wait time used by Consul service discovery. Default value is used if not set - -promscrape.consulSDCheckInterval duration - Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s) - -promscrape.discovery.concurrency int - The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100) - -promscrape.discovery.concurrentWaitTime duration - The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s) -version Show VictoriaMetrics version ``` diff --git a/docs/vmauth.md b/docs/vmauth.md index 5edd39bba..4e087fed4 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -227,6 +227,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html . Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details -envflag.prefix string Prefix for environment variables if -envflag.enable is set + -eula + By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf -fs.disableMmap Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread() -http.connTimeout duration diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 9a1641bc3..97f867775 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -37,9 +37,8 @@ import ( ) var ( - strictParse = flag.Bool("promscrape.config.strictParse", false, "Whether to allow only supported fields in -promscrape.config . "+ - "By default unsupported fields are silently skipped") - dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+ + strictParse = flag.Bool("promscrape.config.strictParse", true, "Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields") + dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+ "Returns non-zero exit code on parsing errors and emits these errors to stderr. "+ "See also -promscrape.config.strictParse command-line flag. "+ "Pass -loggerLevel=ERROR if you don't need to see info messages in the output.") @@ -335,7 +334,9 @@ func unmarshalMaybeStrict(data []byte, dst interface{}) error { data = envtemplate.Replace(data) var err error if *strictParse { - err = yaml.UnmarshalStrict(data, dst) + if err = yaml.UnmarshalStrict(data, dst); err != nil { + err = fmt.Errorf("%w; pass -promscrape.config.strictParse=false command-line flag for ignoring unknown fields in yaml config", err) + } } else { err = yaml.Unmarshal(data, dst) }