From 33f40f4a5f017903274bf886b8498a2cf31bf972 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Fri, 30 Sep 2022 16:28:35 +0200 Subject: [PATCH] app/vminsert: allows parsing tenant id from labels (#3009) * app/vminsert: allows parsing tenant id from labels it should help mitigate issues with vmagent's multiTenant mode, which works incorrectly at heavy load and it cannot handle more then 100 different tenants. This functional hidden with flag and do not change vminsert default behaviour https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2970 * Update docs/Cluster-VictoriaMetrics.md Co-authored-by: Roman Khavronenko * wip * app/vminsert/netstorage: clean remaining labels in order to free up GC * docs/Cluster-VictoriaMetrics.md: typo fix * wip * wip Co-authored-by: Roman Khavronenko Co-authored-by: Aliaksandr Valialkin --- app/vmagent/README.md | 4 +++ docs/CHANGELOG.md | 1 + docs/Cluster-VictoriaMetrics.md | 32 +++++++++++++++++++++-- docs/vmagent.md | 4 +++ lib/auth/auth.go | 44 ++++++++++++++++++++++++-------- lib/auth/auth_test.go | 2 ++ lib/httpserver/path.go | 6 +++-- lib/tenantmetrics/counter_map.go | 7 +++++ 8 files changed, 86 insertions(+), 14 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 51cc9e71d..e7c25dde9 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -142,6 +142,10 @@ While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`. +[VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) supports writing data to multiple tenants +specified via special labels - see [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels). +This allows specifying tenant ids via [relabeling](#relabeling) and writing multitenant data to a single `-remoteWrite.url=http:///insert/multitenant/api/v1/write`. + [Multitenancy](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) support is enabled when `-remoteWrite.multitenantURL` command-line flag is set. In this case `vmagent` accepts multitenant data at `http://vmagent:8429/insert//...` in the same way as cluster version of VictoriaMetrics does according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) and routes it to `<-remoteWrite.multitenantURL>/insert//prometheus/api/v1/write`. If multiple `-remoteWrite.multitenantURL` command-line options are set, then `vmagent` replicates the collected data across all the configured urls. This allows using a single `vmagent` instance in front of VictoriaMetrics clusters for processing the data from all the tenants. If `-remoteWrite.multitenantURL` command-line flag is set and `vmagent` is configured to scrape Prometheus-compatible targets (e.g. if `-promscrape.config` command-line flag is set) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b645e8478..89d9e6961 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -19,6 +19,7 @@ The following tip changes can be tested by building VictoriaMetrics components f **Update note 2:** [vmalert](https://docs.victoriametrics.com/vmalert.html) changes default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval. +* FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): support specifying tenant ids via `vm_account_id` and `vm_project_id` labels. See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2970). * FEATURE: improve [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) performance by up to 3x if non-trivial `regex` values are used. * FEATURE: sanitize metric names for data ingested via [DataDog protocol](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent) according to [DataDog metric naming](https://docs.datadoghq.com/metrics/custom_metrics/#naming-custom-metrics). The behaviour can be disabled by passing `-datadog.sanitizeMetricName=false` command-line flag. Thanks to @PerGon for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3105). * FEATURE: add `-usePromCompatibleNaming` command-line flag to [vmagent](https://docs.victoriametrics.com/vmagent.html), to single-node VictoriaMetrics and to `vminsert` component of VictoriaMetrics cluster. This flag can be used for normalizing the ingested metric names and label names to [Prometheus-compatible form](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels). If this flag is set, then all the chars unsupported by Prometheus are replaced with `_` chars in metric names and labels of the ingested samples. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113). diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index b73d0be77..ffe7aeabb 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -43,7 +43,9 @@ It increases cluster availability, and simplifies cluster maintenance as well as VictoriaMetrics cluster supports multiple isolated tenants (aka namespaces). Tenants are identified by `accountID` or `accountID:projectID`, which are put inside request urls. -See [these docs](#url-format) for details. Some facts about tenants in VictoriaMetrics: +See [these docs](#url-format) for details. + +Some facts about tenants in VictoriaMetrics: - Each `accountID` and `projectID` is identified by an arbitrary 32-bit integer in the range `[0 .. 2^32)`. If `projectID` is missing, then it is automatically assigned to `0`. It is expected that other information about tenants @@ -59,6 +61,30 @@ when different tenants have different amounts of data and different query load. - VictoriaMetrics doesn't support querying multiple tenants in a single request. +See also [multitenancy via labels](#multitenancy-via-labels). + + +## Multitenancy via labels + +`vminsert` can accept data from multiple [tenants](#multitenancy) via a special `multitenant` endpoints `http://vminsert:8480/insert/multitenant/`, +where `` can be replaced with any supported suffix for data ingestion from [this list](#url-format). +In this case the account id and project id are obtained from optional `vm_account_id` and `vm_project_id` labels of the incoming samples. +If `vm_account_id` or `vm_project_id` labels are missing or invalid, then the corresponding `accountID` or `projectID` is set to 0. +These labels are automatically removed from samples before forwarding them to `vmstorage`. +For example, if the following samples are written into `http://vminsert:8480/insert/multitenant/prometheus/api/v1/write`: + +``` +http_requests_total{path="/foo",vm_account_id="42"} 12 +http_requests_total{path="/bar",vm_account_id="7",vm_project_id="9"} 34 +``` + +Then the `http_requests_total{path="/foo"} 12` would be stored in the tenant `accountID=42, projectID=0`, +while the `http_requests_total{path="/bar"} 34` would be stored in the tenant `accountID=7, projectID=9`. + +The `vm_account_id` and `vm_project_id` labels are extracted after applying the [relabeling](https://docs.victoriametrics.com/relabeling.html) +set via `-relabelConfig` command-line flag, so these labels can be set at this stage. + + ## Binaries Compiled binaries for the cluster version are available in the `assets` section of the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases). @@ -218,7 +244,9 @@ See [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html - URLs for data ingestion: `http://:8480/insert//`, where: - `` is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). It is possible to set it as `accountID:projectID`, - where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. + where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. See [multitenancy docs](#multitenancy) for more details. + The `` can be set to `multitenant` string, e.g. `http://:8480/insert/multitenant/`. Such urls accept data from multiple tenants + specified via `vm_account_id` and `vm_project_id` labels. See [multitenancy via labels](#multitenancy-via-labels) for more details. - `` may have the following values: - `prometheus` and `prometheus/api/v1/write` - for inserting data with [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). - `datadog/api/v1/series` - for inserting data with [DataDog submit metrics API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent) for details. diff --git a/docs/vmagent.md b/docs/vmagent.md index 7ef86bad4..315d70418 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -146,6 +146,10 @@ While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`. +[VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) supports writing data to multiple tenants +specified via special labels - see [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels). +This allows specifying tenant ids via [relabeling](#relabeling) and writing multitenant data to a single `-remoteWrite.url=http:///insert/multitenant/api/v1/write`. + [Multitenancy](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) support is enabled when `-remoteWrite.multitenantURL` command-line flag is set. In this case `vmagent` accepts multitenant data at `http://vmagent:8429/insert//...` in the same way as cluster version of VictoriaMetrics does according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) and routes it to `<-remoteWrite.multitenantURL>/insert//prometheus/api/v1/write`. If multiple `-remoteWrite.multitenantURL` command-line options are set, then `vmagent` replicates the collected data across all the configured urls. This allows using a single `vmagent` instance in front of VictoriaMetrics clusters for processing the data from all the tenants. If `-remoteWrite.multitenantURL` command-line flag is set and `vmagent` is configured to scrape Prometheus-compatible targets (e.g. if `-promscrape.config` command-line flag is set) diff --git a/lib/auth/auth.go b/lib/auth/auth.go index 6d27943e3..38e235448 100644 --- a/lib/auth/auth.go +++ b/lib/auth/auth.go @@ -14,30 +14,54 @@ type Token struct { // String returns string representation of t. func (t *Token) String() string { + if t == nil { + return "multitenant" + } if t.ProjectID == 0 { return fmt.Sprintf("%d", t.AccountID) } return fmt.Sprintf("%d:%d", t.AccountID, t.ProjectID) } -// NewToken returns new Token for the given authToken +// NewToken returns new Token for the given authToken. +// +// If authToken == "multitenant", then nil Token is returned. func NewToken(authToken string) (*Token, error) { + if authToken == "multitenant" { + return nil, nil + } + var t Token + if err := t.Init(authToken); err != nil { + return nil, err + } + return &t, nil +} + +// Init initializes t from authToken. +func (t *Token) Init(authToken string) error { tmp := strings.Split(authToken, ":") if len(tmp) > 2 { - return nil, fmt.Errorf("unexpected number of items in authToken %q; got %d; want 1 or 2", authToken, len(tmp)) + return fmt.Errorf("unexpected number of items in authToken %q; got %d; want 1 or 2", authToken, len(tmp)) } - var at Token - accountID, err := strconv.ParseUint(tmp[0], 10, 32) + n, err := strconv.ParseUint(tmp[0], 10, 32) if err != nil { - return nil, fmt.Errorf("cannot parse accountID from %q: %w", tmp[0], err) + return fmt.Errorf("cannot parse accountID from %q: %w", tmp[0], err) } - at.AccountID = uint32(accountID) + accountID := uint32(n) + projectID := uint32(0) if len(tmp) > 1 { - projectID, err := strconv.ParseUint(tmp[1], 10, 32) + n, err := strconv.ParseUint(tmp[1], 10, 32) if err != nil { - return nil, fmt.Errorf("cannot parse projectID from %q: %w", tmp[1], err) + return fmt.Errorf("cannot parse projectID from %q: %w", tmp[1], err) } - at.ProjectID = uint32(projectID) + projectID = uint32(n) } - return &at, nil + t.Set(accountID, projectID) + return nil +} + +// Set sets accountID and projectID for the t. +func (t *Token) Set(accountID, projectID uint32) { + t.AccountID = accountID + t.ProjectID = projectID } diff --git a/lib/auth/auth_test.go b/lib/auth/auth_test.go index aa2b4b4f9..eddf1c6fe 100644 --- a/lib/auth/auth_test.go +++ b/lib/auth/auth_test.go @@ -26,6 +26,8 @@ func TestNewTokenSuccess(t *testing.T) { f("1:4294967295", "1:4294967295") // max uint32 accountID and projectID f("4294967295:4294967295", "4294967295:4294967295") + // multitenant + f("multitenant", "multitenant") } func TestNewTokenFailure(t *testing.T) { diff --git a/lib/httpserver/path.go b/lib/httpserver/path.go index 833c0a6b4..3743341e6 100644 --- a/lib/httpserver/path.go +++ b/lib/httpserver/path.go @@ -22,6 +22,8 @@ func ParsePath(path string) (*Path, error) { // // - prefix must contain `select`, `insert` or `delete`. // - authToken contains `accountID[:projectID]`, where projectID is optional. + // authToken may also contain `multitenant` string. In this case the accountID and projectID + // are obtained from vm_account_id and vm_project_id labels of the ingested samples. // - suffix contains arbitrary suffix. // // prefix must be used for the routing to the appropriate service @@ -29,14 +31,14 @@ func ParsePath(path string) (*Path, error) { s := skipPrefixSlashes(path) n := strings.IndexByte(s, '/') if n < 0 { - return nil, fmt.Errorf("cannot find {prefix}") + return nil, fmt.Errorf("cannot find {prefix} in %q; expecting /{prefix}/{authToken}/{suffix} format", path) } prefix := s[:n] s = skipPrefixSlashes(s[n+1:]) n = strings.IndexByte(s, '/') if n < 0 { - return nil, fmt.Errorf("cannot find {authToken}") + return nil, fmt.Errorf("cannot find {authToken} in %q; expecting /{prefix}/{authToken}/{suffix} format", path) } authToken := s[:n] diff --git a/lib/tenantmetrics/counter_map.go b/lib/tenantmetrics/counter_map.go index 710986b80..3ab106679 100644 --- a/lib/tenantmetrics/counter_map.go +++ b/lib/tenantmetrics/counter_map.go @@ -39,6 +39,13 @@ func (cm *CounterMap) Get(at *auth.Token) *metrics.Counter { return cm.GetByTenant(key) } +// MultiAdd adds multiple values grouped by auth.Token +func (cm *CounterMap) MultiAdd(perTenantValues map[auth.Token]int) { + for token, value := range perTenantValues { + cm.Get(&token).Add(value) + } +} + // GetByTenant returns counter for the given key. func (cm *CounterMap) GetByTenant(key TenantID) *metrics.Counter { m := cm.m.Load().(map[TenantID]*metrics.Counter)