diff --git a/Makefile b/Makefile index b6ba009a6d..5d020de7a5 100644 --- a/Makefile +++ b/Makefile @@ -323,15 +323,7 @@ install-golint: errcheck: install-errcheck errcheck -exclude=errcheck_excludes.txt ./lib/... - errcheck -exclude=errcheck_excludes.txt ./app/vminsert/... - errcheck -exclude=errcheck_excludes.txt ./app/vmselect/... - errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/... - errcheck -exclude=errcheck_excludes.txt ./app/vmagent/... - errcheck -exclude=errcheck_excludes.txt ./app/vmalert/... - errcheck -exclude=errcheck_excludes.txt ./app/vmauth/... - errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/... - errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/... - errcheck -exclude=errcheck_excludes.txt ./app/vmctl/... + errcheck -exclude=errcheck_excludes.txt ./app/... install-errcheck: which errcheck || go install github.com/kisielk/errcheck@latest diff --git a/app/victoria-metrics/main_test.go b/app/victoria-metrics/main_test.go index 805a5567a3..d5f27b0a43 100644 --- a/app/victoria-metrics/main_test.go +++ b/app/victoria-metrics/main_test.go @@ -138,7 +138,7 @@ func setUp() { if err != nil { return false } - resp.Body.Close() + _ = resp.Body.Close() return resp.StatusCode == 200 } if err := waitFor(testStorageInitTimeout, readyStorageCheckFunc); err != nil { @@ -337,7 +337,9 @@ func tcpWrite(t *testing.T, address string, data string) { s := newSuite(t) conn, err := net.Dial("tcp", address) s.noError(err) - defer conn.Close() + defer func() { + _ = conn.Close() + }() n, err := conn.Write([]byte(data)) s.noError(err) s.equalInt(n, len(data)) @@ -348,7 +350,9 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric { s := newSuite(t) resp, err := http.Get(address + query) s.noError(err) - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() s.equalInt(resp.StatusCode, 200) var rows []Metric for dec := json.NewDecoder(resp.Body); dec.More(); { @@ -363,7 +367,9 @@ func httpReadStruct(t *testing.T, address, query string, dst interface{}) { s := newSuite(t) resp, err := http.Get(address + query) s.noError(err) - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() s.equalInt(resp.StatusCode, 200) s.noError(json.NewDecoder(resp.Body).Decode(dst)) } diff --git a/app/victoria-metrics/self_scraper.go b/app/victoria-metrics/self_scraper.go index fb544a0090..e8540eddd8 100644 --- a/app/victoria-metrics/self_scraper.go +++ b/app/victoria-metrics/self_scraper.go @@ -85,7 +85,9 @@ func selfScraper(scrapeInterval time.Duration) { mr.Timestamp = currentTimestamp mr.Value = r.Value } - vmstorage.AddRows(mrs) + if err := vmstorage.AddRows(mrs); err != nil { + logger.Errorf("cannot store self-scraped metrics: %s", err) + } } } diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 51cc9e71d4..e7c25dde90 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -142,6 +142,10 @@ While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`. +[VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) supports writing data to multiple tenants +specified via special labels - see [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels). +This allows specifying tenant ids via [relabeling](#relabeling) and writing multitenant data to a single `-remoteWrite.url=http:///insert/multitenant/api/v1/write`. + [Multitenancy](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) support is enabled when `-remoteWrite.multitenantURL` command-line flag is set. In this case `vmagent` accepts multitenant data at `http://vmagent:8429/insert//...` in the same way as cluster version of VictoriaMetrics does according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) and routes it to `<-remoteWrite.multitenantURL>/insert//prometheus/api/v1/write`. If multiple `-remoteWrite.multitenantURL` command-line options are set, then `vmagent` replicates the collected data across all the configured urls. This allows using a single `vmagent` instance in front of VictoriaMetrics clusters for processing the data from all the tenants. If `-remoteWrite.multitenantURL` command-line flag is set and `vmagent` is configured to scrape Prometheus-compatible targets (e.g. if `-promscrape.config` command-line flag is set) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 13724ae9b4..b020eada70 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -1158,7 +1158,7 @@ is the following: # password and password_file are mutually exclusive. basic_auth: [ username: ] - [ password: ] + [ password: ] [ password_file: ] # Optional `Authorization` header configuration. @@ -1177,10 +1177,41 @@ authorization: tls_config: [ ] +# Configures Bearer authentication token via string +bearer_token: +# or by passing path to the file with token. +bearer_token_file: + +# Configures OAuth 2.0 authentication +# see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2 +oauth2: + [ ] + +# Optional list of HTTP headers in form `header-name: value` +# applied for all requests to notifiers +# For example: +# headers: +# - "CustomHeader: foo" +# - "CustomHeader2: bar" +headers: + [ , ...] + # List of labeled statically configured Notifiers. +# +# Each list of targets may be additionally instructed with +# authorization params. Target's authorization params will +# inherit params from global authorization params if there +# are no conflicts. static_configs: - targets: - [ - '' ] + [ - targets: ] + [ - '' ] + [ oauth2 ] + [ basic_auth ] + [ authorization ] + [ tls_config ] + [ bearer_token ] + [ bearer_token_file ] + [ headers ] # List of Consul service discovery configurations. # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config diff --git a/app/vmalert/notifier/config.go b/app/vmalert/notifier/config.go index 1fb41ef173..f7b7e02ba1 100644 --- a/app/vmalert/notifier/config.go +++ b/app/vmalert/notifier/config.go @@ -68,6 +68,8 @@ type Config struct { // [ - '' ] type StaticConfig struct { Targets []string `yaml:"targets"` + // HTTPClientConfig contains HTTP configuration for the Targets + HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"` } // UnmarshalYAML implements the yaml.Unmarshaler interface. diff --git a/app/vmalert/notifier/config_watcher.go b/app/vmalert/notifier/config_watcher.go index 297eb06ba6..f70fed7c57 100644 --- a/app/vmalert/notifier/config_watcher.go +++ b/app/vmalert/notifier/config_watcher.go @@ -6,6 +6,7 @@ import ( "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns" ) @@ -161,12 +162,13 @@ func (cw *configWatcher) start() error { if len(cw.cfg.StaticConfigs) > 0 { var targets []Target for _, cfg := range cw.cfg.StaticConfigs { + httpCfg := mergeHTTPClientConfigs(cw.cfg.HTTPClientConfig, cfg.HTTPClientConfig) for _, target := range cfg.Targets { address, labels, err := parseLabels(target, nil, cw.cfg) if err != nil { return fmt.Errorf("failed to parse labels for target %q: %s", target, err) } - notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration()) + notifier, err := NewAlertManager(address, cw.genFn, httpCfg, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration()) if err != nil { return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err) } @@ -252,3 +254,30 @@ func (cw *configWatcher) setTargets(key TargetType, targets []Target) { cw.targets[key] = targets cw.targetsMu.Unlock() } + +// mergeHTTPClientConfigs merges fields between child and parent params +// by populating child from parent params if they're missing. +func mergeHTTPClientConfigs(parent, child promauth.HTTPClientConfig) promauth.HTTPClientConfig { + if child.Authorization == nil { + child.Authorization = parent.Authorization + } + if child.BasicAuth == nil { + child.BasicAuth = parent.BasicAuth + } + if child.BearerToken == nil { + child.BearerToken = parent.BearerToken + } + if child.BearerTokenFile == "" { + child.BearerTokenFile = parent.BearerTokenFile + } + if child.OAuth2 == nil { + child.OAuth2 = parent.OAuth2 + } + if child.TLSConfig == nil { + child.TLSConfig = parent.TLSConfig + } + if child.Headers == nil { + child.Headers = parent.Headers + } + return child +} diff --git a/app/vmalert/notifier/config_watcher_test.go b/app/vmalert/notifier/config_watcher_test.go index 8b70df8426..61550a6266 100644 --- a/app/vmalert/notifier/config_watcher_test.go +++ b/app/vmalert/notifier/config_watcher_test.go @@ -8,6 +8,8 @@ import ( "os" "sync" "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" ) func TestConfigWatcherReload(t *testing.T) { @@ -298,3 +300,20 @@ func newFakeConsulServer() *httptest.Server { return httptest.NewServer(mux) } + +func TestMergeHTTPClientConfigs(t *testing.T) { + cfg1 := promauth.HTTPClientConfig{Headers: []string{"Header:Foo"}} + cfg2 := promauth.HTTPClientConfig{BasicAuth: &promauth.BasicAuthConfig{ + Username: "foo", + Password: promauth.NewSecret("bar"), + }} + + result := mergeHTTPClientConfigs(cfg1, cfg2) + + if result.Headers == nil { + t.Fatalf("expected Headers to be inherited") + } + if result.BasicAuth == nil { + t.Fatalf("expected BasicAuth tp be present") + } +} diff --git a/app/vmalert/notifier/testdata/static.good.yaml b/app/vmalert/notifier/testdata/static.good.yaml index 99cb98eb21..1d95d252d1 100644 --- a/app/vmalert/notifier/testdata/static.good.yaml +++ b/app/vmalert/notifier/testdata/static.good.yaml @@ -1,7 +1,21 @@ +headers: + - 'CustomHeader: foo' + static_configs: - targets: - localhost:9093 - localhost:9095 + basic_auth: + username: foo + password: bar + + - targets: + - localhost:9096 + - localhost:9097 + basic_auth: + username: foo + password: baz + alert_relabel_configs: - target_label: "foo" - replacement: "aaa" \ No newline at end of file + replacement: "aaa" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f9080cee11..d26dc52ed9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -19,6 +19,12 @@ The following tip changes can be tested by building VictoriaMetrics components f **Update note 2:** [vmalert](https://docs.victoriametrics.com/vmalert.html) changes default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval. +**Update note 3:** `vm_account_id` and `vm_project_id` labels must be passed to tcp-based `Graphite`, `InfluxDB` and `OpenTSDB` endpoints +at [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) instead of undocumented +`VictoriaMetrics_AccountID` and `VictoriaMetrics_ProjectID` labels when writing samples to the needed tenant. +See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels) for details. + +* FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): support specifying tenant ids via `vm_account_id` and `vm_project_id` labels. See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2970). * FEATURE: improve [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) performance by up to 3x if non-trivial `regex` values are used. * FEATURE: sanitize metric names for data ingested via [DataDog protocol](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent) according to [DataDog metric naming](https://docs.datadoghq.com/metrics/custom_metrics/#naming-custom-metrics). The behaviour can be disabled by passing `-datadog.sanitizeMetricName=false` command-line flag. Thanks to @PerGon for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3105). * FEATURE: add `-usePromCompatibleNaming` command-line flag to [vmagent](https://docs.victoriametrics.com/vmagent.html), to single-node VictoriaMetrics and to `vminsert` component of VictoriaMetrics cluster. This flag can be used for normalizing the ingested metric names and label names to [Prometheus-compatible form](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels). If this flag is set, then all the chars unsupported by Prometheus are replaced with `_` chars in metric names and labels of the ingested samples. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113). @@ -32,6 +38,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `debug` mode to the alerting rule settings for printing additional information into logs during evaluation. See `debug` param in [alerting rule config](https://docs.victoriametrics.com/vmalert.html#alerting-rules). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add experimental feature for displaying last 10 states of the rule (recording or alerting) evaluation. The state is available on the Rule page, which can be opened by clicking on `Details` link next to Rule's name on the `/groups` page. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow using extra labels in annotiations. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3013). +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow configuring authorization params per list of targets in vmalert's notifier config for `static_configs`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2690). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): minimize the time needed for reading large responses from scrape targets in [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). This should reduce scrape durations for such targets as [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) running in a big Kubernetes cluster. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [sort_by_label_numeric](https://docs.victoriametrics.com/MetricsQL.html#sort_by_label_numeric) and [sort_by_label_numeric_desc](https://docs.victoriametrics.com/MetricsQL.html#sort_by_label_numeric_desc) functions for [numeric sort](https://www.gnu.org/software/coreutils/manual/html_node/Version-sort-is-not-the-same-as-numeric-sort.html) of input time series by the specified labels. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2938). * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html) and [vmrestore](https://docs.victoriametrics.com/vmrestore.html): retry GCS operations for up to 3 minutes on temporary failures. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3147). diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index b73d0be776..ffe7aeabbd 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -43,7 +43,9 @@ It increases cluster availability, and simplifies cluster maintenance as well as VictoriaMetrics cluster supports multiple isolated tenants (aka namespaces). Tenants are identified by `accountID` or `accountID:projectID`, which are put inside request urls. -See [these docs](#url-format) for details. Some facts about tenants in VictoriaMetrics: +See [these docs](#url-format) for details. + +Some facts about tenants in VictoriaMetrics: - Each `accountID` and `projectID` is identified by an arbitrary 32-bit integer in the range `[0 .. 2^32)`. If `projectID` is missing, then it is automatically assigned to `0`. It is expected that other information about tenants @@ -59,6 +61,30 @@ when different tenants have different amounts of data and different query load. - VictoriaMetrics doesn't support querying multiple tenants in a single request. +See also [multitenancy via labels](#multitenancy-via-labels). + + +## Multitenancy via labels + +`vminsert` can accept data from multiple [tenants](#multitenancy) via a special `multitenant` endpoints `http://vminsert:8480/insert/multitenant/`, +where `` can be replaced with any supported suffix for data ingestion from [this list](#url-format). +In this case the account id and project id are obtained from optional `vm_account_id` and `vm_project_id` labels of the incoming samples. +If `vm_account_id` or `vm_project_id` labels are missing or invalid, then the corresponding `accountID` or `projectID` is set to 0. +These labels are automatically removed from samples before forwarding them to `vmstorage`. +For example, if the following samples are written into `http://vminsert:8480/insert/multitenant/prometheus/api/v1/write`: + +``` +http_requests_total{path="/foo",vm_account_id="42"} 12 +http_requests_total{path="/bar",vm_account_id="7",vm_project_id="9"} 34 +``` + +Then the `http_requests_total{path="/foo"} 12` would be stored in the tenant `accountID=42, projectID=0`, +while the `http_requests_total{path="/bar"} 34` would be stored in the tenant `accountID=7, projectID=9`. + +The `vm_account_id` and `vm_project_id` labels are extracted after applying the [relabeling](https://docs.victoriametrics.com/relabeling.html) +set via `-relabelConfig` command-line flag, so these labels can be set at this stage. + + ## Binaries Compiled binaries for the cluster version are available in the `assets` section of the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases). @@ -218,7 +244,9 @@ See [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html - URLs for data ingestion: `http://:8480/insert//`, where: - `` is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). It is possible to set it as `accountID:projectID`, - where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. + where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. See [multitenancy docs](#multitenancy) for more details. + The `` can be set to `multitenant` string, e.g. `http://:8480/insert/multitenant/`. Such urls accept data from multiple tenants + specified via `vm_account_id` and `vm_project_id` labels. See [multitenancy via labels](#multitenancy-via-labels) for more details. - `` may have the following values: - `prometheus` and `prometheus/api/v1/write` - for inserting data with [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). - `datadog/api/v1/series` - for inserting data with [DataDog submit metrics API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent) for details. diff --git a/docs/url-examples.md b/docs/url-examples.md index 35e6107651..9f0ac3e571 100644 --- a/docs/url-examples.md +++ b/docs/url-examples.md @@ -604,7 +604,7 @@ Cluster version of VictoriaMetrics:
```console -echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2 VictoriaMetrics_AccountID=0" | nc -N http:// 4242 +echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2" | nc -N http:// 4242 ```
@@ -651,14 +651,12 @@ Cluster version of VictoriaMetrics:
```console -echo "foo.bar.baz;tag1=value1;tag2=value2;VictoriaMetrics_AccountID=42 123 `date +%s`" | nc -N http:// 2003 +echo "foo.bar.baz;tag1=value1;tag2=value2 123 `date +%s`" | nc -N http:// 2003 ```
Additional information: -`VictoriaMetrics_AccountID=42` - [tenant ID](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) in cluster version of VictoriaMetrics - * [How to send Graphite data to VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) * [Multitenancy in cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) diff --git a/docs/vmagent.md b/docs/vmagent.md index 7ef86bad42..315d704186 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -146,6 +146,10 @@ While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`. +[VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) supports writing data to multiple tenants +specified via special labels - see [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels). +This allows specifying tenant ids via [relabeling](#relabeling) and writing multitenant data to a single `-remoteWrite.url=http:///insert/multitenant/api/v1/write`. + [Multitenancy](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) support is enabled when `-remoteWrite.multitenantURL` command-line flag is set. In this case `vmagent` accepts multitenant data at `http://vmagent:8429/insert//...` in the same way as cluster version of VictoriaMetrics does according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) and routes it to `<-remoteWrite.multitenantURL>/insert//prometheus/api/v1/write`. If multiple `-remoteWrite.multitenantURL` command-line options are set, then `vmagent` replicates the collected data across all the configured urls. This allows using a single `vmagent` instance in front of VictoriaMetrics clusters for processing the data from all the tenants. If `-remoteWrite.multitenantURL` command-line flag is set and `vmagent` is configured to scrape Prometheus-compatible targets (e.g. if `-promscrape.config` command-line flag is set) diff --git a/docs/vmalert.md b/docs/vmalert.md index 4ee7fd44db..528728524b 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -1162,7 +1162,7 @@ is the following: # password and password_file are mutually exclusive. basic_auth: [ username: ] - [ password: ] + [ password: ] [ password_file: ] # Optional `Authorization` header configuration. @@ -1181,10 +1181,41 @@ authorization: tls_config: [ ] +# Configures Bearer authentication token via string +bearer_token: +# or by passing path to the file with token. +bearer_token_file: + +# Configures OAuth 2.0 authentication +# see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2 +oauth2: + [ ] + +# Optional list of HTTP headers in form `header-name: value` +# applied for all requests to notifiers +# For example: +# headers: +# - "CustomHeader: foo" +# - "CustomHeader2: bar" +headers: + [ , ...] + # List of labeled statically configured Notifiers. +# +# Each list of targets may be additionally instructed with +# authorization params. Target's authorization params will +# inherit params from global authorization params if there +# are no conflicts. static_configs: - targets: - [ - '' ] + [ - targets: ] + [ - '' ] + [ oauth2 ] + [ basic_auth ] + [ authorization ] + [ tls_config ] + [ bearer_token ] + [ bearer_token_file ] + [ headers ] # List of Consul service discovery configurations. # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config diff --git a/lib/auth/auth.go b/lib/auth/auth.go index 6d27943e3b..38e2354482 100644 --- a/lib/auth/auth.go +++ b/lib/auth/auth.go @@ -14,30 +14,54 @@ type Token struct { // String returns string representation of t. func (t *Token) String() string { + if t == nil { + return "multitenant" + } if t.ProjectID == 0 { return fmt.Sprintf("%d", t.AccountID) } return fmt.Sprintf("%d:%d", t.AccountID, t.ProjectID) } -// NewToken returns new Token for the given authToken +// NewToken returns new Token for the given authToken. +// +// If authToken == "multitenant", then nil Token is returned. func NewToken(authToken string) (*Token, error) { + if authToken == "multitenant" { + return nil, nil + } + var t Token + if err := t.Init(authToken); err != nil { + return nil, err + } + return &t, nil +} + +// Init initializes t from authToken. +func (t *Token) Init(authToken string) error { tmp := strings.Split(authToken, ":") if len(tmp) > 2 { - return nil, fmt.Errorf("unexpected number of items in authToken %q; got %d; want 1 or 2", authToken, len(tmp)) + return fmt.Errorf("unexpected number of items in authToken %q; got %d; want 1 or 2", authToken, len(tmp)) } - var at Token - accountID, err := strconv.ParseUint(tmp[0], 10, 32) + n, err := strconv.ParseUint(tmp[0], 10, 32) if err != nil { - return nil, fmt.Errorf("cannot parse accountID from %q: %w", tmp[0], err) + return fmt.Errorf("cannot parse accountID from %q: %w", tmp[0], err) } - at.AccountID = uint32(accountID) + accountID := uint32(n) + projectID := uint32(0) if len(tmp) > 1 { - projectID, err := strconv.ParseUint(tmp[1], 10, 32) + n, err := strconv.ParseUint(tmp[1], 10, 32) if err != nil { - return nil, fmt.Errorf("cannot parse projectID from %q: %w", tmp[1], err) + return fmt.Errorf("cannot parse projectID from %q: %w", tmp[1], err) } - at.ProjectID = uint32(projectID) + projectID = uint32(n) } - return &at, nil + t.Set(accountID, projectID) + return nil +} + +// Set sets accountID and projectID for the t. +func (t *Token) Set(accountID, projectID uint32) { + t.AccountID = accountID + t.ProjectID = projectID } diff --git a/lib/auth/auth_test.go b/lib/auth/auth_test.go index aa2b4b4f92..eddf1c6fe0 100644 --- a/lib/auth/auth_test.go +++ b/lib/auth/auth_test.go @@ -26,6 +26,8 @@ func TestNewTokenSuccess(t *testing.T) { f("1:4294967295", "1:4294967295") // max uint32 accountID and projectID f("4294967295:4294967295", "4294967295:4294967295") + // multitenant + f("multitenant", "multitenant") } func TestNewTokenFailure(t *testing.T) { diff --git a/lib/httpserver/path.go b/lib/httpserver/path.go index 833c0a6b41..3743341e6e 100644 --- a/lib/httpserver/path.go +++ b/lib/httpserver/path.go @@ -22,6 +22,8 @@ func ParsePath(path string) (*Path, error) { // // - prefix must contain `select`, `insert` or `delete`. // - authToken contains `accountID[:projectID]`, where projectID is optional. + // authToken may also contain `multitenant` string. In this case the accountID and projectID + // are obtained from vm_account_id and vm_project_id labels of the ingested samples. // - suffix contains arbitrary suffix. // // prefix must be used for the routing to the appropriate service @@ -29,14 +31,14 @@ func ParsePath(path string) (*Path, error) { s := skipPrefixSlashes(path) n := strings.IndexByte(s, '/') if n < 0 { - return nil, fmt.Errorf("cannot find {prefix}") + return nil, fmt.Errorf("cannot find {prefix} in %q; expecting /{prefix}/{authToken}/{suffix} format", path) } prefix := s[:n] s = skipPrefixSlashes(s[n+1:]) n = strings.IndexByte(s, '/') if n < 0 { - return nil, fmt.Errorf("cannot find {authToken}") + return nil, fmt.Errorf("cannot find {authToken} in %q; expecting /{prefix}/{authToken}/{suffix} format", path) } authToken := s[:n] diff --git a/lib/tenantmetrics/counter_map.go b/lib/tenantmetrics/counter_map.go index 710986b80f..3ab106679c 100644 --- a/lib/tenantmetrics/counter_map.go +++ b/lib/tenantmetrics/counter_map.go @@ -39,6 +39,13 @@ func (cm *CounterMap) Get(at *auth.Token) *metrics.Counter { return cm.GetByTenant(key) } +// MultiAdd adds multiple values grouped by auth.Token +func (cm *CounterMap) MultiAdd(perTenantValues map[auth.Token]int) { + for token, value := range perTenantValues { + cm.Get(&token).Add(value) + } +} + // GetByTenant returns counter for the given key. func (cm *CounterMap) GetByTenant(key TenantID) *metrics.Counter { m := cm.m.Load().(map[TenantID]*metrics.Counter)