github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-03-01 15:33:35 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

Browse source
This commit is contained in:
Aliaksandr Valialkin 2022-05-05 00:02:14 +03:00
commit 9eb61e67af
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
55 changed files with 2070 additions and 1068 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -1899,6 +1899,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
The maximum number of CPU cores to use for small merges. Default value is used if set to 0
-snapshotAuthKey string
authKey, which must be passed in query string to /snapshot* pages
-snapshotsMaxAge duration
Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted
-sortLabels
Whether to sort labels for incoming samples before writing them to storage. This may be needed for reducing memory usage at storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}. Enabled sorting for labels can slow down ingestion performance a bit
-storage.cacheSizeIndexDBDataBlocks size
@ -1930,4 +1932,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-version
Show VictoriaMetrics version
-vmalert.proxyURL string
Optional URL for proxying alerting API requests from Grafana. For example, if -vmalert.proxyURL is set to http://vmalert:8880 , then requests to /api/v1/rules are proxied to http://vmalert:8880/api/v1/rules
```

15
app/vmagent/README.md
View file

@ -914,6 +914,21 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
-promscrape.suppressScrapeErrors
Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed
-remoteWrite.aws.accessKey array
Optional AWS AccessKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports an array of values separated by comma or specified via multiple flags.
-remoteWrite.aws.region array
Optional AWS region to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports an array of values separated by comma or specified via multiple flags.
-remoteWrite.aws.roleARN array
Optional AWS roleARN to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports an array of values separated by comma or specified via multiple flags.
-remoteWrite.aws.secretKey array
Optional AWS SecretKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports an array of values separated by comma or specified via multiple flags.
-remoteWrite.aws.useSigv4 array
Enables SigV4 request signing for -remoteWrite.url. It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports array of values separated by comma or specified via multiple flags.
-remoteWrite.basicAuth.password array
Optional basic auth password to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports an array of values separated by comma or specified via multiple flags.

51
app/vmagent/remotewrite/client.go
View file

@ -10,6 +10,7 @@ import (
"sync"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
@ -59,6 +60,18 @@ var (
"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
oauth2Scopes = flagutil.NewArray("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for -remoteWrite.url. Scopes must be delimited by ';'. "+
"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for -remoteWrite.url. "+
"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled. "+
"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
awsRegion = flagutil.NewArray("remoteWrite.aws.region", "Optional AWS region to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
awsRoleARN = flagutil.NewArray("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
awsAccessKey = flagutil.NewArray("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
awsSecretKey = flagutil.NewArray("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
"If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url")
)
type client struct {
@ -69,6 +82,7 @@ type client struct {
sendBlock func(block []byte) bool
authCfg *promauth.Config
awsCfg *awsapi.Config
rl rateLimiter
@ -78,6 +92,7 @@ type client struct {
requestsOKCount *metrics.Counter
errorsCount *metrics.Counter
packetsDropped *metrics.Counter
rateLimit *metrics.Gauge
retriesCount *metrics.Counter
sendDuration *metrics.FloatCounter
@ -88,9 +103,13 @@ type client struct {
func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
authCfg, err := getAuthConfig(argIdx)
if err != nil {
logger.Panicf("FATAL: cannot initialize auth config: %s", err)
logger.Panicf("FATAL: cannot initialize auth config for remoteWrite.url=%q: %s", remoteWriteURL, err)
}
tlsCfg := authCfg.NewTLSConfig()
awsCfg, err := getAWSAPIConfig(argIdx)
if err != nil {
logger.Fatalf("FATAL: cannot initialize AWS Config for remoteWrite.url=%q: %s", remoteWriteURL, err)
}
tr := &http.Transport{
DialContext: statDial,
TLSClientConfig: tlsCfg,
@ -115,6 +134,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
sanitizedURL: sanitizedURL,
remoteWriteURL: remoteWriteURL,
authCfg: authCfg,
awsCfg: awsCfg,
fq: fq,
hc: &http.Client{
Transport: tr,
@ -135,6 +155,9 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
return float64(rateLimit.GetOptionalArgOrDefault(argIdx, 0))
})
c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
@ -201,6 +224,21 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
return authCfg, nil
}
func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
if !awsUseSigv4.GetOptionalArg(argIdx) {
return nil, nil
}
region := awsRegion.GetOptionalArg(argIdx)
roleARN := awsRoleARN.GetOptionalArg(argIdx)
accessKey := awsAccessKey.GetOptionalArg(argIdx)
secretKey := awsSecretKey.GetOptionalArg(argIdx)
cfg, err := awsapi.NewConfig(region, roleARN, accessKey, secretKey, "")
if err != nil {
return nil, err
}
return cfg, nil
}
func (c *client) runWorker() {
var ok bool
var block []byte
@ -250,6 +288,10 @@ func (c *client) sendBlockHTTP(block []byte) bool {
retriesCount := 0
c.bytesSent.Add(len(block))
c.blocksSent.Inc()
sigv4Hash := ""
if c.awsCfg != nil {
sigv4Hash = awsapi.HashHex(block)
}
again:
req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
@ -264,7 +306,12 @@ again:
if ah := c.authCfg.GetAuthHeader(); ah != "" {
req.Header.Set("Authorization", ah)
}
if c.awsCfg != nil {
if err := c.awsCfg.SignRequest(req, "aps", sigv4Hash); err != nil {
// there is no need in retry, request will be rejected by client.Do and retried by code below
logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
}
}
startTime := time.Now()
resp, err := c.hc.Do(req)
c.requestDuration.UpdateDuration(startTime)

19
app/vmalert/tpl/header.qtpl
View file

@ -27,8 +27,27 @@
.group-heading:hover {
background-color: #f8f9fa!important;
}
.table {
table-layout: fixed;
}
.table .error-cell{
word-break: break-word;
font-size: 14px;
}
pre {
overflow: scroll;
min-height: 30px;
max-width: 100%;
}
pre::-webkit-scrollbar {
-webkit-appearance: none;
width: 0px;
height: 5px;
}
pre::-webkit-scrollbar-thumb {
border-radius: 5px;
background-color: rgba(0,0,0,.5);
-webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
}
</style>
</head>

49
app/vmalert/tpl/header.qtpl.go
View file

@ -59,44 +59,63 @@ func StreamHeader(qw422016 *qt422016.Writer, title string, pages []NavItem) {
.group-heading:hover {
background-color: #f8f9fa!important;
}
.table {
table-layout: fixed;
}
.table .error-cell{
word-break: break-word;
font-size: 14px;
}
pre {
overflow: scroll;
min-height: 30px;
max-width: 100%;
}
pre::-webkit-scrollbar {
-webkit-appearance: none;
width: 0px;
height: 5px;
}
pre::-webkit-scrollbar-thumb {
border-radius: 5px;
background-color: rgba(0,0,0,.5);
-webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
}
</style>
</head>
<body>
`)
//line app/vmalert/tpl/header.qtpl:36
//line app/vmalert/tpl/header.qtpl:55
StreamPrintNavItems(qw422016, title, pages)
//line app/vmalert/tpl/header.qtpl:36
//line app/vmalert/tpl/header.qtpl:55
qw422016.N().S(`
<main class="px-2">
`)
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
}
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
func WriteHeader(qq422016 qtio422016.Writer, title string, pages []NavItem) {
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
StreamHeader(qw422016, title, pages)
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
qt422016.ReleaseWriter(qw422016)
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
}
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
func Header(title string, pages []NavItem) string {
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
WriteHeader(qb422016, title, pages)
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
qs422016 := string(qb422016.B)
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
return qs422016
//line app/vmalert/tpl/header.qtpl:38
//line app/vmalert/tpl/header.qtpl:57
}

49
app/vmalert/web.qtpl
View file

@ -61,31 +61,44 @@
<table class="table table-striped table-hover table-sm">
<thead>
<tr>
<th scope="col">Rule</th>
<th scope="col" title="Shows if rule's execution ended with error">Error</th>
<th scope="col" title="How many samples were produced by the rule">Samples</th>
<th scope="col" title="How many seconds ago rule was executed">Updated</th>
<th scope="col" style="width: 60%">Rule</th>
<th scope="col" style="width: 20%" class="text-center" title="How many samples were produced by the rule">Samples</th>
<th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
</tr>
</thead>
<tbody>
{% for _, r := range g.Rules %}
<tr{% if r.LastError != "" %} class="alert-danger"{% endif %}>
<td>
{% if r.Type == "alerting" %}
<b>alert:</b> (for: {%v r.Duration %})
{% else %}
<b>record:</b> {%s r.Name %}
{% endif %}
<br>
<code><pre class="text-wrap">{%s r.Query %}</pre></code><br>
{% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
{% for k, v := range r.Labels %}
<span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
{% endfor %}
<div class="row">
<div class="col-12 mb-2">
{% if r.Type == "alerting" %}
<b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds)
{% else %}
<b>record:</b> {%s r.Name %}
{% endif %}
</div>
<div class="col-12">
<code><pre>{%s r.Query %}</pre></code>
</div>
<div class="col-12 mb-2">
{% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
{% for k, v := range r.Labels %}
<span class="ms-1 badge bg-primary">{%s k %}={%s v %}</span>
{% endfor %}
</div>
{% if r.LastError != "" %}
<div class="col-12">
<b>Error:</b>
<div class="error-cell">
{%s r.LastError %}
</div>
</div>
{% endif %}
</div>
</td>
<td><div class="error-cell">{%s r.LastError %}</div></td>
<td>{%d r.LastSamples %}</td>
<td>{%f.3 time.Since(r.LastEvaluation).Seconds() %}s ago</td>
<td class="text-center">{%d r.LastSamples %}</td>
<td class="text-center">{%f.3 time.Since(r.LastEvaluation).Seconds() %}s ago</td>
</tr>
{% endfor %}
</tbody>

697
app/vmalert/web.qtpl.go
View file

File diff suppressed because it is too large Load diff

28
app/vmbackup/README.md
View file

@ -29,12 +29,12 @@ creation of hourly, daily, weekly and monthly backups.
Regular backup can be performed with the following command:
```bash
vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/new/backup>
vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/new/backup>
```
* `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
There is no need to stop VictoriaMetrics for creating backups since they are performed from immutable [instant snapshots](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots).
* `<local-snapshot>` is the snapshot to back up. See [how to create instant snapshots](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots). `vmbackup` can create the snapshot on itself if `-snapshot.createURL` command-line flag is set to an url for creating snapshots. In this case `-snapshotName` flag isn't needed.
* `http://victoriametrics:8428/snapshot/create` is the url for creating snapshots according to [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots). `vmbackup` creates a snapshot by querying the provided `-snapshot.createURL`, then performs the backup and then automatically removes the created snapshot.
* `<bucket>` is an already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
* `<path/to/new/backup>` is the destination path where new backup will be placed.
@ -44,7 +44,7 @@ If the destination GCS bucket already contains the previous backup at `-origin`
with the following command:
```bash
vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/new/backup> -origin=gs://<bucket>/<path/to/existing/backup>
./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/new/backup> -origin=gs://<bucket>/<path/to/existing/backup>
```
It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
@ -55,7 +55,7 @@ Incremental backups are performed if `-dst` points to an already existing backup
It saves time and network bandwidth costs when working with big backups:
```bash
vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/existing/backup>
./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/existing/backup>
```
### Smart backups
@ -65,7 +65,7 @@ Smart backups mean storing full daily backups into `YYYYMMDD` folders and creati
* Run the following command every hour:
```bash
vmbackup -snapshotName=<latest-snapshot> -dst=gs://<bucket>/latest
./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/latest
```
Where `<latest-snapshot>` is the latest [snapshot](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots).
@ -74,7 +74,7 @@ The command will upload only changed data to `gs://<bucket>/latest`.
* Run the following command once a day:
```bash
vmbackup -snapshotName=<daily-snapshot> -dst=gs://<bucket>/<YYYYMMDD> -origin=gs://<bucket>/latest
vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<YYYYMMDD> -origin=gs://<bucket>/latest
```
Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
@ -82,7 +82,7 @@ Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
This apporach saves network bandwidth costs on hourly backups (since they are incremental) and allows recovering data from either the last hour (`latest` backup)
or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when creating daily backup.
Do not forget to remove old snapshots and backups when they are no longer needed in order to save storage costs.
Do not forget to remove old backups when they are no longer needed in order to save storage costs.
See also [vmbackupmanager tool](https://docs.victoriametrics.com/vmbackupmanager.html) for automating smart backups.
@ -90,15 +90,17 @@ See also [vmbackupmanager tool](https://docs.victoriametrics.com/vmbackupmanager
The backup algorithm is the following:
1. Collect information about files in the `-snapshotName`, in the `-dst` and in the `-origin`.
2. Determine which files in `-dst` are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
3. Determine which files in `-snapshotName` are missing in `-dst`. These are usually small new files and bigger merged files.
4. Determine which files from step 3 exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
1. Create a snapshot by querying the provided `-snapshot.createURL`
2. Collect information about files in the created snapshot, in the `-dst` and in the `-origin`.
3. Determine which files in `-dst` are missing in the created snapshot, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
4. Determine which files in the created snapshot are missing in `-dst`. These are usually small new files and bigger merged files.
5. Determine which files from step 3 exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
These are usually the biggest and the oldest files, which are shared between backups.
5. Upload the remaining files from step 3 from `-snapshotName` to `-dst`.
6. Upload the remaining files from step 3 from the created snapshot to `-dst`.
7. Delete the created snapshot.
The algorithm splits source files into 1 GiB chunks in the backup. Each chunk is stored as a separate file in the backup.
Such splitting minimizes the amounts of data to re-transfer after temporary errors.
Such splitting balances between the number of files in the backup and the amounts of data that needs to be re-transfered after temporary errors.
`vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:

10
app/vmbackup/main.go
View file

@ -7,7 +7,6 @@ import (
"strings"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmbackup/snapshot"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
@ -17,6 +16,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
)
var (
@ -71,6 +71,11 @@ func main() {
logger.Fatalf("cannot delete snapshot: %s", err)
}
}()
} else if len(*snapshotName) == 0 {
logger.Fatalf("`-snapshotName` or `-snapshot.createURL` must be provided")
}
if err := snapshot.Validate(*snapshotName); err != nil {
logger.Fatalf("invalid -snapshotName=%q: %s", *snapshotName, err)
}
go httpserver.Serve(*httpListenAddr, nil)
@ -119,9 +124,6 @@ See the docs at https://docs.victoriametrics.com/vmbackup.html .
}
func newSrcFS() (*fslocal.FS, error) {
if len(*snapshotName) == 0 {
return nil, fmt.Errorf("`-snapshotName` or `-snapshot.createURL` must be provided")
}
snapshotPath := *storageDataPath + "/snapshots/" + *snapshotName
// Verify the snapshot exists.

9
app/vmctl/influx.go
View file

@ -82,12 +82,16 @@ func (ip *influxProcessor) run(silent, verbose bool) error {
close(seriesCh)
wg.Wait()
ip.im.Close()
close(errCh)
// drain import errors channel
for vmErr := range ip.im.Errors() {
if vmErr.Err != nil {
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
}
}
for err := range errCh {
return fmt.Errorf("import process failed: %s", err)
}
barpool.Stop()
log.Println("Import finished!")
log.Print(ip.im.Stats())
@ -142,11 +146,14 @@ func (ip *influxProcessor) do(s *influx.Series) error {
if len(time) < 1 {
continue
}
ip.im.Input() <- &vm.TimeSeries{
ts := vm.TimeSeries{
Name: name,
LabelPairs: labels,
Timestamps: time,
Values: values,
}
if err := ip.im.Input(&ts); err != nil {
return err
}
}
}

12
app/vmctl/main.go
View file

@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"log"
"os"
@ -26,6 +27,7 @@ func main() {
importer *vm.Importer
)
ctx, cancelCtx := context.WithCancel(context.Background())
start := time.Now()
app := &cli.App{
Name: "vmctl",
@ -98,8 +100,11 @@ func main() {
return fmt.Errorf("failed to create VM importer: %s", err)
}
processor := newInfluxProcessor(influxClient, importer,
c.Int(influxConcurrency), c.String(influxMeasurementFieldSeparator))
processor := newInfluxProcessor(
influxClient,
importer,
c.Int(influxConcurrency),
c.String(influxMeasurementFieldSeparator))
return processor.run(c.Bool(globalSilent), c.Bool(globalVerbose))
},
},
@ -167,7 +172,7 @@ func main() {
extraLabels: c.StringSlice(vmExtraLabel),
},
}
return p.run()
return p.run(ctx)
},
},
{
@ -214,6 +219,7 @@ func main() {
if importer != nil {
importer.Close()
}
cancelCtx()
}()
err = app.Run(os.Args)

6
app/vmctl/opentsdb.go
View file

@ -120,6 +120,7 @@ func (op *otsdbProcessor) run(silent, verbose bool) error {
}
}
}
// Drain channels per metric
close(seriesCh)
wg.Wait()
@ -156,11 +157,14 @@ func (op *otsdbProcessor) do(s queryObj) error {
for k, v := range data.Tags {
labels = append(labels, vm.LabelPair{Name: k, Value: v})
}
op.im.Input() <- &vm.TimeSeries{
ts := vm.TimeSeries{
Name: data.Metric,
LabelPairs: labels,
Timestamps: data.Timestamps,
Values: data.Values,
}
if err := op.im.Input(&ts); err != nil {
return err
}
return nil
}

10
app/vmctl/prometheus.go
View file

@ -62,7 +62,6 @@ func (pp *prometheusProcessor) run(silent, verbose bool) error {
}
}()
}
// any error breaks the import
for _, br := range blocks {
select {
@ -80,12 +79,16 @@ func (pp *prometheusProcessor) run(silent, verbose bool) error {
wg.Wait()
// wait for all buffers to flush
pp.im.Close()
close(errCh)
// drain import errors channel
for vmErr := range pp.im.Errors() {
if vmErr.Err != nil {
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
}
}
for err := range errCh {
return fmt.Errorf("import process failed: %s", err)
}
barpool.Stop()
log.Println("Import finished!")
log.Print(pp.im.Stats())
@ -127,12 +130,15 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
if err := it.Err(); err != nil {
return err
}
pp.im.Input() <- &vm.TimeSeries{
ts := vm.TimeSeries{
Name: name,
LabelPairs: labels,
Timestamps: timestamps,
Values: values,
}
if err := pp.im.Input(&ts); err != nil {
return err
}
}
return ss.Err()
}

164
app/vmctl/prometheus_test.go Normal file
View file

@ -0,0 +1,164 @@
package main
import (
"os"
"testing"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
)
// If you want to run this test:
// 1. provide test snapshot path in const testSnapshot
// 2. define httpAddr const with your victoriametrics address
// 3. run victoria metrics with defined address
// 4. remove t.Skip() from Test_prometheusProcessor_run
// 5. run tests one by one not all at one time
const (
httpAddr = "http://127.0.0.1:8428/"
testSnapshot = "./testdata/20220427T130947Z-70ba49b1093fd0bf"
)
// This test simulates close process if user abort it
func Test_prometheusProcessor_run(t *testing.T) {
t.Skip()
type fields struct {
cfg prometheus.Config
vmCfg vm.Config
cl func(prometheus.Config) *prometheus.Client
im func(vm.Config) *vm.Importer
closer func(importer *vm.Importer)
cc int
}
type args struct {
silent bool
verbose bool
}
tests := []struct {
name string
fields fields
args args
wantErr bool
}{
{
name: "simulate syscall.SIGINT",
fields: fields{
cfg: prometheus.Config{
Snapshot: testSnapshot,
Filter: prometheus.Filter{},
},
cl: func(cfg prometheus.Config) *prometheus.Client {
client, err := prometheus.NewClient(cfg)
if err != nil {
t.Fatalf("error init prometeus client: %s", err)
}
return client
},
im: func(vmCfg vm.Config) *vm.Importer {
importer, err := vm.NewImporter(vmCfg)
if err != nil {
t.Fatalf("error init importer: %s", err)
}
return importer
},
closer: func(importer *vm.Importer) {
// simulate syscall.SIGINT
time.Sleep(time.Second * 5)
if importer != nil {
importer.Close()
}
},
vmCfg: vm.Config{Addr: httpAddr, Concurrency: 1},
cc: 2,
},
args: args{
silent: false,
verbose: false,
},
wantErr: true,
},
{
name: "simulate correct work",
fields: fields{
cfg: prometheus.Config{
Snapshot: testSnapshot,
Filter: prometheus.Filter{},
},
cl: func(cfg prometheus.Config) *prometheus.Client {
client, err := prometheus.NewClient(cfg)
if err != nil {
t.Fatalf("error init prometeus client: %s", err)
}
return client
},
im: func(vmCfg vm.Config) *vm.Importer {
importer, err := vm.NewImporter(vmCfg)
if err != nil {
t.Fatalf("error init importer: %s", err)
}
return importer
},
closer: nil,
vmCfg: vm.Config{Addr: httpAddr, Concurrency: 5},
cc: 2,
},
args: args{
silent: true,
verbose: false,
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := tt.fields.cl(tt.fields.cfg)
importer := tt.fields.im(tt.fields.vmCfg)
pp := &prometheusProcessor{
cl: client,
im: importer,
cc: tt.fields.cc,
}
// we should answer on prompt
if !tt.args.silent {
input := []byte("Y\n")
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
_, err = w.Write(input)
if err != nil {
t.Error(err)
}
err = w.Close()
if err != nil {
t.Error(err)
}
stdin := os.Stdin
// Restore stdin right after the test.
defer func() {
os.Stdin = stdin
_ = r.Close()
_ = w.Close()
}()
os.Stdin = r
}
// simulate close if needed
if tt.fields.closer != nil {
go tt.fields.closer(importer)
}
if err := pp.run(tt.args.silent, tt.args.verbose); (err != nil) != tt.wantErr {
t.Errorf("run() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

12
app/vmctl/vm/vm.go
View file

@ -182,7 +182,17 @@ func (im *Importer) Errors() chan *ImportError { return im.errors }
// Input returns a channel for sending timeseries
// that need to be imported
func (im *Importer) Input() chan<- *TimeSeries { return im.input }
func (im *Importer) Input(ts *TimeSeries) error {
select {
case im.input <- ts:
return nil
case err := <-im.errors:
if err != nil && err.Err != nil {
return err.Err
}
return fmt.Errorf("process aborted")
}
}
// Close sends signal to all goroutines to exit
// and waits until they are finished

16
app/vmctl/vm_native.go
View file

@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"io"
"io/ioutil"
@ -51,25 +52,25 @@ const (
nativeBarTpl = `Total: {{counters . }} {{ cycle . "↖" "↗" "↘" "↙" }} Speed: {{speed . }} {{string . "suffix"}}`
)
func (p *vmNativeProcessor) run() error {
func (p *vmNativeProcessor) run(ctx context.Context) error {
pr, pw := io.Pipe()
fmt.Printf("Initing export pipe from %q with filters: %s\n", p.src.addr, p.filter)
exportReader, err := p.exportPipe()
exportReader, err := p.exportPipe(ctx)
if err != nil {
return fmt.Errorf("failed to init export pipe: %s", err)
}
sync := make(chan struct{})
nativeImportAddr, err := vm.AddExtraLabelsToImportPath(nativeImportAddr, p.dst.extraLabels)
if err != nil {
return err
}
sync := make(chan struct{})
go func() {
defer func() { close(sync) }()
u := fmt.Sprintf("%s/%s", p.dst.addr, nativeImportAddr)
req, err := http.NewRequest("POST", u, pr)
req, err := http.NewRequestWithContext(ctx, "POST", u, pr)
if err != nil {
log.Fatalf("cannot create import request to %q: %s", p.dst.addr, err)
}
@ -95,22 +96,25 @@ func (p *vmNativeProcessor) run() error {
rl := limiter.NewLimiter(p.rateLimit)
w = limiter.NewWriteLimiter(pw, rl)
}
_, err = io.Copy(w, barReader)
if err != nil {
return fmt.Errorf("failed to write into %q: %s", p.dst.addr, err)
}
if err := pw.Close(); err != nil {
return err
}
<-sync
barpool.Stop()
log.Println("Import finished!")
return nil
}
func (p *vmNativeProcessor) exportPipe() (io.ReadCloser, error) {
func (p *vmNativeProcessor) exportPipe(ctx context.Context) (io.ReadCloser, error) {
u := fmt.Sprintf("%s/%s", p.src.addr, nativeExportAddr)
req, err := http.NewRequest("GET", u, nil)
req, err := http.NewRequestWithContext(ctx, "GET", u, nil)
if err != nil {
return nil, fmt.Errorf("cannot create request to %q: %s", p.src.addr, err)
}

95
app/vmctl/vm_native_test.go Normal file
View file

@ -0,0 +1,95 @@
package main
import (
"context"
"testing"
"time"
)
// If you want to run this test:
// 1. run two instances of victoriametrics and define -httpListenAddr for both or just for second instance
// 2. define srcAddr and dstAddr const with your victoriametrics addresses
// 3. define matchFilter const with your importing data
// 4. define timeStartFilter
// 5. run each test one by one
const (
matchFilter = `{job="avalanche"}`
timeStartFilter = "2020-01-01T20:07:00Z"
srcAddr = "http://127.0.0.1:8428"
dstAddr = "http://127.0.0.1:8528"
)
// This test simulates close process if user abort it
func Test_vmNativeProcessor_run(t *testing.T) {
t.Skip()
type fields struct {
filter filter
rateLimit int64
dst *vmNativeClient
src *vmNativeClient
}
tests := []struct {
name string
fields fields
closer func(cancelFunc context.CancelFunc)
wantErr bool
}{
{
name: "simulate syscall.SIGINT",
fields: fields{
filter: filter{
match: matchFilter,
timeStart: timeStartFilter,
},
rateLimit: 0,
dst: &vmNativeClient{
addr: dstAddr,
},
src: &vmNativeClient{
addr: srcAddr,
},
},
closer: func(cancelFunc context.CancelFunc) {
time.Sleep(time.Second * 5)
cancelFunc()
},
wantErr: true,
},
{
name: "simulate correct work",
fields: fields{
filter: filter{
match: matchFilter,
timeStart: timeStartFilter,
},
rateLimit: 0,
dst: &vmNativeClient{
addr: dstAddr,
},
src: &vmNativeClient{
addr: srcAddr,
},
},
closer: func(cancelFunc context.CancelFunc) {},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancelFn := context.WithCancel(context.Background())
p := &vmNativeProcessor{
filter: tt.fields.filter,
rateLimit: tt.fields.rateLimit,
dst: tt.fields.dst,
src: tt.fields.src,
}
tt.closer(cancelFn)
if err := p.run(ctx); (err != nil) != tt.wantErr {
t.Errorf("run() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

49
app/vmselect/main.go
View file

@ -6,6 +6,8 @@ import (
"flag"
"fmt"
"net/http"
"net/http/httputil"
"net/url"
"strings"
"time"
@ -31,6 +33,7 @@ var (
"limit is reached; see also -search.maxQueryDuration")
resetCacheAuthKey = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging")
vmalertProxyURL = flag.String("vmalert.proxyURL", "", "Optional URL for proxying alerting API requests from Grafana. For example, if -vmalert.proxyURL is set to http://vmalert:8880 , then requests to /api/v1/rules are proxied to http://vmalert:8880/api/v1/rules")
)
var slowQueries = metrics.NewCounter(`vm_slow_queries_total`)
@ -57,6 +60,7 @@ func Init() {
promql.InitRollupResultCache(*vmstorage.DataPath + "/cache/rollupResult")
concurrencyCh = make(chan struct{}, *maxConcurrentRequests)
initVMAlertProxy()
}
// Stop stops vmselect
@ -404,14 +408,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
case "/api/v1/rules", "/rules":
// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#rules
rulesRequests.Inc()
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, "%s", `{"status":"success","data":{"groups":[]}}`)
mayProxyVMAlertRequests(w, r, `{"status":"success","data":{"groups":[]}}`)
return true
case "/api/v1/alerts", "/alerts":
// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
alertsRequests.Inc()
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`)
mayProxyVMAlertRequests(w, r, `{"status":"success","data":{"alerts":[]}}`)
return true
case "/api/v1/metadata":
// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
@ -562,3 +564,42 @@ var (
buildInfoRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/buildinfo"}`)
queryExemplarsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/query_exemplars"}`)
)
func mayProxyVMAlertRequests(w http.ResponseWriter, r *http.Request, stubResponse string) {
if len(*vmalertProxyURL) == 0 {
// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#rules
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, "%s", stubResponse)
return
}
defer func() {
err := recover()
if err == nil || err == http.ErrAbortHandler {
// Suppress http.ErrAbortHandler panic.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1353
return
}
// Forward other panics to the caller.
panic(err)
}()
r.Host = vmalertProxyHost
vmalertProxy.ServeHTTP(w, r)
}
var (
vmalertProxyHost string
vmalertProxy *httputil.ReverseProxy
)
// initVMAlertProxy must be called after flag.Parse(), since it uses command-line flags.
func initVMAlertProxy() {
if len(*vmalertProxyURL) == 0 {
return
}
proxyURL, err := url.Parse(*vmalertProxyURL)
if err != nil {
logger.Fatalf("cannot parse -vmalert.proxyURL=%q: %s", *vmalertProxyURL, err)
}
vmalertProxyHost = proxyURL.Host
vmalertProxy = httputil.NewSingleHostReverseProxy(proxyURL)
}

147
app/vmselect/prometheus/prometheus.go
View file

@ -118,7 +118,6 @@ var federateDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/fe
func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
defer exportCSVDuration.UpdateDuration(startTime)
ct := startTime.UnixNano() / 1e6
if err := r.ParseForm(); err != nil {
return fmt.Errorf("cannot parse request form values: %w", err)
}
@ -127,21 +126,13 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
return fmt.Errorf("missing `format` arg; see https://docs.victoriametrics.com/#how-to-export-csv-data")
}
fieldNames := strings.Split(format, ",")
start, err := searchutils.GetTime(r, "start", 0)
if err != nil {
return err
}
end, err := searchutils.GetTime(r, "end", ct)
if err != nil {
return err
}
reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage")
deadline := searchutils.GetDeadlineForExport(r, startTime)
tagFilterss, err := getTagFilterssFromRequest(r)
ep, err := getExportParams(r, startTime)
if err != nil {
return err
}
sq := storage.NewSearchQuery(start, end, tagFilterss, *maxExportSeries)
sq := storage.NewSearchQuery(ep.start, ep.end, ep.filterss, *maxExportSeries)
w.Header().Set("Content-Type", "text/csv; charset=utf-8")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
@ -157,7 +148,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
}
doneCh := make(chan error, 1)
if !reduceMemUsage {
rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
rss, err := netstorage.ProcessSearchQuery(sq, true, ep.deadline)
if err != nil {
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
}
@ -180,7 +171,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
}()
} else {
go func() {
err := netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
err := netstorage.ExportBlocks(sq, ep.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
if err := bw.Error(); err != nil {
return err
}
@ -221,36 +212,27 @@ var exportCSVDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/a
func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
defer exportNativeDuration.UpdateDuration(startTime)
ct := startTime.UnixNano() / 1e6
if err := r.ParseForm(); err != nil {
return fmt.Errorf("cannot parse request form values: %w", err)
}
start, err := searchutils.GetTime(r, "start", 0)
ep, err := getExportParams(r, startTime)
if err != nil {
return err
}
end, err := searchutils.GetTime(r, "end", ct)
if err != nil {
return err
}
deadline := searchutils.GetDeadlineForExport(r, startTime)
tagFilterss, err := getTagFilterssFromRequest(r)
if err != nil {
return err
}
sq := storage.NewSearchQuery(start, end, tagFilterss, *maxExportSeries)
sq := storage.NewSearchQuery(ep.start, ep.end, ep.filterss, *maxExportSeries)
w.Header().Set("Content-Type", "VictoriaMetrics/native")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
// Marshal tr
trBuf := make([]byte, 0, 16)
trBuf = encoding.MarshalInt64(trBuf, start)
trBuf = encoding.MarshalInt64(trBuf, end)
trBuf = encoding.MarshalInt64(trBuf, ep.start)
trBuf = encoding.MarshalInt64(trBuf, ep.end)
_, _ = bw.Write(trBuf)
// Marshal native blocks.
err = netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
err = netstorage.ExportBlocks(sq, ep.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
if err := bw.Error(); err != nil {
return err
}
@ -295,42 +277,25 @@ var bbPool bytesutil.ByteBufferPool
func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
defer exportDuration.UpdateDuration(startTime)
ct := startTime.UnixNano() / 1e6
if err := r.ParseForm(); err != nil {
return fmt.Errorf("cannot parse request form values: %w", err)
}
matches := getMatchesFromRequest(r)
if len(matches) == 0 {
return fmt.Errorf("missing `match[]` query arg")
}
start, err := searchutils.GetTime(r, "start", 0)
if err != nil {
return err
}
end, err := searchutils.GetTime(r, "end", ct)
ep, err := getExportParams(r, startTime)
if err != nil {
return err
}
format := r.FormValue("format")
maxRowsPerLine := int(fastfloat.ParseInt64BestEffort(r.FormValue("max_rows_per_line")))
reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage")
deadline := searchutils.GetDeadlineForExport(r, startTime)
if start >= end {
end = start + defaultStep
}
etfs, err := searchutils.GetExtraTagFilters(r)
if err != nil {
return err
}
if err := exportHandler(w, matches, etfs, start, end, format, maxRowsPerLine, reduceMemUsage, deadline); err != nil {
return fmt.Errorf("error when exporting data for queries=%q on the time range (start=%d, end=%d): %w", matches, start, end, err)
if err := exportHandler(w, ep, format, maxRowsPerLine, reduceMemUsage); err != nil {
return fmt.Errorf("error when exporting data on the time range (start=%d, end=%d): %w", ep.start, ep.end, err)
}
return nil
}
var exportDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export"}`)
func exportHandler(w http.ResponseWriter, matches []string, etfs [][]storage.TagFilter, start, end int64, format string, maxRowsPerLine int, reduceMemUsage bool, deadline searchutils.Deadline) error {
func exportHandler(w http.ResponseWriter, ep *exportParams, format string, maxRowsPerLine int, reduceMemUsage bool) error {
writeResponseFunc := WriteExportStdResponse
writeLineFunc := func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
bb := quicktemplate.AcquireByteBuffer()
@ -383,13 +348,7 @@ func exportHandler(w http.ResponseWriter, matches []string, etfs [][]storage.Tag
}
}
tagFilterss, err := getTagFilterssFromMatches(matches)
if err != nil {
return err
}
tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
sq := storage.NewSearchQuery(start, end, tagFilterss, *maxExportSeries)
sq := storage.NewSearchQuery(ep.start, ep.end, ep.filterss, *maxExportSeries)
w.Header().Set("Content-Type", contentType)
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
@ -397,7 +356,7 @@ func exportHandler(w http.ResponseWriter, matches []string, etfs [][]storage.Tag
resultsCh := make(chan *quicktemplate.ByteBuffer, cgroup.AvailableCPUs())
doneCh := make(chan error, 1)
if !reduceMemUsage {
rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
rss, err := netstorage.ProcessSearchQuery(sq, true, ep.deadline)
if err != nil {
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
}
@ -420,7 +379,7 @@ func exportHandler(w http.ResponseWriter, matches []string, etfs [][]storage.Tag
}()
} else {
go func() {
err := netstorage.ExportBlocks(sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
err := netstorage.ExportBlocks(sq, ep.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
if err := bw.Error(); err != nil {
return err
}
@ -447,7 +406,7 @@ func exportHandler(w http.ResponseWriter, matches []string, etfs [][]storage.Tag
if err := bw.Flush(); err != nil {
return err
}
err = <-doneCh
err := <-doneCh
if err != nil {
return fmt.Errorf("error during sending the data to remote client: %w", err)
}
@ -1049,7 +1008,20 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e
if end < start {
end = start
}
if err := exportHandler(w, []string{childQuery}, etfs, start, end, "promapi", 0, false, deadline); err != nil {
tagFilterss, err := getTagFilterssFromMatches([]string{childQuery})
if err != nil {
return err
}
filterss := searchutils.JoinTagFilterss(tagFilterss, etfs)
ep := &exportParams{
deadline: deadline,
start: start,
end: end,
filterss: filterss,
}
if err := exportHandler(w, ep, "promapi", 0, false); err != nil {
return fmt.Errorf("error when exporting data for query=%q on the time range (start=%d, end=%d): %w", childQuery, start, end, err)
}
queryDuration.UpdateDuration(startTime)
@ -1379,3 +1351,56 @@ func QueryStatsHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
}
var queryStatsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/status/top_queries"}`)
// exportParams contains common parameters for all /api/v1/export* handlers
//
// deadline, start, end, match[], extra_label, extra_filters
type exportParams struct {
deadline searchutils.Deadline
start int64
end int64
filterss [][]storage.TagFilter
}
// getExportParams obtains common params from r, which are used for /api/v1/export* handlers
func getExportParams(r *http.Request, startTime time.Time) (*exportParams, error) {
deadline := searchutils.GetDeadlineForExport(r, startTime)
start, err := searchutils.GetTime(r, "start", 0)
if err != nil {
return nil, err
}
ct := startTime.UnixNano() / 1e6
end, err := searchutils.GetTime(r, "end", ct)
if err != nil {
return nil, err
}
if end < start {
end = start
}
matches := r.Form["match[]"]
if len(matches) == 0 {
// Maintain backwards compatibility
match := r.FormValue("match")
if len(match) == 0 {
return nil, fmt.Errorf("missing `match[]` arg")
}
matches = []string{match}
}
tagFilterss, err := getTagFilterssFromMatches(matches)
if err != nil {
return nil, err
}
etfs, err := searchutils.GetExtraTagFilters(r)
if err != nil {
return nil, err
}
filterss := searchutils.JoinTagFilterss(tagFilterss, etfs)
return &exportParams{
deadline: deadline,
start: start,
end: end,
filterss: filterss,
}, nil
}

4
app/vmselect/vmui/asset-manifest.json
View file

@ -1,7 +1,7 @@
{
"files": {
"main.css": "./static/css/main.d8362c27.css",
"main.js": "./static/js/main.9635653f.js",
"main.js": "./static/js/main.214cd305.js",
"static/js/362.1f16598a.chunk.js": "./static/js/362.1f16598a.chunk.js",
"static/js/27.939f971b.chunk.js": "./static/js/27.939f971b.chunk.js",
"static/media/README.md": "./static/media/README.40ebc3a1f4adae949154.md",
@ -9,6 +9,6 @@
},
"entrypoints": [
"static/css/main.d8362c27.css",
"static/js/main.9635653f.js"
"static/js/main.214cd305.js"
]
}

2
app/vmselect/vmui/index.html
View file

@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script defer="defer" src="./static/js/main.9635653f.js"></script><link href="./static/css/main.d8362c27.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script defer="defer" src="./static/js/main.214cd305.js"></script><link href="./static/css/main.d8362c27.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

4
app/vmselect/vmui/static/js/main.9635653f.js → app/vmselect/vmui/static/js/main.214cd305.js
View file

File diff suppressed because one or more lines are too long

0
app/vmselect/vmui/static/js/main.9635653f.js.LICENSE.txt → app/vmselect/vmui/static/js/main.214cd305.js.LICENSE.txt
View file

16
app/vmstorage/main.go
View file

@ -75,7 +75,7 @@ func CheckTimeRange(tr storage.TimeRange) error {
// Init initializes vmstorage.
func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
InitWithoutMetrics(resetCacheIfNeeded)
registerStorageMetrics()
registerStorageMetrics(Storage)
}
// InitWithoutMetrics must be called instead of Init inside tests.
@ -104,10 +104,10 @@ func InitWithoutMetrics(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
logger.Fatalf("cannot open a storage at %s with -retentionPeriod=%s: %s", *DataPath, retentionPeriod, err)
}
Storage = strg
initStaleSnapshotsRemover()
initStaleSnapshotsRemover(strg)
var m storage.Metrics
Storage.UpdateMetrics(&m)
strg.UpdateMetrics(&m)
tm := &m.TableMetrics
partsCount := tm.SmallPartsCount + tm.BigPartsCount
blocksCount := tm.SmallBlocksCount + tm.BigBlocksCount
@ -381,7 +381,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
}
}
func initStaleSnapshotsRemover() {
func initStaleSnapshotsRemover(strg *storage.Storage) {
staleSnapshotsRemoverCh = make(chan struct{})
if *snapshotsMaxAge <= 0 {
return
@ -397,7 +397,7 @@ func initStaleSnapshotsRemover() {
return
case <-t.C:
}
if err := Storage.DeleteStaleSnapshots(*snapshotsMaxAge); err != nil {
if err := strg.DeleteStaleSnapshots(*snapshotsMaxAge); err != nil {
// Use logger.Errorf instead of logger.Fatalf in the hope the error is temporary.
logger.Errorf("cannot delete stale snapshots: %s", err)
}
@ -417,7 +417,7 @@ var (
var activeForceMerges = metrics.NewCounter("vm_active_force_merges")
func registerStorageMetrics() {
func registerStorageMetrics(strg *storage.Storage) {
mCache := &storage.Metrics{}
var mCacheLock sync.Mutex
var lastUpdateTime time.Time
@ -429,7 +429,7 @@ func registerStorageMetrics() {
return mCache
}
var mc storage.Metrics
Storage.UpdateMetrics(&mc)
strg.UpdateMetrics(&mc)
mCache = &mc
lastUpdateTime = time.Now()
return mCache
@ -450,7 +450,7 @@ func registerStorageMetrics() {
return float64(minFreeDiskSpaceBytes.N)
})
metrics.NewGauge(fmt.Sprintf(`vm_storage_is_read_only{path=%q}`, *DataPath), func() float64 {
if Storage.IsReadOnly() {
if strg.IsReadOnly() {
return 1
}
return 0

2
app/vmui/packages/vmui/src/components/CustomPanel/Configurator/Query/QueryEditor.tsx
View file

@ -63,7 +63,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
const hasAutocomplete = openAutocomplete && actualOptions.length;
if ((arrowUp || arrowDown || enter) && (hasAutocomplete || ctrlMetaKey || !shiftKey)) {
if (((arrowUp || arrowDown) && (hasAutocomplete || ctrlMetaKey)) || (enter && (hasAutocomplete || ctrlMetaKey || !shiftKey))) {
e.preventDefault();
}

13
docs/CHANGELOG.md
View file

@ -15,32 +15,39 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for sending data to remote storage with AWS sigv4 authorization. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1287).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow filtering targets by target url and by target labels with [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) on `http://vmagent:8429/targets` page. This may be useful when `vmagent` scrapes big number of targets. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1796).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): reduce `-promscrape.config` reload duration when the config contains big number of jobs (aka [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) sections) and only a few of them are changed. Previously all the jobs were restarted. Now only the jobs with changed configs are restarted. This should reduce the probability of data miss because of slow config reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2270).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): improve service discovery speed for big number of scrape targets. This should help when `vmagent` discovers big number of targets (e.g. thousands) in Kubernetes cluster. The service discovery speed now should scale with the number of CPU cores available to `vmagent`.
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to attach node-level labels and annotations to discovered Kubernetes pod targets in the same way as Prometheus 2.35 does. See [this feature request](https://github.com/prometheus/prometheus/issues/9510) and [this pull request](https://github.com/prometheus/prometheus/pull/10080).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for `tls_config` and `proxy_url` options at `oauth2` section in the same way as Prometheus does. See [oauth2 docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for `min_version` option at `tls_config` section in the same way as Prometheus does. See [tls_config docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add support for DNS-based discovery for notifiers in the same way as Prometheus does. See [these docs](https://docs.victoriametrics.com/vmalert.html#notifier-configuration-file) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2460).
* FEATURE: [vmagent](): expose `vmagent_remotewrite_rate_limit` metric at `http://vmagent:8429/metrics`, which can be used for alerting rules such as `rate(vmagent_remotewrite_conn_bytes_written_total) / vmagent_remotewrite_rate_limit > 0.8` when `-remoteWrite.rateLimit` command-line flag is set. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2521).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add support for DNS-based discovery for notifiers in the same way as Prometheus does (aka `dns_sd_configs`). See [these docs](https://docs.victoriametrics.com/vmalert.html#notifier-configuration-file) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2460).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `-replay.disableProgressBar` command-line flag, which allows disabling progressbar in [rules' backfilling mode](https://docs.victoriametrics.com/vmalert.html#rules-backfilling). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1761).
* FEATURE: allow specifying TLS cipher suites for incoming https requests via `-tlsCipherSuites` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2404).
* FEATURE: allow specifying TLS cipher suites for mTLS connections between cluster components via `-cluster.tlsCipherSuites` command-line flag. See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection).
* FEATURE: vmstorage: add `-snapshotsMaxAge` command-line flag for automatic removal of snapshots older than the given age.
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): shown an empty graph on the selected time range when there is no data on it. Previously `No data to show` placeholder was shown instead of the graph in this case. This prevented from zooming and scrolling of such a graph.
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): show an empty graph on the selected time range when there is no data on it. Previously `No data to show` placeholder was shown instead of the graph in this case. This prevented from zooming and scrolling of such a graph.
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): show the selected `last N minutes/hours/days` in the top right corner. Previously the `start - end` duration was shown instead, which could be hard to interpret. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2402).
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): execute the query when `enter` button is present in the same way as Prometheus does. Multi-line query can be entered by pressing `shift-enter` in the query input field.
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): execute the query when `enter` button is pressed in the same way as Prometheus does. Multi-line query can be entered by pressing `shift-enter` in the query input field.
* FEATURE: expose `vm_indexdb_items_added_total` and `vm_indexdb_items_added_size_bytes_total` counters at `/metrics` page, which can be used for monitoring the rate for addition of new entries in `indexdb` (aka `inverted index`) alongside the total size in bytes for the added entries. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2471).
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): show data pocessing speed during data migration.
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `drop_common_labels()` function, which drops common `label="name"` pairs from the passed time series. See [these docs](https://docs.victoriametrics.com/MetricsQL.html#drop_common_labels).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `tlast_change_over_time(m[d])` function, which returns the timestamp of the last change of `m` on the given lookbehind window `d`. See [these docs](https://docs.victoriametrics.com/MetricsQL.html#tlast_change_over_time).
* FEATURE: leave the last raw sample per each `-dedup.minScrapeInterval` discrete interval when the [deduplication](https://docs.victoriametrics.com/#deduplication) is enabled. This aligns better with the [staleness rules in Prometheus](https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness) comparing to the previous behaviour when the first sample per each `-dedup.minScrapeInterval` was left.
* FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): add ability to disable peer TLS certificate verification with `-cluster.tlsInsecureSkipVerify` command-line flag. See [mTLS docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection) for details. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2490).
* FEATURE: add a handler for `/api/v1/status/buildinfo` endpoint, which is used by Grafana starting from v8.5.0 . See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2515).
* FEATURE: add ability to proxy alerting API requests from Grafana to vmalert by passing `-vmalert.proxyURL` command-line flag to single-node VictoriaMetrics or to `vmselect` at cluster version of VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1739).
* BUGFIX: export staleness markers as `null` values from [JSON export API](https://docs.victoriametrics.com/#how-to-export-data-in-json-line-format). Previously they were exported as `NaN` values. This could break the exported JSON parsing, since `NaN` values aren't supported by [JSON specification](https://www.json.org/).
* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): close `vmselect->vmstorage` connections if they were idle for more than 30 seconds. Expose `vm_tcpdialer_conns_idle` metric at `http://vmselect:8481/metrics` with the number of idle connections to `vmstorage`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2508).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): return non-zero exit code on error. This allows handling `vmctl` errors in shell scripts. Previously `vmctl` was returning 0 exit code on error. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2322).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): prevent from indefinite hang on `Ctrl+C`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2491).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly show `scrape_timeout` and `scrape_interval` options at `http://vmagent:8429/config` page. Previously these options weren't displayed even if they were set in `-promscrape.config`.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): handle non-standard http redirect status codes, which may be returned by scrape targets, in the same way as Prometheus does. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2482).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): skip template execution during rules' validation. This should prevent from `error evaluating annotation template` errors when some template functions expect non-empty args. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2514).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fixed truncating alerts expression in table, updated table cell layout. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2484).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle joins on time series filtered by values. For example, `kube_pod_container_resource_requests{resource="cpu"} * on (namespace,pod) group_left() (kube_pod_status_phase{phase=~"Pending|Running"}==1)`. This query could result in `duplicate time series on the right side` error even if `==1` filter leaves only a single time series per `(namespace,pod)` labels. Now such query is properly executed.
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle `scalar default vector`, `scalar if vector` and `scalar ifnot vector` queries. Previously such queries could return unexpected results from the `vector` part.
* BUGFIX: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): take into account `indexdb` when calculating disk space usage. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2368).

41
docs/Cluster-VictoriaMetrics.md
View file

@ -152,17 +152,6 @@ It is possible manualy setting up a toy cluster on a single host. In this case e
- `-vminsertAddr` - every `vmstorage` node must listen for a distinct tcp address for accepting data from `vminsert` nodes.
- `-vmselectAddr` - every `vmstorage` node must listen for a distinct tcp address for accepting requests from `vmselect` nodes.
## mTLS protection
By default `vminsert` and `vmselect` nodes use unencrypted connections to `vmstorage` nodes, since it is assumed that all the cluster components run in a protected environment. [Enterprise version of VictoriaMetrics](https://victoriametrics.com/products/enterprise/) provides optional support for [mTLS connections](https://en.wikipedia.org/wiki/Mutual_authentication#mTLS) between cluster components. Pass `-cluster.tls=true` command-line flag to `vminsert`, `vmselect` and `vmstorage` nodes in order to enable mTLS protection. Additionally, `vminsert`, `vmselect` and `vmstorage` must be configured with mTLS certificates via `-cluster.tlsCertFile`, `-cluster.tlsKeyFile` command-line options. These certificates are mutually verified when `vminsert` and `vmselect` dial `vmstorage`.
The following optional command-line flags related to mTLS are supported:
- `-cluster.tlsCAFile` can be set at `vminsert`, `vmselect` and `vmstorage` for verifying peer certificates issued with custom [certificate authority](https://en.wikipedia.org/wiki/Certificate_authority). By default system-wide certificate authority is used for peer certificate verification.
- `-cluster.tlsCipherSuites` can be set to the list of supported TLS cipher suites at `vmstorage`. See [the list of supported TLS cipher suites](https://pkg.go.dev/crypto/tls#pkg-constants).
[Enterprise version of VictoriaMetrics](https://victoriametrics.com/products/enterprise/) can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
### Environment variables
Each flag values can be set thru environment variables by following these rules:
@ -172,6 +161,20 @@ Each flag values can be set thru environment variables by following these rules:
- For repeating flags, an alternative syntax can be used by joining the different values into one using `,` as separator (for example `-storageNode <nodeA> -storageNode <nodeB>` will translate to `storageNode=<nodeA>,<nodeB>`)
- It is possible setting prefix for environment vars with `-envflag.prefix`. For instance, if `-envflag.prefix=VM_`, then env vars must be prepended with `VM_`
## mTLS protection
By default `vminsert` and `vmselect` nodes use unencrypted connections to `vmstorage` nodes, since it is assumed that all the cluster components run in a protected environment. [Enterprise version of VictoriaMetrics](https://victoriametrics.com/products/enterprise/) provides optional support for [mTLS connections](https://en.wikipedia.org/wiki/Mutual_authentication#mTLS) between cluster components. Pass `-cluster.tls=true` command-line flag to `vminsert`, `vmselect` and `vmstorage` nodes in order to enable mTLS protection. Additionally, `vminsert`, `vmselect` and `vmstorage` must be configured with mTLS certificates via `-cluster.tlsCertFile`, `-cluster.tlsKeyFile` command-line options. These certificates are mutually verified when `vminsert` and `vmselect` dial `vmstorage`.
The following optional command-line flags related to mTLS are supported:
- `-cluster.tlsInsecureSkipVerify` can be set at `vminsert`, `vmselect` and `vmstorage` in order to disable peer certificate verification. Note that this breaks security.
- `-cluster.tlsCAFile` can be set at `vminsert`, `vmselect` and `vmstorage` for verifying peer certificates issued with custom [certificate authority](https://en.wikipedia.org/wiki/Certificate_authority). By default system-wide certificate authority is used for peer certificate verification.
- `-cluster.tlsCipherSuites` can be set to the list of supported TLS cipher suites at `vmstorage`. See [the list of supported TLS cipher suites](https://pkg.go.dev/crypto/tls#pkg-constants).
See [these docs](https://gist.github.com/f41gh7/76ed8e5fb1ebb9737fe746bae9175ee6) on how to set up mTLS in VictoriaMetrics cluster.
[Enterprise version of VictoriaMetrics](https://victoriametrics.com/products/enterprise/) can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
## Monitoring
All the cluster components expose various metrics in Prometheus-compatible format at `/metrics` page on the TCP port set in `-httpListenAddr` command-line flag.
@ -498,6 +501,8 @@ Below is the output for `/path/to/vminsert -help`:
Path to TLS CA file to use for verifying certificates provided by -storageNode if -cluster.tls flag is set. By default system CA is used. See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection
-cluster.tlsCertFile string
Path to client-side TLS certificate file to use when connecting to -storageNode if -cluster.tls flag is set. See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection
-cluster.tlsInsecureSkipVerify
Whether to skip verification of TLS certificates provided by -storageNode nodes if -cluster.tls flag is set. Note that disabled TLS certificate verification breaks security
-cluster.tlsKeyFile string
Path to client-side TLS key file to use when connecting to -storageNode if -cluster.tls flag is set. See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection
-clusternativeListenAddr string
@ -640,10 +645,12 @@ Below is the output for `/path/to/vmselect -help`:
Path to TLS CA file to use for verifying certificates provided by -storageNode if -cluster.tls flag is set. By default system CA is used. See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection
-cluster.tlsCertFile string
Path to client-side TLS certificate file to use when connecting to -storageNode if -cluster.tls flag is set. See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection
-cluster.tlsInsecureSkipVerify
Whether to skip verification of TLS certificates provided by -storageNode nodes if -cluster.tls flag is set. Note that disabled TLS certificate verification breaks security
-cluster.tlsKeyFile string
Path to client-side TLS key file to use when connecting to -storageNode if -cluster.tls flag is set. See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection
-dedup.minScrapeInterval duration
Leave only the first sample in every time series per each discrete interval equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details
Leave only the last sample in every time series per each discrete interval equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details
-downsampling.period array
Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details
Supports an array of values separated by comma or specified via multiple flags.
@ -758,7 +765,7 @@ Below is the output for `/path/to/vmselect -help`:
-search.treatDotsAsIsInRegexps
Whether to treat dots as is in regexp label filters used in queries. For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. This option is DEPRECATED in favor of {__graphite__="a.*.c"} syntax for selecting metrics matching the given Graphite metrics filter
-selectNode array
Comma-serparated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN
Comma-separated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN
Supports an array of values separated by comma or specified via multiple flags.
-storageNode array
Comma-separated addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN
@ -774,6 +781,8 @@ Below is the output for `/path/to/vmselect -help`:
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-version
Show VictoriaMetrics version
-vmalert.proxyURL string
Optional URL for proxying alerting API requests from Grafana. For example, if -vmalert.proxyURL is set to http://vmalert:8880 , then requests to /api/v1/rules are proxied to http://vmalert:8880/api/v1/rules
```
### List of command-line flags for vmstorage
@ -792,10 +801,12 @@ Below is the output for `/path/to/vmstorage -help`:
-cluster.tlsCipherSuites array
Optional list of TLS cipher suites used for connections from vminsert and vmselect if -cluster.tls flag is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants
Supports an array of values separated by comma or specified via multiple flags.
-cluster.tlsInsecureSkipVerify
Whether to skip verification of TLS certificates provided by vminsert and vmselect if -cluster.tls flag is set. Note that disabled TLS certificate verification breaks security
-cluster.tlsKeyFile string
Path to server-side TLS key file to use when accepting connections from vminsert and vmselect if -cluster.tls flag is set. See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#mtls-protection
-dedup.minScrapeInterval duration
Leave only the first sample in every time series per each discrete interval equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details
Leave only the last sample in every time series per each discrete interval equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details
-denyQueriesOutsideRetention
Whether to deny queries outside of the configured -retentionPeriod. When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. This may be useful when multiple data sources with distinct retentions are hidden behind query-tee
-downsampling.period array
@ -871,6 +882,8 @@ Below is the output for `/path/to/vmstorage -help`:
The maximum number of CPU cores to use for small merges. Default value is used if set to 0
-snapshotAuthKey string
authKey, which must be passed in query string to /snapshot* pages
-snapshotsMaxAge duration
Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted
-storage.cacheSizeIndexDBDataBlocks size
Overrides max size for indexdb/dataBlocks cache. See https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#cache-tuning
Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 0)

4
docs/README.md
View file

@ -1899,6 +1899,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
The maximum number of CPU cores to use for small merges. Default value is used if set to 0
-snapshotAuthKey string
authKey, which must be passed in query string to /snapshot* pages
-snapshotsMaxAge duration
Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted
-sortLabels
Whether to sort labels for incoming samples before writing them to storage. This may be needed for reducing memory usage at storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}. Enabled sorting for labels can slow down ingestion performance a bit
-storage.cacheSizeIndexDBDataBlocks size
@ -1930,4 +1932,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-version
Show VictoriaMetrics version
-vmalert.proxyURL string
Optional URL for proxying alerting API requests from Grafana. For example, if -vmalert.proxyURL is set to http://vmalert:8880 , then requests to /api/v1/rules are proxied to http://vmalert:8880/api/v1/rules
```

4
docs/Single-server-VictoriaMetrics.md
View file

@ -1903,6 +1903,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
The maximum number of CPU cores to use for small merges. Default value is used if set to 0
-snapshotAuthKey string
authKey, which must be passed in query string to /snapshot* pages
-snapshotsMaxAge duration
Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted
-sortLabels
Whether to sort labels for incoming samples before writing them to storage. This may be needed for reducing memory usage at storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}. Enabled sorting for labels can slow down ingestion performance a bit
-storage.cacheSizeIndexDBDataBlocks size
@ -1934,4 +1936,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-version
Show VictoriaMetrics version
-vmalert.proxyURL string
Optional URL for proxying alerting API requests from Grafana. For example, if -vmalert.proxyURL is set to http://vmalert:8880 , then requests to /api/v1/rules are proxied to http://vmalert:8880/api/v1/rules
```

10
docs/guides/multi-regional-setup-dedicated-regions.md
View file

@ -26,15 +26,15 @@ Using this schema, you can achieve:
### How to write the data to Ground Control regions
* You need to specify two remote write URLs in vmagent configuration
* You need to pass two `-remoteWrite.url` command-line options to `vmagent`:
```bash
/vmagent-prod
-remoteWrite.url=<ground-control-1-remote-write>
-remoteWrite.url=<ground-control-2-remote-write>
/path/to/vmagent-prod \
-remoteWrite.url=<ground-control-1-remote-write> \
-remoteWrite.url=<ground-control-2-remote-write>
```
* If you use the Pull model for data collection, please specify -promscrape.config parameter as well
* If you scrape data from Prometheus-compatible targets, then please specify `-promscrape.config` parameter as well.
Here is a Quickstart guide for [vmagent](https://docs.victoriametrics.com/vmagent.html#quick-start)

15
docs/vmagent.md
View file

@ -918,6 +918,21 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/vmagent.html#troubleshooting for details
-promscrape.suppressScrapeErrors
Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed
-remoteWrite.aws.accessKey array
Optional AWS AccessKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports an array of values separated by comma or specified via multiple flags.
-remoteWrite.aws.region array
Optional AWS region to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports an array of values separated by comma or specified via multiple flags.
-remoteWrite.aws.roleARN array
Optional AWS roleARN to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports an array of values separated by comma or specified via multiple flags.
-remoteWrite.aws.secretKey array
Optional AWS SecretKey to use for -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports an array of values separated by comma or specified via multiple flags.
-remoteWrite.aws.useSigv4 array
Enables SigV4 request signing for -remoteWrite.url. It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports array of values separated by comma or specified via multiple flags.
-remoteWrite.basicAuth.password array
Optional basic auth password to use for -remoteWrite.url. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url
Supports an array of values separated by comma or specified via multiple flags.

28
docs/vmbackup.md
View file

@ -33,12 +33,12 @@ creation of hourly, daily, weekly and monthly backups.
Regular backup can be performed with the following command:
```bash
vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/new/backup>
vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/new/backup>
```
* `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
There is no need to stop VictoriaMetrics for creating backups since they are performed from immutable [instant snapshots](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots).
* `<local-snapshot>` is the snapshot to back up. See [how to create instant snapshots](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots). `vmbackup` can create the snapshot on itself if `-snapshot.createURL` command-line flag is set to an url for creating snapshots. In this case `-snapshotName` flag isn't needed.
* `http://victoriametrics:8428/snapshot/create` is the url for creating snapshots according to [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots). `vmbackup` creates a snapshot by querying the provided `-snapshot.createURL`, then performs the backup and then automatically removes the created snapshot.
* `<bucket>` is an already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
* `<path/to/new/backup>` is the destination path where new backup will be placed.
@ -48,7 +48,7 @@ If the destination GCS bucket already contains the previous backup at `-origin`
with the following command:
```bash
vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/new/backup> -origin=gs://<bucket>/<path/to/existing/backup>
./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/new/backup> -origin=gs://<bucket>/<path/to/existing/backup>
```
It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
@ -59,7 +59,7 @@ Incremental backups are performed if `-dst` points to an already existing backup
It saves time and network bandwidth costs when working with big backups:
```bash
vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gs://<bucket>/<path/to/existing/backup>
./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/existing/backup>
```
### Smart backups
@ -69,7 +69,7 @@ Smart backups mean storing full daily backups into `YYYYMMDD` folders and creati
* Run the following command every hour:
```bash
vmbackup -snapshotName=<latest-snapshot> -dst=gs://<bucket>/latest
./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/latest
```
Where `<latest-snapshot>` is the latest [snapshot](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots).
@ -78,7 +78,7 @@ The command will upload only changed data to `gs://<bucket>/latest`.
* Run the following command once a day:
```bash
vmbackup -snapshotName=<daily-snapshot> -dst=gs://<bucket>/<YYYYMMDD> -origin=gs://<bucket>/latest
vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<YYYYMMDD> -origin=gs://<bucket>/latest
```
Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
@ -86,7 +86,7 @@ Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
This apporach saves network bandwidth costs on hourly backups (since they are incremental) and allows recovering data from either the last hour (`latest` backup)
or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when creating daily backup.
Do not forget to remove old snapshots and backups when they are no longer needed in order to save storage costs.
Do not forget to remove old backups when they are no longer needed in order to save storage costs.
See also [vmbackupmanager tool](https://docs.victoriametrics.com/vmbackupmanager.html) for automating smart backups.
@ -94,15 +94,17 @@ See also [vmbackupmanager tool](https://docs.victoriametrics.com/vmbackupmanager
The backup algorithm is the following:
1. Collect information about files in the `-snapshotName`, in the `-dst` and in the `-origin`.
2. Determine which files in `-dst` are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
3. Determine which files in `-snapshotName` are missing in `-dst`. These are usually small new files and bigger merged files.
4. Determine which files from step 3 exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
1. Create a snapshot by querying the provided `-snapshot.createURL`
2. Collect information about files in the created snapshot, in the `-dst` and in the `-origin`.
3. Determine which files in `-dst` are missing in the created snapshot, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
4. Determine which files in the created snapshot are missing in `-dst`. These are usually small new files and bigger merged files.
5. Determine which files from step 3 exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
These are usually the biggest and the oldest files, which are shared between backups.
5. Upload the remaining files from step 3 from `-snapshotName` to `-dst`.
6. Upload the remaining files from step 3 from the created snapshot to `-dst`.
7. Delete the created snapshot.
The algorithm splits source files into 1 GiB chunks in the backup. Each chunk is stored as a separate file in the backup.
Such splitting minimizes the amounts of data to re-transfer after temporary errors.
Such splitting balances between the number of files in the backup and the amounts of data that needs to be re-transfered after temporary errors.
`vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:

10
go.mod
View file

@ -11,7 +11,7 @@ require (
github.com/VictoriaMetrics/fasthttp v1.1.0
github.com/VictoriaMetrics/metrics v1.18.1
github.com/VictoriaMetrics/metricsql v0.43.0
github.com/aws/aws-sdk-go v1.44.4
github.com/aws/aws-sdk-go v1.44.7
github.com/cespare/xxhash/v2 v2.1.2
github.com/cheggaaa/pb/v3 v3.0.9-0.20211222075416-90c02fa07ea4
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
@ -33,13 +33,13 @@ require (
github.com/valyala/quicktemplate v1.7.0
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba
google.golang.org/api v0.77.0
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6
google.golang.org/api v0.78.0
gopkg.in/yaml.v2 v2.4.0
)
require (
cloud.google.com/go v0.101.0 // indirect
cloud.google.com/go v0.101.1 // indirect
cloud.google.com/go/compute v1.6.1 // indirect
cloud.google.com/go/iam v0.3.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
@ -69,7 +69,7 @@ require (
golang.org/x/text v0.3.7 // indirect
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e // indirect
google.golang.org/genproto v0.0.0-20220504150022-98cd25cafc72 // indirect
google.golang.org/grpc v1.46.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect

18
go.sum
View file

@ -30,8 +30,8 @@ cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW
cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
cloud.google.com/go v0.101.0 h1:g+LL+JvpvdyGtcaD2xw2mSByE/6F9s471eJSoaysM84=
cloud.google.com/go v0.101.0/go.mod h1:hEiddgDb77jDQ+I80tURYNJEnuwPzFU8awCFFRLKjW0=
cloud.google.com/go v0.101.1 h1:3+/0TAm9JD/PyhkrDWQWi2L197h3euCsM+H+J4iYTR8=
cloud.google.com/go v0.101.1/go.mod h1:55HwjsGW4CHD3JrNuMdZtSDsgTs0CuCB/bBTugD+7AA=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@ -166,8 +166,8 @@ github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go v1.44.4 h1:ePN0CVJMdiz2vYUcJH96eyxRrtKGSDMgyhP6rah2OgE=
github.com/aws/aws-sdk-go v1.44.4/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go v1.44.7 h1:LpCM8Fpw/L58vgdve6A+UqJr8dzo6Xj7HX7DIIGHg2A=
github.com/aws/aws-sdk-go v1.44.7/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o=
@ -1330,8 +1330,9 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba h1:AyHWHCBVlIYI5rgEM3o+1PLd0sLPcIAoaUckGQMaWtw=
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@ -1487,8 +1488,9 @@ google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/S
google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
google.golang.org/api v0.77.0 h1:msijLTxwkJ7Jub5tv9KBVCKtHOQwnvnvkX7ErFFCVxY=
google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
google.golang.org/api v0.78.0 h1:5ewPyCwP43C3i8B6C2Kb+eVAevbnke2xR8VbcSWjS4I=
google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1581,8 +1583,10 @@ google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX
google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e h1:gMjH4zLGs9m+dGzR7qHCHaXMOwsJHJKKkHtyXhtOrJk=
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/genproto v0.0.0-20220504150022-98cd25cafc72 h1:iif0mpUetMBqcQPUoq+JnCcmzvfpp8wRx515va8wP1c=
google.golang.org/genproto v0.0.0-20220504150022-98cd25cafc72/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=

428
lib/awsapi/config.go Normal file
View file

@ -0,0 +1,428 @@
package awsapi
import (
"encoding/json"
"encoding/xml"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
// Config represent aws access configuration.
type Config struct {
client *http.Client
region string
roleARN string
webTokenPath string
filtersQueryString string
ec2Endpoint string
stsEndpoint string
// these keys are needed for obtaining creds.
defaultAccessKey string
defaultSecretKey string
// Real credentials used for accessing EC2 API.
creds *credentials
credsLock sync.Mutex
}
// credentials represent aws api credentials.
type credentials struct {
AccessKeyID string
SecretAccessKey string
Token string
Expiration time.Time
}
// NewConfig returns new AWS Config.
//
// filtersQueryString must contain an optional percent-encoded query string for aws filters.
// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html for examples.
func NewConfig(region, roleARN, accessKey, secretKey, filtersQueryString string) (*Config, error) {
cfg := &Config{
client: http.DefaultClient,
region: region,
roleARN: roleARN,
filtersQueryString: filtersQueryString,
defaultAccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
defaultSecretKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
}
cfg.region = region
if cfg.region == "" {
r, err := getDefaultRegion(cfg.client)
if err != nil {
return nil, fmt.Errorf("cannot determine default AWS region: %w", err)
}
cfg.region = r
}
cfg.ec2Endpoint = buildAPIEndpoint(cfg.ec2Endpoint, cfg.region, "ec2")
cfg.stsEndpoint = buildAPIEndpoint(cfg.stsEndpoint, cfg.region, "sts")
if cfg.roleARN == "" {
cfg.roleARN = os.Getenv("AWS_ROLE_ARN")
}
cfg.webTokenPath = os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
if cfg.webTokenPath != "" && cfg.roleARN == "" {
return nil, fmt.Errorf("roleARN is missing for AWS_WEB_IDENTITY_TOKEN_FILE=%q; set it via env var AWS_ROLE_ARN", cfg.webTokenPath)
}
// explicitly set credentials has priority over env variables
cfg.defaultAccessKey = os.Getenv("AWS_ACCESS_KEY_ID")
cfg.defaultSecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
if len(accessKey) > 0 {
cfg.defaultAccessKey = accessKey
}
if len(secretKey) > 0 {
cfg.defaultSecretKey = secretKey
}
cfg.creds = &credentials{
AccessKeyID: cfg.defaultAccessKey,
SecretAccessKey: cfg.defaultSecretKey,
}
return cfg, nil
}
// GetEC2APIResponse performs EC2 API request with ghe given action.
func (cfg *Config) GetEC2APIResponse(action, nextPageToken string) ([]byte, error) {
ac, err := cfg.getFreshAPICredentials()
if err != nil {
return nil, err
}
apiURL := fmt.Sprintf("%s?Action=%s", cfg.ec2Endpoint, url.QueryEscape(action))
if len(cfg.filtersQueryString) > 0 {
apiURL += "&" + cfg.filtersQueryString
}
if len(nextPageToken) > 0 {
apiURL += fmt.Sprintf("&NextToken=%s", url.QueryEscape(nextPageToken))
}
apiURL += "&Version=2013-10-15"
req, err := newSignedGetRequest(apiURL, "ec2", cfg.region, ac)
if err != nil {
return nil, fmt.Errorf("cannot create signed request: %w", err)
}
resp, err := cfg.client.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot perform http request to %q: %w", apiURL, err)
}
return readResponseBody(resp, apiURL)
}
// SignRequest signs request for service access and payloadHash.
func (cfg *Config) SignRequest(req *http.Request, service string, payloadHash string) error {
ac, err := cfg.getFreshAPICredentials()
if err != nil {
return err
}
return signRequestWithTime(req, service, cfg.region, payloadHash, ac, time.Now().UTC())
}
func readResponseBody(resp *http.Response, apiURL string) ([]byte, error) {
data, err := ioutil.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
return nil, fmt.Errorf("cannot read response from %q: %w", apiURL, err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code for %q; got %d; want %d; response body: %q",
apiURL, resp.StatusCode, http.StatusOK, data)
}
return data, nil
}
func getDefaultRegion(client *http.Client) (string, error) {
envRegion := os.Getenv("AWS_REGION")
if envRegion != "" {
return envRegion, nil
}
data, err := getMetadataByPath(client, "dynamic/instance-identity/document")
if err != nil {
return "", err
}
var id IdentityDocument
if err := json.Unmarshal(data, &id); err != nil {
return "", fmt.Errorf("cannot parse identity document: %w", err)
}
return id.Region, nil
}
// IdentityDocument is identity document returned from AWS metadata server.
//
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-identity-documents.html
type IdentityDocument struct {
Region string
}
// getFreshAPICredentials returns fresh EC2 API credentials.
//
// The credentials are refreshed if needed.
func (cfg *Config) getFreshAPICredentials() (*credentials, error) {
cfg.credsLock.Lock()
defer cfg.credsLock.Unlock()
if len(cfg.defaultAccessKey) > 0 && len(cfg.defaultSecretKey) > 0 && len(cfg.roleARN) == 0 {
// There is no need in refreshing statically set api credentials if roleARN isn't set.
return cfg.creds, nil
}
if time.Until(cfg.creds.Expiration) > 10*time.Second {
// credentials aren't expired yet.
return cfg.creds, nil
}
// credentials have been expired. Update them.
ac, err := cfg.getAPICredentials()
if err != nil {
return nil, fmt.Errorf("cannot obtain new EC2 API credentials: %w", err)
}
cfg.creds = ac
return ac, nil
}
// getAPICredentials obtains new EC2 API credentials from instance metadata and role_arn.
func (cfg *Config) getAPICredentials() (*credentials, error) {
acNew := &credentials{
AccessKeyID: cfg.defaultAccessKey,
SecretAccessKey: cfg.defaultSecretKey,
}
if len(cfg.webTokenPath) > 0 {
token, err := ioutil.ReadFile(cfg.webTokenPath)
if err != nil {
return nil, fmt.Errorf("cannot read webToken from path: %q, err: %w", cfg.webTokenPath, err)
}
return cfg.getRoleWebIdentityCredentials(string(token))
}
if ecsMetaURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); len(ecsMetaURI) > 0 {
path := "http://169.254.170.2" + ecsMetaURI
return getECSRoleCredentialsByPath(cfg.client, path)
}
// we need instance credentials if dont have access keys
if len(acNew.AccessKeyID) == 0 && len(acNew.SecretAccessKey) == 0 {
ac, err := getInstanceRoleCredentials(cfg.client)
if err != nil {
return nil, fmt.Errorf("cannot obtain instance role credentials: %w", err)
}
acNew = ac
}
// read credentials from sts api, if role_arn is defined
if len(cfg.roleARN) > 0 {
ac, err := cfg.getRoleARNCredentials(acNew)
if err != nil {
return nil, fmt.Errorf("cannot get credentials for role_arn %q: %w", cfg.roleARN, err)
}
acNew = ac
}
if len(acNew.AccessKeyID) == 0 {
return nil, fmt.Errorf("missing AWS access_key; it may be set via env var AWS_ACCESS_KEY_ID or use instance iam role")
}
if len(acNew.SecretAccessKey) == 0 {
return nil, fmt.Errorf("missing AWS secret_key; it may be set via env var AWS_SECRET_ACCESS_KEY or use instance iam role")
}
return acNew, nil
}
// getECSRoleCredentialsByPath makes request to ecs metadata service
// and retrieves instances credentails
// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
func getECSRoleCredentialsByPath(client *http.Client, path string) (*credentials, error) {
resp, err := client.Get(path)
if err != nil {
return nil, fmt.Errorf("cannot get ECS instance role credentials: %w", err)
}
data, err := readResponseBody(resp, path)
if err != nil {
return nil, err
}
return parseMetadataSecurityCredentials(data)
}
// getInstanceRoleCredentials makes request to local ec2 instance metadata service
// and tries to retrieve credentials from assigned iam role.
//
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
func getInstanceRoleCredentials(client *http.Client) (*credentials, error) {
instanceRoleName, err := getMetadataByPath(client, "meta-data/iam/security-credentials/")
if err != nil {
return nil, fmt.Errorf("cannot get instanceRoleName: %w", err)
}
data, err := getMetadataByPath(client, "meta-data/iam/security-credentials/"+string(instanceRoleName))
if err != nil {
return nil, fmt.Errorf("cannot get security credentails for instanceRoleName %q: %w", instanceRoleName, err)
}
return parseMetadataSecurityCredentials(data)
}
// parseMetadataSecurityCredentials parses apiCredentials from metadata response to http://169.254.169.254/latest/meta-data/iam/security-credentials/*
//
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
func parseMetadataSecurityCredentials(data []byte) (*credentials, error) {
var msc MetadataSecurityCredentials
if err := json.Unmarshal(data, &msc); err != nil {
return nil, fmt.Errorf("cannot parse metadata security credentials from %q: %w", data, err)
}
return &credentials{
AccessKeyID: msc.AccessKeyID,
SecretAccessKey: msc.SecretAccessKey,
Token: msc.Token,
Expiration: msc.Expiration,
}, nil
}
// MetadataSecurityCredentials represents credentials obtained from http://169.254.169.254/latest/meta-data/iam/security-credentials/*
//
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
type MetadataSecurityCredentials struct {
AccessKeyID string `json:"AccessKeyId"`
SecretAccessKey string `json:"SecretAccessKey"`
Token string `json:"Token"`
Expiration time.Time `json:"Expiration"`
}
// getMetadataByPath returns instance metadata by url path
func getMetadataByPath(client *http.Client, apiPath string) ([]byte, error) {
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html
// Obtain session token
sessionTokenURL := "http://169.254.169.254/latest/api/token"
req, err := http.NewRequest("PUT", sessionTokenURL, nil)
if err != nil {
return nil, fmt.Errorf("cannot create request for IMDSv2 session token at url %q: %w", sessionTokenURL, err)
}
req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "60")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot obtain IMDSv2 session token from %q: %w", sessionTokenURL, err)
}
token, err := readResponseBody(resp, sessionTokenURL)
if err != nil {
return nil, fmt.Errorf("cannot read IMDSv2 session token from %q: %w", sessionTokenURL, err)
}
// Use session token in the request.
apiURL := "http://169.254.169.254/latest/" + apiPath
req, err = http.NewRequest("GET", apiURL, nil)
if err != nil {
return nil, fmt.Errorf("cannot create request to %q: %w", apiURL, err)
}
req.Header.Set("X-aws-ec2-metadata-token", string(token))
resp, err = client.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot obtain response for %q: %w", apiURL, err)
}
return readResponseBody(resp, apiURL)
}
// getRoleWebIdentityCredentials obtains credentials fo the given roleARN with webToken.
// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
// aws IRSA for kubernetes.
// https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/
func (cfg *Config) getRoleWebIdentityCredentials(token string) (*credentials, error) {
data, err := cfg.getSTSAPIResponse("AssumeRoleWithWebIdentity", func(apiURL string) (*http.Request, error) {
apiURL += fmt.Sprintf("&WebIdentityToken=%s", url.QueryEscape(token))
return http.NewRequest("GET", apiURL, nil)
})
if err != nil {
return nil, err
}
return parseARNCredentials(data, "AssumeRoleWithWebIdentity")
}
// getSTSAPIResponse makes request to aws sts api with the given cfg and returns temporary credentials with expiration time.
//
// See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
func (cfg *Config) getSTSAPIResponse(action string, reqBuilder func(apiURL string) (*http.Request, error)) ([]byte, error) {
// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Query-Requests.html
apiURL := fmt.Sprintf("%s?Action=%s", cfg.stsEndpoint, action)
apiURL += "&Version=2011-06-15"
apiURL += fmt.Sprintf("&RoleArn=%s", cfg.roleARN)
// we have to provide unique session name for cloudtrail audit
apiURL += "&RoleSessionName=vmagent-ec2-discovery"
req, err := reqBuilder(apiURL)
if err != nil {
return nil, fmt.Errorf("cannot create signed request: %w", err)
}
resp, err := cfg.client.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot perform http request to %q: %w", apiURL, err)
}
return readResponseBody(resp, apiURL)
}
// getRoleARNCredentials obtains credentials fo the given roleARN.
func (cfg *Config) getRoleARNCredentials(creds *credentials) (*credentials, error) {
data, err := cfg.getSTSAPIResponse("AssumeRole", func(apiURL string) (*http.Request, error) {
return newSignedGetRequest(apiURL, "sts", cfg.region, creds)
})
if err != nil {
return nil, err
}
return parseARNCredentials(data, "AssumeRole")
}
// parseARNCredentials parses apiCredentials from AssumeRole response.
//
// See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
func parseARNCredentials(data []byte, role string) (*credentials, error) {
var arr AssumeRoleResponse
if err := xml.Unmarshal(data, &arr); err != nil {
return nil, fmt.Errorf("cannot parse AssumeRoleResponse response from %q: %w", data, err)
}
var cred assumeCredentials
switch role {
case "AssumeRole":
cred = arr.AssumeRoleResult.Credentials
case "AssumeRoleWithWebIdentity":
cred = arr.AssumeRoleWithWebIdentityResult.Credentials
default:
logger.Panicf("BUG: unexpected role: %q", role)
}
return &credentials{
AccessKeyID: cred.AccessKeyID,
SecretAccessKey: cred.SecretAccessKey,
Token: cred.SessionToken,
Expiration: cred.Expiration,
}, nil
}
type assumeCredentials struct {
AccessKeyID string `xml:"AccessKeyId"`
SecretAccessKey string `xml:"SecretAccessKey"`
SessionToken string `xml:"SessionToken"`
Expiration time.Time `xml:"Expiration"`
}
// AssumeRoleResponse represents AssumeRole response
//
// See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
type AssumeRoleResponse struct {
AssumeRoleResult struct {
Credentials assumeCredentials `xml:"Credentials"`
} `xml:"AssumeRoleResult"`
AssumeRoleWithWebIdentityResult struct {
Credentials assumeCredentials `xml:"Credentials"`
} `xml:"AssumeRoleWithWebIdentityResult"`
}
// buildAPIEndpoint creates endpoint for aws api access
func buildAPIEndpoint(customEndpoint, region, service string) string {
// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Query-Requests.html
if len(customEndpoint) == 0 {
return fmt.Sprintf("https://%s.%s.amazonaws.com/", service, region)
}
endpoint := customEndpoint
// endpoint may contain only hostname. Convert it to proper url then.
if !strings.Contains(endpoint, "://") {
endpoint = "https://" + endpoint
}
if !strings.HasSuffix(endpoint, "/") {
endpoint += "/"
}
return endpoint
}

10
lib/promscrape/discovery/ec2/api_test.go → lib/awsapi/config_test.go
View file

@ -1,4 +1,4 @@
package ec2
package awsapi
import (
"fmt"
@ -37,7 +37,7 @@ func TestParseMetadataSecurityCredentialsSuccess(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
credsExpected := &apiCredentials{
credsExpected := &credentials{
AccessKeyID: "ASIAIOSFODNN7EXAMPLE",
SecretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
Token: "token",
@ -64,7 +64,7 @@ func TestParseARNCredentialsFailure(t *testing.T) {
}
func TestParseARNCredentialsSuccess(t *testing.T) {
f := func(data, role string, credsExpected *apiCredentials) {
f := func(data, role string, credsExpected *credentials) {
t.Helper()
creds, err := parseARNCredentials([]byte(data), role)
if err != nil {
@ -102,7 +102,7 @@ func TestParseARNCredentialsSuccess(t *testing.T) {
</ResponseMetadata>
</AssumeRoleResponse>
`
credsExpected := &apiCredentials{
credsExpected := &credentials{
AccessKeyID: "ASIAIOSFODNN7EXAMPLE",
SecretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY",
Token: `
@ -134,7 +134,7 @@ func TestParseARNCredentialsSuccess(t *testing.T) {
<RequestId>1214124-7bb0-4673-ad6d-af9e67fc1141</RequestId>
</ResponseMetadata>
</AssumeRoleWithWebIdentityResponse>`
credsExpected2 := &apiCredentials{
credsExpected2 := &credentials{
AccessKeyID: "ASIABYASSDASF",
SecretAccessKey: "asffasfasf/RvxIQpCid4iRMGm56nnRs2oKgV",
Token: "asfafsassssssssss/MlyKUPOYAiEAq5HgS19Mf8SJ3kIKU3NCztDeZW5EUW4NrPrPyXQ8om0q/AQIjv//////////",

46
lib/promscrape/discovery/ec2/sign.go → lib/awsapi/sign.go
View file

@ -1,4 +1,4 @@
package ec2
package awsapi
import (
"crypto/hmac"
@ -6,37 +6,46 @@ import (
"encoding/hex"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
// newSignedRequest signed request for apiURL according to aws signature algorithm.
// for get requests there is no need to calculate payload hash each time.
var emptyPayloadHash = hashHex("")
// newSignedGetRequest creates signed http get request for apiURL according to aws signature algorithm.
//
// See the algorithm at https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
func newSignedRequest(apiURL, service, region string, creds *apiCredentials) (*http.Request, error) {
t := time.Now().UTC()
return newSignedRequestWithTime(apiURL, service, region, creds, t)
func newSignedGetRequest(apiURL, service, region string, creds *credentials) (*http.Request, error) {
return newSignedGetRequestWithTime(apiURL, service, region, creds, time.Now().UTC())
}
func newSignedRequestWithTime(apiURL, service, region string, creds *apiCredentials, t time.Time) (*http.Request, error) {
uri, err := url.Parse(apiURL)
func newSignedGetRequestWithTime(apiURL, service, region string, creds *credentials, t time.Time) (*http.Request, error) {
req, err := http.NewRequest("GET", apiURL, nil)
if err != nil {
return nil, fmt.Errorf("cannot parse %q: %w", apiURL, err)
return nil, fmt.Errorf("cannot create http request with given apiURL: %s, err: %w", apiURL, err)
}
if err := signRequestWithTime(req, service, region, emptyPayloadHash, creds, t); err != nil {
return nil, err
}
return req, nil
}
// signRequestWithTime - signs http request with AWS API credentials for given payload
func signRequestWithTime(req *http.Request, service, region, payloadHash string, creds *credentials, t time.Time) error {
uri := req.URL
// Create canonicalRequest
amzdate := t.Format("20060102T150405Z")
datestamp := t.Format("20060102")
canonicalURL := uri.Path
canonicalQS := uri.Query().Encode()
canonicalHeaders := fmt.Sprintf("host:%s\nx-amz-date:%s\n", uri.Host, amzdate)
signedHeaders := "host;x-amz-date"
payloadHash := hashHex("")
tmp := []string{
"GET",
req.Method,
canonicalURL,
canonicalQS,
canonicalHeaders,
@ -45,7 +54,6 @@ func newSignedRequestWithTime(apiURL, service, region string, creds *apiCredenti
}
canonicalRequest := strings.Join(tmp, "\n")
// Create stringToSign
algorithm := "AWS4-HMAC-SHA256"
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
tmp = []string{
@ -63,16 +71,13 @@ func newSignedRequestWithTime(apiURL, service, region string, creds *apiCredenti
// Calculate autheader
authHeader := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", algorithm, creds.AccessKeyID, credentialScope, signedHeaders, signature)
req, err := http.NewRequest("GET", apiURL, nil)
if err != nil {
return nil, fmt.Errorf("cannot create request from %q: %w", apiURL, err)
}
req.Header.Set("x-amz-date", amzdate)
req.Header.Set("Authorization", authHeader)
// special case for token auth
if creds.Token != "" {
req.Header.Set("X-Amz-Security-Token", creds.Token)
}
return req, nil
return nil
}
func getSignatureKey(key, datestamp, region, service string) string {
@ -83,7 +88,12 @@ func getSignatureKey(key, datestamp, region, service string) string {
}
func hashHex(s string) string {
h := sha256.Sum256([]byte(s))
return HashHex([]byte(s))
}
// HashHex hashes given s
func HashHex(s []byte) string {
h := sha256.Sum256(s)
return hex.EncodeToString(h[:])
}

6
lib/promscrape/discovery/ec2/sign_test.go → lib/awsapi/sign_test.go
View file

@ -1,4 +1,4 @@
package ec2
package awsapi
import (
"testing"
@ -10,12 +10,12 @@ func TestNewSignedRequest(t *testing.T) {
t.Helper()
service := "ec2"
region := "us-east-1"
ac := &apiCredentials{
ac := &credentials{
AccessKeyID: "fake-access-key",
SecretAccessKey: "foobar",
}
ct := time.Unix(0, 0).UTC()
req, err := newSignedRequestWithTime(apiURL, service, region, ac, ct)
req, err := newSignedGetRequestWithTime(apiURL, service, region, ac, ct)
if err != nil {
t.Fatalf("error in newSignedRequest: %s", err)
}

11
lib/promscrape/client.go
View file

@ -201,6 +201,15 @@ func (c *client) GetStreamReader() (*streamReader, error) {
}, nil
}
// checks fasthttp status code for redirect as standard http/client does.
func isStatusRedirect(statusCode int) bool {
switch statusCode {
case 301, 302, 303, 307, 308:
return true
}
return false
}
func (c *client) ReadData(dst []byte) ([]byte, error) {
deadline := time.Now().Add(c.hc.ReadTimeout)
req := fasthttp.AcquireRequest()
@ -237,7 +246,7 @@ func (c *client) ReadData(dst []byte) ([]byte, error) {
err := doRequestWithPossibleRetry(c.hc, req, resp, deadline)
statusCode := resp.StatusCode()
redirectsCount := 0
for err == nil && (statusCode == fasthttp.StatusMovedPermanently || statusCode == fasthttp.StatusFound) {
for err == nil && isStatusRedirect(statusCode) {
if redirectsCount > 5 {
err = fmt.Errorf("too many redirects")
break

413
lib/promscrape/discovery/ec2/api.go
View file

@ -1,52 +1,24 @@
package ec2
import (
"encoding/json"
"encoding/xml"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
type apiConfig struct {
region string
roleARN string
webTokenPath string
filters string
port int
ec2Endpoint string
stsEndpoint string
// these keys are needed for obtaining creds.
defaultAccessKey string
defaultSecretKey string
// Real credentials used for accessing EC2 API.
creds *apiCredentials
credsLock sync.Mutex
awsConfig *awsapi.Config
port int
// A map from AZ name to AZ id.
azMap map[string]string
azMapLock sync.Mutex
}
// apiCredentials represents aws api credentials
type apiCredentials struct {
AccessKeyID string
SecretAccessKey string
Token string
Expiration time.Time
}
var configMap = discoveryutils.NewConfigMap()
func getAPIConfig(sdc *SDConfig) (*apiConfig, error) {
@ -58,47 +30,18 @@ func getAPIConfig(sdc *SDConfig) (*apiConfig, error) {
}
func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
region := sdc.Region
if len(region) == 0 {
r, err := getDefaultRegion()
if err != nil {
return nil, fmt.Errorf("cannot determine default ec2 region; probably, `region` param in `ec2_sd_configs` is missing; the error: %w", err)
}
region = r
}
filters := getFiltersQueryString(sdc.Filters)
fqs := getFiltersQueryString(sdc.Filters)
port := 80
if sdc.Port != nil {
port = *sdc.Port
}
awsCfg, err := awsapi.NewConfig(sdc.Region, sdc.RoleARN, sdc.AccessKey, sdc.SecretKey.String(), fqs)
if err != nil {
return nil, err
}
cfg := &apiConfig{
region: region,
roleARN: sdc.RoleARN,
filters: filters,
port: port,
}
cfg.ec2Endpoint = buildAPIEndpoint(sdc.Endpoint, region, "ec2")
cfg.stsEndpoint = buildAPIEndpoint(sdc.Endpoint, region, "sts")
if cfg.roleARN == "" {
cfg.roleARN = os.Getenv("AWS_ROLE_ARN")
}
cfg.webTokenPath = os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
if cfg.webTokenPath != "" && cfg.roleARN == "" {
return nil, fmt.Errorf("roleARN is missing for AWS_WEB_IDENTITY_TOKEN_FILE=%q, set it either in `ec2_sd_config` or via env var AWS_ROLE_ARN", cfg.webTokenPath)
}
// explicitly set credentials has priority over env variables
cfg.defaultAccessKey = os.Getenv("AWS_ACCESS_KEY_ID")
cfg.defaultSecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
if len(sdc.AccessKey) > 0 {
cfg.defaultAccessKey = sdc.AccessKey
}
if sdc.SecretKey != nil {
cfg.defaultSecretKey = sdc.SecretKey.String()
}
cfg.creds = &apiCredentials{
AccessKeyID: cfg.defaultAccessKey,
SecretAccessKey: cfg.defaultSecretKey,
awsConfig: awsCfg,
port: port,
}
return cfg, nil
}
@ -114,339 +57,3 @@ func getFiltersQueryString(filters []Filter) string {
}
return strings.Join(args, "&")
}
func getDefaultRegion() (string, error) {
envRegion := os.Getenv("AWS_REGION")
if envRegion != "" {
return envRegion, nil
}
data, err := getMetadataByPath("dynamic/instance-identity/document")
if err != nil {
return "", err
}
var id IdentityDocument
if err := json.Unmarshal(data, &id); err != nil {
return "", fmt.Errorf("cannot parse identity document: %w", err)
}
return id.Region, nil
}
// IdentityDocument is identity document returned from AWS metadata server.
//
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-identity-documents.html
type IdentityDocument struct {
Region string
}
// getFreshAPICredentials returns fresh EC2 API credentials.
//
// The credentials are refreshed if needed.
func (cfg *apiConfig) getFreshAPICredentials() (*apiCredentials, error) {
cfg.credsLock.Lock()
defer cfg.credsLock.Unlock()
if len(cfg.defaultAccessKey) > 0 && len(cfg.defaultSecretKey) > 0 && len(cfg.roleARN) == 0 {
// There is no need in refreshing statically set api credentials if `role_arn` isn't set.
return cfg.creds, nil
}
if time.Until(cfg.creds.Expiration) > 10*time.Second {
// Credentials aren't expired yet.
return cfg.creds, nil
}
// Credentials have been expired. Update them.
ac, err := getAPICredentials(cfg)
if err != nil {
return nil, err
}
cfg.creds = ac
return ac, nil
}
// getAPICredentials obtains new EC2 API credentials from instance metadata and role_arn.
func getAPICredentials(cfg *apiConfig) (*apiCredentials, error) {
acNew := &apiCredentials{
AccessKeyID: cfg.defaultAccessKey,
SecretAccessKey: cfg.defaultSecretKey,
}
if len(cfg.webTokenPath) > 0 {
token, err := ioutil.ReadFile(cfg.webTokenPath)
if err != nil {
return nil, fmt.Errorf("cannot read webToken from path: %q, err: %w", cfg.webTokenPath, err)
}
return getRoleWebIdentityCredentials(cfg.stsEndpoint, cfg.roleARN, string(token))
}
if ecsMetaURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); len(ecsMetaURI) > 0 {
path := "http://169.254.170.2" + ecsMetaURI
return getECSRoleCredentialsByPath(path)
}
// we need instance credentials if dont have access keys
if len(acNew.AccessKeyID) == 0 && len(acNew.SecretAccessKey) == 0 {
ac, err := getInstanceRoleCredentials()
if err != nil {
return nil, err
}
acNew = ac
}
// read credentials from sts api, if role_arn is defined
if len(cfg.roleARN) > 0 {
ac, err := getRoleARNCredentials(cfg.region, cfg.stsEndpoint, cfg.roleARN, acNew)
if err != nil {
return nil, fmt.Errorf("cannot get credentials for role_arn %q: %w", cfg.roleARN, err)
}
acNew = ac
}
if len(acNew.AccessKeyID) == 0 {
return nil, fmt.Errorf("missing `access_key`, you can set it with env var AWS_ACCESS_KEY_ID, " +
"directly at `ec2_sd_config` as `access_key` or use instance iam role")
}
if len(acNew.SecretAccessKey) == 0 {
return nil, fmt.Errorf("missing `secret_key`, you can set it with env var AWS_SECRET_ACCESS_KEY," +
"directly at `ec2_sd_config` as `secret_key` or use instance iam role")
}
return acNew, nil
}
// getECSRoleCredentialsByPath makes request to ecs metadata service
// and retrieves instances credentails
// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
func getECSRoleCredentialsByPath(path string) (*apiCredentials, error) {
client := discoveryutils.GetHTTPClient()
resp, err := client.Get(path)
if err != nil {
return nil, fmt.Errorf("cannot get ECS instance role credentials: %w", err)
}
data, err := readResponseBody(resp, path)
if err != nil {
return nil, err
}
return parseMetadataSecurityCredentials(data)
}
// getInstanceRoleCredentials makes request to local ec2 instance metadata service
// and tries to retrieve credentials from assigned iam role.
//
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
func getInstanceRoleCredentials() (*apiCredentials, error) {
instanceRoleName, err := getMetadataByPath("meta-data/iam/security-credentials/")
if err != nil {
return nil, fmt.Errorf("cannot get instanceRoleName: %w", err)
}
data, err := getMetadataByPath("meta-data/iam/security-credentials/" + string(instanceRoleName))
if err != nil {
return nil, fmt.Errorf("cannot get security credentails for instanceRoleName %q: %w", instanceRoleName, err)
}
return parseMetadataSecurityCredentials(data)
}
// parseMetadataSecurityCredentials parses apiCredentials from metadata response to http://169.254.169.254/latest/meta-data/iam/security-credentials/*
//
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
func parseMetadataSecurityCredentials(data []byte) (*apiCredentials, error) {
var msc MetadataSecurityCredentials
if err := json.Unmarshal(data, &msc); err != nil {
return nil, fmt.Errorf("cannot parse metadata security credentials from %q: %w", data, err)
}
return &apiCredentials{
AccessKeyID: msc.AccessKeyID,
SecretAccessKey: msc.SecretAccessKey,
Token: msc.Token,
Expiration: msc.Expiration,
}, nil
}
// MetadataSecurityCredentials represents credentials obtained from http://169.254.169.254/latest/meta-data/iam/security-credentials/*
//
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
type MetadataSecurityCredentials struct {
AccessKeyID string `json:"AccessKeyId"`
SecretAccessKey string `json:"SecretAccessKey"`
Token string `json:"Token"`
Expiration time.Time `json:"Expiration"`
}
// getMetadataByPath returns instance metadata by url path
func getMetadataByPath(apiPath string) ([]byte, error) {
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html
client := discoveryutils.GetHTTPClient()
// Obtain session token
sessionTokenURL := "http://169.254.169.254/latest/api/token"
req, err := http.NewRequest("PUT", sessionTokenURL, nil)
if err != nil {
return nil, fmt.Errorf("cannot create request for IMDSv2 session token at url %q: %w", sessionTokenURL, err)
}
req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "60")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot obtain IMDSv2 session token from %q; probably, `region` is missing in `ec2_sd_config`; error: %w", sessionTokenURL, err)
}
token, err := readResponseBody(resp, sessionTokenURL)
if err != nil {
return nil, fmt.Errorf("cannot read IMDSv2 session token from %q; probably, `region` is missing in `ec2_sd_config`; error: %w", sessionTokenURL, err)
}
// Use session token in the request.
apiURL := "http://169.254.169.254/latest/" + apiPath
req, err = http.NewRequest("GET", apiURL, nil)
if err != nil {
return nil, fmt.Errorf("cannot create request to %q: %w", apiURL, err)
}
req.Header.Set("X-aws-ec2-metadata-token", string(token))
resp, err = client.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot obtain response for %q: %w", apiURL, err)
}
return readResponseBody(resp, apiURL)
}
// getRoleWebIdentityCredentials obtains credentials fo the given roleARN with webToken.
// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
// aws IRSA for kubernetes.
// https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/
func getRoleWebIdentityCredentials(stsEndpoint, roleARN string, token string) (*apiCredentials, error) {
data, err := getSTSAPIResponse("AssumeRoleWithWebIdentity", stsEndpoint, roleARN, func(apiURL string) (*http.Request, error) {
apiURL += fmt.Sprintf("&WebIdentityToken=%s", url.QueryEscape(token))
return http.NewRequest("GET", apiURL, nil)
})
if err != nil {
return nil, err
}
return parseARNCredentials(data, "AssumeRoleWithWebIdentity")
}
// getRoleARNCredentials obtains credentials fo the given roleARN.
func getRoleARNCredentials(region, stsEndpoint, roleARN string, creds *apiCredentials) (*apiCredentials, error) {
data, err := getSTSAPIResponse("AssumeRole", stsEndpoint, roleARN, func(apiURL string) (*http.Request, error) {
return newSignedRequest(apiURL, "sts", region, creds)
})
if err != nil {
return nil, err
}
return parseARNCredentials(data, "AssumeRole")
}
// parseARNCredentials parses apiCredentials from AssumeRole response.
//
// See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
func parseARNCredentials(data []byte, role string) (*apiCredentials, error) {
var arr AssumeRoleResponse
if err := xml.Unmarshal(data, &arr); err != nil {
return nil, fmt.Errorf("cannot parse AssumeRoleResponse response from %q: %w", data, err)
}
var cred assumeCredentials
switch role {
case "AssumeRole":
cred = arr.AssumeRoleResult.Credentials
case "AssumeRoleWithWebIdentity":
cred = arr.AssumeRoleWithWebIdentityResult.Credentials
default:
logger.Panicf("BUG: unexpected role: %q", role)
}
return &apiCredentials{
AccessKeyID: cred.AccessKeyID,
SecretAccessKey: cred.SecretAccessKey,
Token: cred.SessionToken,
Expiration: cred.Expiration,
}, nil
}
type assumeCredentials struct {
AccessKeyID string `xml:"AccessKeyId"`
SecretAccessKey string `xml:"SecretAccessKey"`
SessionToken string `xml:"SessionToken"`
Expiration time.Time `xml:"Expiration"`
}
// AssumeRoleResponse represents AssumeRole response
//
// See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
type AssumeRoleResponse struct {
AssumeRoleResult struct {
Credentials assumeCredentials
}
AssumeRoleWithWebIdentityResult struct {
Credentials assumeCredentials
}
}
// buildAPIEndpoint creates endpoint for aws api access
func buildAPIEndpoint(customEndpoint, region, service string) string {
// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Query-Requests.html
if len(customEndpoint) == 0 {
return fmt.Sprintf("https://%s.%s.amazonaws.com/", service, region)
}
endpoint := customEndpoint
// endpoint may contain only hostname. Convert it to proper url then.
if !strings.Contains(endpoint, "://") {
endpoint = "https://" + endpoint
}
if !strings.HasSuffix(endpoint, "/") {
endpoint += "/"
}
return endpoint
}
// getSTSAPIResponse makes request to aws sts api with roleARN
// and returns temporary credentials with expiration time
//
// See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
func getSTSAPIResponse(action, stsEndpoint, roleARN string, reqBuilder func(apiURL string) (*http.Request, error)) ([]byte, error) {
// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Query-Requests.html
apiURL := fmt.Sprintf("%s?Action=%s", stsEndpoint, action)
apiURL += "&Version=2011-06-15"
apiURL += fmt.Sprintf("&RoleArn=%s", roleARN)
// we have to provide unique session name for cloudtrail audit
apiURL += "&RoleSessionName=vmagent-ec2-discovery"
req, err := reqBuilder(apiURL)
if err != nil {
return nil, fmt.Errorf("cannot create signed request: %w", err)
}
resp, err := discoveryutils.GetHTTPClient().Do(req)
if err != nil {
return nil, fmt.Errorf("cannot perform http request to %q: %w", apiURL, err)
}
return readResponseBody(resp, apiURL)
}
// getEC2APIResponse performs EC2 API request with given action.
func getEC2APIResponse(cfg *apiConfig, action, filters, nextPageToken string) ([]byte, error) {
ac, err := cfg.getFreshAPICredentials()
if err != nil {
return nil, fmt.Errorf("cannot obtain fresh credentials for EC2 API: %w", err)
}
apiURL := fmt.Sprintf("%s?Action=%s", cfg.ec2Endpoint, url.QueryEscape(action))
if len(filters) > 0 {
apiURL += "&" + filters
}
if len(nextPageToken) > 0 {
apiURL += fmt.Sprintf("&NextToken=%s", url.QueryEscape(nextPageToken))
}
apiURL += "&Version=2013-10-15"
req, err := newSignedRequest(apiURL, "ec2", cfg.region, ac)
if err != nil {
return nil, fmt.Errorf("cannot create signed request: %w", err)
}
resp, err := discoveryutils.GetHTTPClient().Do(req)
if err != nil {
return nil, fmt.Errorf("cannot perform http request to %q: %w", apiURL, err)
}
return readResponseBody(resp, apiURL)
}
func readResponseBody(resp *http.Response, apiURL string) ([]byte, error) {
data, err := ioutil.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
return nil, fmt.Errorf("cannot read response from %q: %w", apiURL, err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code for %q; got %d; want %d; response body: %q",
apiURL, resp.StatusCode, http.StatusOK, data)
}
return data, nil
}

65
lib/promscrape/discovery/ec2/az.go Normal file
View file

@ -0,0 +1,65 @@
package ec2
import (
"encoding/xml"
"fmt"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
func getAZMap(cfg *apiConfig) map[string]string {
cfg.azMapLock.Lock()
defer cfg.azMapLock.Unlock()
if cfg.azMap != nil {
return cfg.azMap
}
azs, err := getAvailabilityZones(cfg)
cfg.azMap = make(map[string]string, len(azs))
if err != nil {
logger.Warnf("couldn't load availability zones map, so __meta_ec2_availability_zone_id label isn't set: %s", err)
return cfg.azMap
}
for _, az := range azs {
cfg.azMap[az.ZoneName] = az.ZoneID
}
return cfg.azMap
}
func getAvailabilityZones(cfg *apiConfig) ([]AvailabilityZone, error) {
// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html
data, err := cfg.awsConfig.GetEC2APIResponse("DescribeAvailabilityZones", "")
if err != nil {
return nil, fmt.Errorf("cannot obtain availability zones: %w", err)
}
azr, err := parseAvailabilityZonesResponse(data)
if err != nil {
return nil, fmt.Errorf("cannot parse availability zones list: %w", err)
}
return azr.AvailabilityZoneInfo.Items, nil
}
// AvailabilityZonesResponse represents the response for https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html
type AvailabilityZonesResponse struct {
AvailabilityZoneInfo AvailabilityZoneInfo `xml:"availabilityZoneInfo"`
}
// AvailabilityZoneInfo represents availabilityZoneInfo for https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html
type AvailabilityZoneInfo struct {
Items []AvailabilityZone `xml:"item"`
}
// AvailabilityZone represents availabilityZone for https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_AvailabilityZone.html
type AvailabilityZone struct {
ZoneName string `xml:"zoneName"`
ZoneID string `xml:"zoneId"`
}
func parseAvailabilityZonesResponse(data []byte) (*AvailabilityZonesResponse, error) {
var v AvailabilityZonesResponse
if err := xml.Unmarshal(data, &v); err != nil {
return nil, fmt.Errorf("cannot unmarshal DescribeAvailabilityZonesResponse from %q: %w", data, err)
}
return &v, nil
}

60
lib/promscrape/discovery/ec2/instance.go
View file

@ -5,7 +5,6 @@ import (
"fmt"
"strings"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
@ -30,7 +29,7 @@ func getReservations(cfg *apiConfig) ([]Reservation, error) {
var rs []Reservation
pageToken := ""
for {
data, err := getEC2APIResponse(cfg, "DescribeInstances", cfg.filters, pageToken)
data, err := cfg.awsConfig.GetEC2APIResponse("DescribeInstances", pageToken)
if err != nil {
return nil, fmt.Errorf("cannot obtain instances: %w", err)
}
@ -133,63 +132,6 @@ func parseInstancesResponse(data []byte) (*InstancesResponse, error) {
return &v, nil
}
func getAZMap(cfg *apiConfig) map[string]string {
cfg.azMapLock.Lock()
defer cfg.azMapLock.Unlock()
if cfg.azMap != nil {
return cfg.azMap
}
azs, err := getAvailabilityZones(cfg)
cfg.azMap = make(map[string]string, len(azs))
if err != nil {
logger.Warnf("couldn't load availability zones map, so __meta_ec2_availability_zone_id label isn't set: %s", err)
return cfg.azMap
}
for _, az := range azs {
cfg.azMap[az.ZoneName] = az.ZoneID
}
return cfg.azMap
}
func getAvailabilityZones(cfg *apiConfig) ([]AvailabilityZone, error) {
// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html
data, err := getEC2APIResponse(cfg, "DescribeAvailabilityZones", "", "")
if err != nil {
return nil, fmt.Errorf("cannot obtain availability zones: %w", err)
}
azr, err := parseAvailabilityZonesResponse(data)
if err != nil {
return nil, fmt.Errorf("cannot parse availability zones list: %w", err)
}
return azr.AvailabilityZoneInfo.Items, nil
}
// AvailabilityZonesResponse represents the response for https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html
type AvailabilityZonesResponse struct {
AvailabilityZoneInfo AvailabilityZoneInfo `xml:"availabilityZoneInfo"`
}
// AvailabilityZoneInfo represents availabilityZoneInfo for https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html
type AvailabilityZoneInfo struct {
Items []AvailabilityZone `xml:"item"`
}
// AvailabilityZone represents availabilityZone for https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_AvailabilityZone.html
type AvailabilityZone struct {
ZoneName string `xml:"zoneName"`
ZoneID string `xml:"zoneId"`
}
func parseAvailabilityZonesResponse(data []byte) (*AvailabilityZonesResponse, error) {
var v AvailabilityZonesResponse
if err := xml.Unmarshal(data, &v); err != nil {
return nil, fmt.Errorf("cannot unmarshal DescribeAvailabilityZonesResponse from %q: %w", data, err)
}
return &v, nil
}
func (inst *Instance) appendTargetLabels(ms []map[string]string, ownerID string, port int, azMap map[string]string) []map[string]string {
if len(inst.PrivateIPAddress) == 0 {
// Cannot scrape instance without private IP address

45
app/vmbackup/snapshot/snapshot.go → lib/snapshot/snapshot.go
View file

@ -7,18 +7,23 @@ import (
"io/ioutil"
"net/http"
"net/url"
"regexp"
"strings"
"sync/atomic"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
var snapshotNameRegexp = regexp.MustCompile(`^[0-9]{14}-[0-9A-Fa-f]+$`)
type snapshot struct {
Status string `json:"status"`
Snapshot string `json:"snapshot"`
Msg string `json:"msg"`
}
// Create creates a snapshot and the provided api endpoint and returns
// the snapshot name
// Create creates a snapshot via the provided api endpoint and returns the snapshot name
func Create(createSnapshotURL string) (string, error) {
logger.Infof("Creating snapshot")
u, err := url.Parse(createSnapshotURL)
@ -53,7 +58,7 @@ func Create(createSnapshotURL string) (string, error) {
}
}
// Delete deletes a snapshot and the provided api endpoint returns any failure
// Delete deletes a snapshot via the provided api endpoint
func Delete(deleteSnapshotURL string, snapshotName string) error {
logger.Infof("Deleting snapshot %s", snapshotName)
formData := url.Values{
@ -90,3 +95,37 @@ func Delete(deleteSnapshotURL string, snapshotName string) error {
return fmt.Errorf("Unkown status: %v", snap.Status)
}
}
// Validate validates the snapshotName
func Validate(snapshotName string) error {
_, err := Time(snapshotName)
return err
}
// Time returns snapshot creation time from the given snapshotName
func Time(snapshotName string) (time.Time, error) {
if !snapshotNameRegexp.MatchString(snapshotName) {
return time.Time{}, fmt.Errorf("unexpected snapshot name=%q; it must match %q regexp", snapshotName, snapshotNameRegexp.String())
}
n := strings.IndexByte(snapshotName, '-')
if n < 0 {
logger.Panicf("BUG: cannot find `-` in snapshotName=%q", snapshotName)
}
s := snapshotName[:n]
t, err := time.Parse("20060102150405", s)
if err != nil {
return time.Time{}, fmt.Errorf("unexpected timestamp=%q in snapshot name: %w; it must match YYYYMMDDhhmmss pattern", s, err)
}
return t, nil
}
// NewName returns new name for new snapshot
func NewName() string {
return fmt.Sprintf("%s-%08X", time.Now().UTC().Format("20060102150405"), nextSnapshotIdx())
}
func nextSnapshotIdx() uint64 {
return atomic.AddUint64(&snapshotIdx, 1)
}
var snapshotIdx = uint64(time.Now().UnixNano())

51
app/vmbackup/snapshot/snapshot_test.go → lib/snapshot/snapshot_test.go
View file

@ -104,3 +104,54 @@ func TestDeleteSnapshotFailed(t *testing.T) {
t.Fatalf("Snapshot should have failed, got: %v", err)
}
}
func Test_Validate(t *testing.T) {
tests := []struct {
name string
snapshotName string
want bool
}{
{
name: "empty snapshot name",
snapshotName: "",
want: false,
},
{
name: "short snapshot name",
snapshotName: "",
want: false,
},
{
name: "short first part of the snapshot name",
snapshotName: "2022050312163-16EB56ADB4110CF2",
want: false,
},
{
name: "short second part of the snapshot name",
snapshotName: "20220503121638-16EB56ADB4110CF",
want: true,
},
{
name: "correct snapshot name",
snapshotName: "20220503121638-16EB56ADB4110CF2",
want: true,
},
{
name: "invalid time part snapshot name",
snapshotName: "00000000000000-16EB56ADB4110CF2",
want: false,
},
{
name: "not enough parts of the snapshot name",
snapshotName: "2022050312163816EB56ADB4110CF2",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := Validate(tt.snapshotName); (err == nil) != tt.want {
t.Errorf("checkSnapshotName() = %v, want %v", err, tt.want)
}
})
}
}

31
lib/storage/storage.go
View file

@ -24,6 +24,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
@ -317,7 +318,7 @@ func (s *Storage) CreateSnapshot() (string, error) {
s.snapshotLock.Lock()
defer s.snapshotLock.Unlock()
snapshotName := fmt.Sprintf("%s-%08X", time.Now().UTC().Format("20060102150405"), nextSnapshotIdx())
snapshotName := snapshot.NewName()
srcDir := s.path
dstDir := fmt.Sprintf("%s/snapshots/%s", srcDir, snapshotName)
if err := fs.MkdirAllFailIfExist(dstDir); err != nil {
@ -372,8 +373,6 @@ func (s *Storage) CreateSnapshot() (string, error) {
return snapshotName, nil
}
var snapshotNameRegexp = regexp.MustCompile("^[0-9]{14}-[0-9A-Fa-f]+$")
// ListSnapshots returns sorted list of existing snapshots for s.
func (s *Storage) ListSnapshots() ([]string, error) {
snapshotsPath := s.path + "/snapshots"
@ -389,7 +388,7 @@ func (s *Storage) ListSnapshots() ([]string, error) {
}
snapshotNames := make([]string, 0, len(fnames))
for _, fname := range fnames {
if !snapshotNameRegexp.MatchString(fname) {
if err := snapshot.Validate(fname); err != nil {
continue
}
snapshotNames = append(snapshotNames, fname)
@ -400,8 +399,8 @@ func (s *Storage) ListSnapshots() ([]string, error) {
// DeleteSnapshot deletes the given snapshot.
func (s *Storage) DeleteSnapshot(snapshotName string) error {
if !snapshotNameRegexp.MatchString(snapshotName) {
return fmt.Errorf("invalid snapshotName %q", snapshotName)
if err := snapshot.Validate(snapshotName); err != nil {
return fmt.Errorf("invalid snapshotName %q: %w", snapshotName, err)
}
snapshotPath := s.path + "/snapshots/" + snapshotName
@ -426,7 +425,7 @@ func (s *Storage) DeleteStaleSnapshots(maxAge time.Duration) error {
}
expireDeadline := time.Now().UTC().Add(-maxAge)
for _, snapshotName := range list {
t, err := snapshotTime(snapshotName)
t, err := snapshot.Time(snapshotName)
if err != nil {
return fmt.Errorf("cannot parse snapshot date from %q: %w", snapshotName, err)
}
@ -439,24 +438,6 @@ func (s *Storage) DeleteStaleSnapshots(maxAge time.Duration) error {
return nil
}
func snapshotTime(snapshotName string) (time.Time, error) {
if !snapshotNameRegexp.MatchString(snapshotName) {
return time.Time{}, fmt.Errorf("unexpected snapshotName must be in the format `YYYYMMDDhhmmss-idx`; got %q", snapshotName)
}
n := strings.IndexByte(snapshotName, '-')
if n < 0 {
return time.Time{}, fmt.Errorf("cannot find `-` in snapshotName=%q", snapshotName)
}
s := snapshotName[:n]
return time.Parse("20060102150405", s)
}
var snapshotIdx = uint64(time.Now().UnixNano())
func nextSnapshotIdx() uint64 {
return atomic.AddUint64(&snapshotIdx, 1)
}
func (s *Storage) idb() *indexDB {
return s.idbCurr.Load().(*indexDB)
}

6
vendor/cloud.google.com/go/.release-please-manifest-submodules.json generated vendored
View file

@ -14,11 +14,11 @@
"billing": "1.2.0",
"binaryauthorization": "0.4.0",
"certificatemanager": "0.2.0",
"channel": "1.5.0",
"channel": "1.6.0",
"cloudbuild": "1.2.0",
"clouddms": "1.2.0",
"cloudtasks": "1.3.0",
"compute": "1.6.0",
"compute": "1.6.1",
"contactcenterinsights": "1.2.0",
"container": "1.2.0",
"containeranalysis": "0.3.0",
@ -68,7 +68,7 @@
"phishingprotection": "0.3.0",
"policytroubleshooter": "1.2.0",
"privatecatalog": "0.3.0",
"recaptchaenterprise": "1.3.0",
"recaptchaenterprise": "1.3.1",
"recommendationengine": "0.2.0",
"recommender": "1.3.0",
"redis": "1.5.0",

2
vendor/cloud.google.com/go/.release-please-manifest.json generated vendored
View file

@ -1,3 +1,3 @@
{
".": "0.101.0"
".": "0.101.1"
}

7
vendor/cloud.google.com/go/CHANGES.md generated vendored
View file

@ -1,5 +1,12 @@
# Changes
### [0.101.1](https://github.com/googleapis/google-cloud-go/compare/v0.101.0...v0.101.1) (2022-05-03)
### Bug Fixes
* **internal/gapicgen:** properly update modules that have no gapic changes ([#5945](https://github.com/googleapis/google-cloud-go/issues/5945)) ([de2befc](https://github.com/googleapis/google-cloud-go/commit/de2befcaa2a886499db9da6d4d04d28398c8d44b))
## [0.101.0](https://github.com/googleapis/google-cloud-go/compare/v0.100.2...v0.101.0) (2022-04-20)

2
vendor/cloud.google.com/go/CONTRIBUTING.md generated vendored
View file

@ -2,7 +2,7 @@
1. [File an issue](https://github.com/googleapis/google-cloud-go/issues/new/choose).
The issue will be used to discuss the bug or feature and should be created
before sending a CL.
before sending a PR.
1. [Install Go](https://golang.org/dl/).
1. Ensure that your `GOBIN` directory (by default `$(go env GOPATH)/bin`)

1
vendor/cloud.google.com/go/README.md generated vendored
View file

@ -35,6 +35,7 @@ For an updated list of all of our released APIs please see our
Our libraries are compatible with at least the three most recent, major Go
releases. They are currently compatible with:
- Go 1.18
- Go 1.17
- Go 1.16
- Go 1.15

300
vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go generated vendored
View file

@ -2226,63 +2226,183 @@ var awsPartition = partition{
endpointKey{
Region: "af-south-1",
}: endpoint{},
endpointKey{
Region: "af-south-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.af-south-1.api.aws",
},
endpointKey{
Region: "ap-east-1",
}: endpoint{},
endpointKey{
Region: "ap-east-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.ap-east-1.api.aws",
},
endpointKey{
Region: "ap-northeast-1",
}: endpoint{},
endpointKey{
Region: "ap-northeast-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.ap-northeast-1.api.aws",
},
endpointKey{
Region: "ap-northeast-2",
}: endpoint{},
endpointKey{
Region: "ap-northeast-2",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.ap-northeast-2.api.aws",
},
endpointKey{
Region: "ap-south-1",
}: endpoint{},
endpointKey{
Region: "ap-south-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.ap-south-1.api.aws",
},
endpointKey{
Region: "ap-southeast-1",
}: endpoint{},
endpointKey{
Region: "ap-southeast-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.ap-southeast-1.api.aws",
},
endpointKey{
Region: "ap-southeast-2",
}: endpoint{},
endpointKey{
Region: "ap-southeast-2",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.ap-southeast-2.api.aws",
},
endpointKey{
Region: "ca-central-1",
}: endpoint{},
endpointKey{
Region: "ca-central-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.ca-central-1.api.aws",
},
endpointKey{
Region: "eu-central-1",
}: endpoint{},
endpointKey{
Region: "eu-central-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.eu-central-1.api.aws",
},
endpointKey{
Region: "eu-north-1",
}: endpoint{},
endpointKey{
Region: "eu-north-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.eu-north-1.api.aws",
},
endpointKey{
Region: "eu-south-1",
}: endpoint{},
endpointKey{
Region: "eu-south-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.eu-south-1.api.aws",
},
endpointKey{
Region: "eu-west-1",
}: endpoint{},
endpointKey{
Region: "eu-west-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.eu-west-1.api.aws",
},
endpointKey{
Region: "eu-west-2",
}: endpoint{},
endpointKey{
Region: "eu-west-2",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.eu-west-2.api.aws",
},
endpointKey{
Region: "eu-west-3",
}: endpoint{},
endpointKey{
Region: "eu-west-3",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.eu-west-3.api.aws",
},
endpointKey{
Region: "me-south-1",
}: endpoint{},
endpointKey{
Region: "me-south-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.me-south-1.api.aws",
},
endpointKey{
Region: "sa-east-1",
}: endpoint{},
endpointKey{
Region: "sa-east-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.sa-east-1.api.aws",
},
endpointKey{
Region: "us-east-1",
}: endpoint{},
endpointKey{
Region: "us-east-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.us-east-1.api.aws",
},
endpointKey{
Region: "us-east-2",
}: endpoint{},
endpointKey{
Region: "us-east-2",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.us-east-2.api.aws",
},
endpointKey{
Region: "us-west-1",
}: endpoint{},
endpointKey{
Region: "us-west-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.us-west-1.api.aws",
},
endpointKey{
Region: "us-west-2",
}: endpoint{},
endpointKey{
Region: "us-west-2",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.us-west-2.api.aws",
},
},
},
"apprunner": service{
@ -7100,6 +7220,9 @@ var awsPartition = partition{
endpointKey{
Region: "ap-southeast-2",
}: endpoint{},
endpointKey{
Region: "ap-southeast-3",
}: endpoint{},
endpointKey{
Region: "ca-central-1",
}: endpoint{},
@ -11321,6 +11444,19 @@ var awsPartition = partition{
}: endpoint{},
},
},
"ivschat": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "eu-west-1",
}: endpoint{},
endpointKey{
Region: "us-east-1",
}: endpoint{},
endpointKey{
Region: "us-west-2",
}: endpoint{},
},
},
"kafka": service{
Endpoints: serviceEndpoints{
endpointKey{
@ -12571,6 +12707,9 @@ var awsPartition = partition{
endpointKey{
Region: "ap-southeast-2",
}: endpoint{},
endpointKey{
Region: "ap-southeast-3",
}: endpoint{},
endpointKey{
Region: "ca-central-1",
}: endpoint{},
@ -13123,6 +13262,52 @@ var awsPartition = partition{
}: endpoint{},
},
},
"media-pipelines-chime": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "ap-southeast-1",
}: endpoint{},
endpointKey{
Region: "eu-central-1",
}: endpoint{},
endpointKey{
Region: "us-east-1",
}: endpoint{},
endpointKey{
Region: "us-east-1",
Variant: fipsVariant,
}: endpoint{
Hostname: "media-pipelines-chime-fips.us-east-1.amazonaws.com",
},
endpointKey{
Region: "us-east-1-fips",
}: endpoint{
Hostname: "media-pipelines-chime-fips.us-east-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-east-1",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "us-west-2",
}: endpoint{},
endpointKey{
Region: "us-west-2",
Variant: fipsVariant,
}: endpoint{
Hostname: "media-pipelines-chime-fips.us-west-2.amazonaws.com",
},
endpointKey{
Region: "us-west-2-fips",
}: endpoint{
Hostname: "media-pipelines-chime-fips.us-west-2.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-west-2",
},
Deprecated: boxedTrue,
},
},
},
"mediaconnect": service{
Endpoints: serviceEndpoints{
endpointKey{
@ -14485,6 +14670,9 @@ var awsPartition = partition{
},
"nimble": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "ap-northeast-1",
}: endpoint{},
endpointKey{
Region: "ap-southeast-2",
}: endpoint{},
@ -16651,6 +16839,9 @@ var awsPartition = partition{
endpointKey{
Region: "ap-southeast-2",
}: endpoint{},
endpointKey{
Region: "ap-southeast-3",
}: endpoint{},
endpointKey{
Region: "ca-central-1",
}: endpoint{},
@ -20719,6 +20910,42 @@ var awsPartition = partition{
endpointKey{
Region: "eu-west-3",
}: endpoint{},
endpointKey{
Region: "fips-us-east-1",
}: endpoint{
Hostname: "synthetics-fips.us-east-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-east-1",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-us-east-2",
}: endpoint{
Hostname: "synthetics-fips.us-east-2.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-east-2",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-us-west-1",
}: endpoint{
Hostname: "synthetics-fips.us-west-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-west-1",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-us-west-2",
}: endpoint{
Hostname: "synthetics-fips.us-west-2.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-west-2",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "me-south-1",
}: endpoint{},
@ -20728,15 +20955,39 @@ var awsPartition = partition{
endpointKey{
Region: "us-east-1",
}: endpoint{},
endpointKey{
Region: "us-east-1",
Variant: fipsVariant,
}: endpoint{
Hostname: "synthetics-fips.us-east-1.amazonaws.com",
},
endpointKey{
Region: "us-east-2",
}: endpoint{},
endpointKey{
Region: "us-east-2",
Variant: fipsVariant,
}: endpoint{
Hostname: "synthetics-fips.us-east-2.amazonaws.com",
},
endpointKey{
Region: "us-west-1",
}: endpoint{},
endpointKey{
Region: "us-west-1",
Variant: fipsVariant,
}: endpoint{
Hostname: "synthetics-fips.us-west-1.amazonaws.com",
},
endpointKey{
Region: "us-west-2",
}: endpoint{},
endpointKey{
Region: "us-west-2",
Variant: fipsVariant,
}: endpoint{
Hostname: "synthetics-fips.us-west-2.amazonaws.com",
},
},
},
"tagging": service{
@ -22584,9 +22835,21 @@ var awscnPartition = partition{
endpointKey{
Region: "cn-north-1",
}: endpoint{},
endpointKey{
Region: "cn-north-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.cn-north-1.api.amazonwebservices.com.cn",
},
endpointKey{
Region: "cn-northwest-1",
}: endpoint{},
endpointKey{
Region: "cn-northwest-1",
Variant: dualStackVariant,
}: endpoint{
Hostname: "appmesh.cn-northwest-1.api.amazonwebservices.com.cn",
},
},
},
"appsync": service{
@ -28531,12 +28794,42 @@ var awsusgovPartition = partition{
},
"synthetics": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "fips-us-gov-east-1",
}: endpoint{
Hostname: "synthetics-fips.us-gov-east-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-gov-east-1",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-us-gov-west-1",
}: endpoint{
Hostname: "synthetics-fips.us-gov-west-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-gov-west-1",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "us-gov-east-1",
}: endpoint{},
endpointKey{
Region: "us-gov-east-1",
Variant: fipsVariant,
}: endpoint{
Hostname: "synthetics-fips.us-gov-east-1.amazonaws.com",
},
endpointKey{
Region: "us-gov-west-1",
}: endpoint{},
endpointKey{
Region: "us-gov-west-1",
Variant: fipsVariant,
}: endpoint{
Hostname: "synthetics-fips.us-gov-west-1.amazonaws.com",
},
},
},
"tagging": service{
@ -29942,6 +30235,13 @@ var awsisobPartition = partition{
}: endpoint{},
},
},
"ram": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "us-isob-east-1",
}: endpoint{},
},
},
"rds": service{
Endpoints: serviceEndpoints{
endpointKey{

2
vendor/github.com/aws/aws-sdk-go/aws/version.go generated vendored
View file

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.44.4"
const SDKVersion = "1.44.7"

27
vendor/github.com/aws/aws-sdk-go/service/s3/api.go generated vendored
View file

@ -8113,8 +8113,9 @@ func (c *S3) PutBucketLifecycleConfigurationRequest(input *PutBucketLifecycleCon
// Rules
//
// You specify the lifecycle configuration in your request body. The lifecycle
// configuration is specified as XML consisting of one or more rules. Each rule
// consists of the following:
// configuration is specified as XML consisting of one or more rules. An Amazon
// S3 Lifecycle configuration can have up to 1,000 rules. This limit is not
// adjustable. Each rule consists of the following:
//
// * Filter identifying a subset of objects to which the rule applies. The
// filter can be based on a key name prefix, object tags, or a combination
@ -10918,9 +10919,11 @@ func (c *S3) UploadPartRequest(input *UploadPartInput) (req *request.Request, ou
// Part numbers can be any number from 1 to 10,000, inclusive. A part number
// uniquely identifies a part and also defines its position within the object
// being created. If you upload a new part using the same part number that was
// used with a previous part, the previously uploaded part is overwritten. Each
// part must be at least 5 MB in size, except the last part. There is no size
// limit on the last part of your multipart upload.
// used with a previous part, the previously uploaded part is overwritten.
//
// For information about maximum and minimum part sizes and other multipart
// upload specifications, see Multipart upload limits (https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html)
// in the Amazon S3 User Guide.
//
// To ensure that data is not corrupted when traversing the network, specify
// the Content-MD5 header in the upload part request. Amazon S3 checks the part
@ -11068,8 +11071,8 @@ func (c *S3) UploadPartCopyRequest(input *UploadPartCopyInput) (req *request.Req
// your request and a byte range by adding the request header x-amz-copy-source-range
// in your request.
//
// The minimum allowable part size for a multipart upload is 5 MB. For more
// information about multipart upload limits, go to Quick Facts (https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html)
// For information about maximum and minimum part sizes and other multipart
// upload specifications, see Multipart upload limits (https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html)
// in the Amazon S3 User Guide.
//
// Instead of using an existing object as part data, you might use the UploadPart
@ -29347,9 +29350,9 @@ type NoncurrentVersionExpiration struct {
NewerNoncurrentVersions *int64 `type:"integer"`
// Specifies the number of days an object is noncurrent before Amazon S3 can
// perform the associated action. For information about the noncurrent days
// calculations, see How Amazon S3 Calculates When an Object Became Noncurrent
// (https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#non-current-days-calculations)
// perform the associated action. The value must be a non-zero positive integer.
// For information about the noncurrent days calculations, see How Amazon S3
// Calculates When an Object Became Noncurrent (https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#non-current-days-calculations)
// in the Amazon S3 User Guide.
NoncurrentDays *int64 `type:"integer"`
}
@ -29662,7 +29665,9 @@ type Object struct {
//
// * If an object is created by either the Multipart Upload or Part Copy
// operation, the ETag is not an MD5 digest, regardless of the method of
// encryption.
// encryption. If an object is larger than 16 MB, the Amazon Web Services
// Management Console will upload or copy that object as a Multipart Upload,
// and therefore the ETag will not be an MD5 digest.
ETag *string `type:"string"`
// The name that you assign to an object. You use the object key to retrieve

2
vendor/google.golang.org/api/internal/version.go generated vendored
View file

@ -5,4 +5,4 @@
package internal
// Version is the current tagged release of the library.
const Version = "0.77.0"
const Version = "0.78.0"

10
vendor/modules.txt vendored
View file

@ -1,4 +1,4 @@
# cloud.google.com/go v0.101.0
# cloud.google.com/go v0.101.1
## explicit; go 1.15
cloud.google.com/go
cloud.google.com/go/internal
@ -34,7 +34,7 @@ github.com/VictoriaMetrics/metricsql/binaryop
# github.com/VividCortex/ewma v1.2.0
## explicit; go 1.12
github.com/VividCortex/ewma
# github.com/aws/aws-sdk-go v1.44.4
# github.com/aws/aws-sdk-go v1.44.7
## explicit; go 1.11
github.com/aws/aws-sdk-go/aws
github.com/aws/aws-sdk-go/aws/arn
@ -295,7 +295,7 @@ golang.org/x/oauth2/jwt
# golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
## explicit
golang.org/x/sync/errgroup
# golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba
# golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6
## explicit; go 1.17
golang.org/x/sys/internal/unsafeheader
golang.org/x/sys/unix
@ -310,7 +310,7 @@ golang.org/x/text/unicode/norm
## explicit; go 1.11
golang.org/x/xerrors
golang.org/x/xerrors/internal
# google.golang.org/api v0.77.0
# google.golang.org/api v0.78.0
## explicit; go 1.15
google.golang.org/api/googleapi
google.golang.org/api/googleapi/transport
@ -343,7 +343,7 @@ google.golang.org/appengine/internal/socket
google.golang.org/appengine/internal/urlfetch
google.golang.org/appengine/socket
google.golang.org/appengine/urlfetch
# google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e
# google.golang.org/genproto v0.0.0-20220504150022-98cd25cafc72
## explicit; go 1.15
google.golang.org/genproto/googleapis/api/annotations
google.golang.org/genproto/googleapis/iam/v1