Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

This commit is contained in:
Aliaksandr Valialkin 2022-12-19 13:37:52 -08:00
commit be5bbb7ba7
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
95 changed files with 1192 additions and 445 deletions

24
.github/workflows/nightly-build.yml vendored Normal file
View file

@ -0,0 +1,24 @@
name: nightly-build
on:
schedule:
# Daily at 2:48am
- cron: '48 2 * * *'
permissions:
contents: read
jobs:
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Setup Go
uses: actions/setup-go@main
with:
go-version: 1.19.4
id: go
- name: Code checkout
uses: actions/checkout@master
- name: Publish
run: |
LATEST_TAG=nightly PKG_TAG=nightly make publish

View file

@ -3,6 +3,7 @@ PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
LATEST_TAG ?= latest
PKG_TAG ?= $(shell git tag -l --points-at HEAD)
ifeq ($(PKG_TAG),)

View file

@ -2326,7 +2326,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
-search.graphiteStorageStep duration
The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overridden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 10s)
-search.latencyOffset duration
The time when data points become visible in query results after the collection. Too small value can result in incomplete last points for query results (default 30s)
The time when data points become visible in query results after the collection. It can be overridden on per-query basis via latency_offset arg. Too small value can result in incomplete last points for query results (default 30s)
-search.logSlowQueryDuration duration
Log queries with execution time exceeding this value. Zero disables slow query logging (default 5s)
-search.maxConcurrentRequests int

View file

@ -195,7 +195,7 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt
}
bb := writeRequestBufPool.Get()
bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
if len(bb.B) <= maxUnpackedBlockSize.N {
if len(bb.B) <= maxUnpackedBlockSize.IntN() {
zb := snappyBufPool.Get()
zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
writeRequestBufPool.Put(bb)

View file

@ -695,10 +695,10 @@ may get empty response from datasource and produce empty recording rules or rese
<img alt="vmalert evaluation when data is delayed" src="vmalert_ts_data_delay.gif">
_By default recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
This behavior is controlled by `-search.latencyOffset` command-line flag on vmselect. Usually, this results into
a 30s shift for recording rules results.
Note that too small value passed to `-search.latencyOffset` may lead to incomplete query results._
By default recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
This behavior is controlled by `-search.latencyOffset` command-line flag and the `latency_offset` query ag at `vmselect`.
Usually, this results into a 30s shift for recording rules results.
Note that too small value passed to `-search.latencyOffset` or to `latency_offest` query arg may lead to incomplete query results.
Try the following recommendations in such cases:

View file

@ -145,7 +145,7 @@ func newSrcFS() (*fslocal.FS, error) {
fs := &fslocal.FS{
Dir: snapshotPath,
MaxBytesPerSecond: maxBytesPerSecond.N,
MaxBytesPerSecond: maxBytesPerSecond.IntN(),
}
if err := fs.Init(); err != nil {
return nil, fmt.Errorf("cannot initialize fs: %w", err)

View file

@ -83,7 +83,7 @@ func newDstFS() (*fslocal.FS, error) {
}
fs := &fslocal.FS{
Dir: *storageDataPath,
MaxBytesPerSecond: maxBytesPerSecond.N,
MaxBytesPerSecond: maxBytesPerSecond.IntN(),
}
if err := fs.Init(); err != nil {
return nil, fmt.Errorf("cannot initialize local fs: %w", err)

View file

@ -30,6 +30,7 @@ import (
var (
latencyOffset = flag.Duration("search.latencyOffset", time.Second*30, "The time when data points become visible in query results after the collection. "+
"It can be overridden on per-query basis via latency_offset arg. "+
"Too small value can result in incomplete last points for query results")
maxQueryLen = flagutil.NewBytes("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
maxLookback = flag.Duration("search.maxLookback", 0, "Synonym to -search.lookback-delta from Prometheus. "+
@ -681,7 +682,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
step = defaultStep
}
if len(query) > maxQueryLen.N {
if len(query) > maxQueryLen.IntN() {
return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxQueryLen.N)
}
etfs, err := searchutils.GetExtraTagFilters(r)
@ -735,7 +736,10 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
return nil
}
queryOffset := getLatencyOffsetMilliseconds()
queryOffset, err := getLatencyOffsetMilliseconds(r)
if err != nil {
return err
}
if !searchutils.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset {
// Adjust start time only if `nocache` arg isn't set.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241
@ -829,7 +833,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
}
// Validate input args.
if len(query) > maxQueryLen.N {
if len(query) > maxQueryLen.IntN() {
return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxQueryLen.N)
}
if start > end {
@ -860,7 +864,10 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
return err
}
if step < maxStepForPointsAdjustment.Milliseconds() {
queryOffset := getLatencyOffsetMilliseconds()
queryOffset, err := getLatencyOffsetMilliseconds(r)
if err != nil {
return err
}
if ct-queryOffset < end {
result = adjustLastPoints(result, ct-queryOffset, ct+step)
}
@ -1000,12 +1007,12 @@ func getRoundDigits(r *http.Request) int {
return n
}
func getLatencyOffsetMilliseconds() int64 {
func getLatencyOffsetMilliseconds(r *http.Request) (int64, error) {
d := latencyOffset.Milliseconds()
if d <= 1000 {
d = 1000
}
return d
return searchutils.GetDuration(r, "latency_offset", d)
}
// QueryStatsHandler returns query stats at `/api/v1/status/top_queries`

View file

@ -2,6 +2,7 @@ package prometheus
import (
"math"
"net/http"
"reflect"
"testing"
@ -195,3 +196,36 @@ func TestAdjustLastPoints(t *testing.T) {
},
})
}
func TestGetLatencyOffsetMillisecondsSuccess(t *testing.T) {
f := func(url string, expectedOffset int64) {
t.Helper()
r, err := http.NewRequest("GET", url, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest(%q): %s", url, err)
}
offset, err := getLatencyOffsetMilliseconds(r)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if offset != expectedOffset {
t.Fatalf("unexpected offset got %d; want %d", offset, expectedOffset)
}
}
f("http://localhost", latencyOffset.Milliseconds())
f("http://localhost?latency_offset=1.234s", 1234)
}
func TestGetLatencyOffsetMillisecondsFailure(t *testing.T) {
f := func(url string) {
t.Helper()
r, err := http.NewRequest("GET", url, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest(%q): %s", url, err)
}
if _, err := getLatencyOffsetMilliseconds(r); err == nil {
t.Fatalf("expecting non-nil error")
}
}
f("http://localhost?latency_offset=foobar")
}

View file

@ -104,10 +104,10 @@ func InitWithoutMetrics(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
storage.SetMergeWorkersCount(*smallMergeConcurrency)
storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset)
storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
storage.SetTSIDCacheSize(cacheSizeStorageTSID.N)
storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.N)
mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.N)
mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.N)
storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN())
mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN())
if retentionPeriod.Msecs < 24*3600*1000 {
logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)

View file

@ -8646,81 +8646,6 @@
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"definition": "label_values(vm_app_version{version=~\"^vminsert.*\"}, job)",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "job_insert",
"options": [],
"query": {
"query": "label_values(vm_app_version{version=~\"^vminsert.*\"}, job)",
"refId": "VictoriaMetrics-job_insert-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"definition": "label_values(vm_app_version{version=~\"^vmselect.*\"}, job)",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "job_select",
"options": [],
"query": {
"query": "label_values(vm_app_version{version=~\"^vmselect.*\"}, job)",
"refId": "VictoriaMetrics-job_select-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"definition": "label_values(vm_app_version{version=~\"^vmstorage.*\"}, job)",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "job_storage",
"options": [],
"query": {
"query": "label_values(vm_app_version{version=~\"^vmstorage.*\"}, job)",
"refId": "VictoriaMetrics-job_storage-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"current": {},
"datasource": {
@ -8746,6 +8671,81 @@
"type": "query",
"useTags": false
},
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "job_insert",
"options": [],
"query": {
"query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "job_select",
"options": [],
"query": {
"query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "job_storage",
"options": [],
"query": {
"query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {},

View file

@ -77,7 +77,7 @@ publish-via-docker: \
--build-arg root_image=$(ROOT_IMAGE) \
--build-arg APP_NAME=$(APP_NAME) \
--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \
--tag $(DOCKER_NAMESPACE)/$(APP_NAME):latest$(RACE) \
--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE) \
-o type=image \
-f app/$(APP_NAME)/multiarch/Dockerfile \
--push \

View file

@ -2,7 +2,7 @@ version: '3.5'
services:
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.84.0
image: victoriametrics/vmagent:v1.85.2
depends_on:
- "vminsert"
ports:
@ -32,7 +32,7 @@ services:
vmstorage-1:
container_name: vmstorage-1
image: victoriametrics/vmstorage:v1.84.0-cluster
image: victoriametrics/vmstorage:v1.85.2-cluster
ports:
- 8482
- 8400
@ -44,7 +44,7 @@ services:
restart: always
vmstorage-2:
container_name: vmstorage-2
image: victoriametrics/vmstorage:v1.84.0-cluster
image: victoriametrics/vmstorage:v1.85.2-cluster
ports:
- 8482
- 8400
@ -56,19 +56,19 @@ services:
restart: always
vminsert:
container_name: vminsert
image: victoriametrics/vminsert:v1.84.0-cluster
image: victoriametrics/vminsert:v1.85.2-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
command:
- '--storageNode=vmstorage-1:8400'
- '--storageNode=vmstorage-1.85.2'
- '--storageNode=vmstorage-2:8400'
ports:
- 8480:8480
restart: always
vmselect:
container_name: vmselect
image: victoriametrics/vmselect:v1.84.0-cluster
image: victoriametrics/vmselect:v1.85.2-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@ -82,7 +82,7 @@ services:
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.84.0
image: victoriametrics/vmalert:v1.85.2
depends_on:
- "vmselect"
ports:

View file

@ -2,7 +2,7 @@ version: "3.5"
services:
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.84.0
image: victoriametrics/vmagent:v1.85.2
depends_on:
- "victoriametrics"
ports:
@ -18,7 +18,7 @@ services:
restart: always
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.84.0
image: victoriametrics/victoria-metrics:v1.85.2
ports:
- 8428:8428
- 8089:8089
@ -56,7 +56,7 @@ services:
restart: always
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.84.0
image: victoriametrics/vmalert:v1.85.2
depends_on:
- "victoriametrics"
- "alertmanager"

View file

@ -16,9 +16,29 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip
## [v1.85.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.2)
Released at 2022-12-19
* FEATURE: support overriding of `-search.latencyOffset` value via URL param `latency_offset` when performing requests to [/api/v1/query](https://docs.victoriametrics.com/keyConcepts.html#instant-query) and [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3481).
* FEATURE: allow changing field names in JSON logs if VictoriaMetrics components are started with `-loggerFormat=json` command-line flags. The field names can be changed with the `-loggerJSONFields` command-line flag. For example `-loggerJSONFields=ts:timestamp,msg:message` would rename `ts` and `msg` fields on the output JSON to `timestamp` and `message` fields. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2348). Thanks to @michal-kralik for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3488).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `__meta_consul_tag_<tagname>` and `__meta_consul_tagpresent_<tagname>` labels for targets discovered via [consul_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs). This simplifies converting [Consul service tags](https://developer.hashicorp.com/consul/docs/discovery/services#service-definition) to target labels with a simple [relabeling rule](https://docs.victoriametrics.com/vmagent.html#relabeling):
```yml
- action: labelmap
regex: __meta_consul_tag_(.+)
```
This resolves [this StackOverflow question](https://stackoverflow.com/questions/44339461/relabeling-in-prometheus).
* BUGFIX: properly return query results for time series, which stop receiving new samples after the rotation of `indexdb`. Previously such time series could be missing in query results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502). The issue has been introduced in [v1.83.0](https://docs.victoriametrics.com/CHANGELOG.html#v1830).
* BUGFIX: allow specifying values bigger than 2GiB to the following command-line flag values on 32-bit architectures (`386` and `arm`): `-storage.minFreeDiskSpaceBytes` and `-remoteWrite.maxDiskUsagePerURL`. Previously values bigger than 2GiB were incorrectly truncated on these architectures.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): stop dropping metric name by a mistake on the [/metric-relabel-debug](https://docs.victoriametrics.com/vmagent.html#relabel-debug) page.
## [v1.85.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.1)
Released at 14-12-2022
Released at 2022-12-14
**It is recommended upgrading to [VictoriaMetrics v1.85.2](https://docs.victoriametrics.com/CHANGELOG.html#v1852) because of [the bug](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502), which may result in incomplete query results for historical time series.**
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): support `$for` or `.For` template variables in alert's annotations. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3246).
@ -30,7 +50,9 @@ Released at 14-12-2022
## [v1.85.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.0)
Released at 11-12-2022
Released at 2022-12-11
**It is recommended upgrading to [VictoriaMetrics v1.85.2](https://docs.victoriametrics.com/CHANGELOG.html#v1852) because of [the bug](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502), which may result in incomplete query results for historical time series.**
**Update note 1:** this release drops support for direct upgrade from VictoriaMetrics versions prior [v1.28.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.28.0). Please upgrade to `v1.84.0`, wait until `finished round 2 of background conversion` line is emitted to log by single-node VictoriaMetrics or by `vmstorage`, and then upgrade to newer releases.
@ -96,7 +118,9 @@ Released at 11-12-2022
## [v1.84.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.84.0)
Released at 25-11-2022
Released at 2022-11-25
**It is recommended upgrading to [VictoriaMetrics v1.85.2](https://docs.victoriametrics.com/CHANGELOG.html#v1852) because of [the bug](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502), which may result in incomplete query results for historical time series.**
* FEATURE: add support for [Pushgateway data import format](https://github.com/prometheus/pushgateway#url) via `/api/v1/import/prometheus` url. See [these docs](https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1415). Thanks to @PerGon for [the intial implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3360).
* FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): add `http://<vmselect>:8481/admin/tenants` API endpoint for returning a list of registered tenants. See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) for details.
@ -124,7 +148,9 @@ Released at 25-11-2022
## [v1.83.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.83.1)
Released at 10-11-2022
Released at 2022-11-10
**It is recommended upgrading to [VictoriaMetrics v1.85.2](https://docs.victoriametrics.com/CHANGELOG.html#v1852) because of [the bug](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502), which may result in incomplete query results for historical time series.**
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `__meta_consul_partition` label for targets discovered via [consul_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs) in the same way as [Prometheus 2.40 does](https://github.com/prometheus/prometheus/pull/11482).
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): show the [query trace](https://docs.victoriametrics.com/#query-tracing) in JSON view. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2814). Thanks to @michal-kralik for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3316).
@ -137,7 +163,9 @@ Released at 10-11-2022
## [v1.83.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.83.0)
Released at 29-10-2022
Released at 2022-10-29
**It is recommended upgrading to [VictoriaMetrics v1.85.2](https://docs.victoriametrics.com/CHANGELOG.html#v1852) because of [the bug](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502), which may result in incomplete query results for historical time series.**
**Update note 1:** the `indexdb/tagFilters` cache type at [/metrics](https://docs.victoriametrics.com/#monitoring) has been renamed to `indexdb/tagFiltersToMetricIDs` in order to make its puropose more clear.
@ -192,7 +220,7 @@ Released at 29-10-2022
## [v1.82.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.82.1)
Released at 14-10-2022
Released at 2022-10-14
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): automatically update graph, legend and url after the removal of query field. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3169) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3196#issuecomment-1269765205).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove duplicate `alertname` JSON entry from generated alerts. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3053). Thanks to @Howie59 for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3182)!
@ -204,7 +232,7 @@ Released at 14-10-2022
## [v1.82.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.82.0)
Released at 07-10-2022
Released at 2022-10-07
**It isn't recommended to use VictoriaMetrics and vmagent v1.82.0 because of [the bug](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3227), which may result in incorrect query results and [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) results. Upgrade to [v1.82.1](https://docs.victoriametrics.com/CHANGELOG.html#v1821) instead.**
@ -268,13 +296,13 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m
## [v1.81.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.81.2)
Released at 08-09-2022
Released at 2022-09-08
* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly calculate query results at `vmselect`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3067). The issue has been introduced in [v1.81.0](https://docs.victoriametrics.com/CHANGELOG.html#v1810).
## [v1.81.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.81.1)
Released at 02-09-2022
Released at 2022-09-02
**It isn't recommended to use VictoriaMetrics cluster v1.81.1 because of [the bug](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3067), which may result in incorrect query results. Upgrade to [v1.81.2](https://docs.victoriametrics.com/CHANGELOG.html#v1812) instead.**
@ -287,7 +315,7 @@ Released at 02-09-2022
**It isn't recommended to use VictoriaMetrics cluster v1.81.0 because of [the bug](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3058), which may result in `vmselect` crashes under high load. Upgrade to [v1.81.2](https://docs.victoriametrics.com/CHANGELOG.html#v1812) instead.**
Released at 31-08-2022
Released at 2022-08-31
**Update note 1:** [vmalert](https://docs.victoriametrics.com/vmalert.html) by default hides values of `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` in logs and at `http://vmalert:8880/flags` for security reasons. See the corresponding SECURITY change in the Chagelog below for additional info.
@ -323,7 +351,7 @@ Released at 31-08-2022
## [v1.80.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.80.0)
Released at 08-08-2022
Released at 2022-08-08
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow configuring additional HTTP request headers for `-datasource.url`, `-remoteWrite.url` and `-remoteRead.url` via `-datasource.headers`, `-remoteWrite.headers` and `-remoteRead.headers` command-line flags. Additional HTTP request headers also can be set on group level via `headers` param - see [these docs](https://docs.victoriametrics.com/vmalert.html#groups) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2860).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): execute left and right sides of certain operations in parallel. For example, `q1 or q2`, `aggr_func(q1) <op> q2`, `q1 <op> aggr_func(q1)`. This may improve query performance if VictoriaMetrics has enough free resources for parallel processing of both sides of the operation. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886).
@ -350,7 +378,7 @@ Released at 08-08-2022
## [v1.79.6](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.6)
Released at 11-12-2022
Released at 2022-12-11
**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
@ -366,7 +394,7 @@ The v1.79.x line will be supported for at least 12 months since [v1.79.0](https:
## [v1.79.5](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.5)
Released at 10-11-2022
Released at 2022-11-10
**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
@ -384,7 +412,7 @@ The v1.79.x line will be supported for at least 12 months since [v1.79.0](https:
## [v1.79.4](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.4)
Released at 07-10-2022
Released at 2022-10-07
**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
@ -408,7 +436,7 @@ The v1.79.x line will be supported for at least 12 months since [v1.79.0](https:
## [v1.79.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.3)
Released at 30-08-2022
Released at 2022-08-30
**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
@ -423,7 +451,7 @@ The v1.79.x line will be supported for at least 12 months since [v1.79.0](https:
## [v1.79.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.2)
Released at 08-08-2022
Released at 2022-08-08
**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
@ -444,7 +472,7 @@ The v1.79.x line will be supported for at least 12 months since [v1.79.0](https:
## [v1.79.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.1)
Released at 02-08-2022
Released at 2022-08-02
**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
@ -454,7 +482,7 @@ The v1.79.x line will be supported for at least 12 months since [v1.79.0](https:
## [v1.79.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.0)
Released at 14-07-2022
Released at 2022-07-14
**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
@ -531,7 +559,7 @@ scrape_configs:
## [v1.78.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.1)
Released at 08-07-2022
Released at 2022-07-08
**Update notes:** it is recommended [clearing caches](https://docs.victoriametrics.com/#cache-removal) after the upgrade from [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0) in order to immediately fix the issue for newly ingested data. Otherwise the issue may exist for newly ingested data for up to a day after the upgrade.
@ -539,9 +567,9 @@ Released at 08-07-2022
## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0)
Released at 20-06-2022
Released at 2022-06-20
**Warning (03-07-2022):** VictoriaMetrics v1.78.0 contains a bug, which may result in missing time series during queries. It is recommended upgrading to [v1.78.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.1), which fixes the bug.
**Warning (2022-07-03):** VictoriaMetrics v1.78.0 contains a bug, which may result in missing time series during queries. It is recommended upgrading to [v1.78.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.1), which fixes the bug.
**Update notes:** this release introduces backwards-incompatible changes to communication protocol between `vmselect` and `vmstorage` nodes in cluster version of VictoriaMetrics because of added [query tracing](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#query-tracing), so read requests to `vmselect` will fail until the upgrade is complete. These errors will stop after all the `vmselect` and `vmstorage` nodes are updated to the new release. It is safe to downgrade to previous releases.
@ -584,7 +612,7 @@ Released at 20-06-2022
## [v1.77.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.77.2)
Released at 21-05-2022
Released at 2022-05-21
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): support [reusable templates](https://prometheus.io/docs/prometheus/latest/configuration/template_examples/#defining-reusable-templates) for rules annotations. The path to the template files can be specified via `-rule.templates` flag. See more about this feature [here](https://docs.victoriametrics.com/vmalert.html#reusable-templates). Thanks to @AndrewChubatiuk for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2532). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2510).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): expose `vmalert_iteration_interval_seconds` metric at `http://vmalert:8880/metrics`. This metric shows the configured per-group evaluation interval. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2618).
@ -603,7 +631,7 @@ Released at 21-05-2022
## [v1.77.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.77.1)
Released at 07-05-2022
Released at 2022-05-07
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to specify filters for Availability Zones in [ec2_sd_config](https://docs.victoriametrics.com/sd_configs.html#ec2_sd_configs) via `az_filters` section. This section can contain AZ-specific set of filters in the same way as the existing `filters` section, which is used for filtering EC2 instances. The list of supported AZ-specific filters is available [here](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `vmagent_remotewrite_global_rows_pushed_before_relabel_total` and `vmagent_remotewrite_rows_pushed_after_relabel_total` metrics at `http://vmagent:8429/metrics`, which can be used for monitoring the rate of rows (aka samples) pushed to remote storage before and after the relabeling via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`. See [relabeling docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for details.
@ -621,7 +649,7 @@ Released at 07-05-2022
## [v1.77.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.77.0)
Released at 05-05-2022
Released at 2022-05-05
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for sending data to remote storage with AWS sigv4 authorization. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1287).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow filtering targets by target url and by target labels with [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) on `http://vmagent:8429/targets` page. This may be useful when `vmagent` scrapes big number of targets. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1796).
@ -663,7 +691,7 @@ Released at 05-05-2022
## [v1.76.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.76.1)
Released at 12-04-2022
Released at 2022-04-12
**Update notes:** this release introduces backwards-incompatible changes to communication protocol between `vmselect` and `vmstorage` nodes in cluster version of VictoriaMetrics, so read requests to `vmselect` will fail until the upgrade is complete. These errors will stop after all the `vmselect` and `vmstorage` nodes are updated to the new release. It is safe to downgrade to previous releases.
@ -679,7 +707,7 @@ Released at 12-04-2022
## [v1.76.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.76.0)
Released at 07-04-2022
Released at 2022-04-07
**Update notes:** this release introduces backwards-incompatible changes to communication protocol between `vmselect` and `vmstorage` nodes in cluster version of VictoriaMetrics, so read requests to `vmselect` will fail until the upgrade is complete. These errors will stop after all the `vmselect` and `vmstorage` nodes are updated to the new release. It is safe to downgrade to previous releases.
@ -707,13 +735,13 @@ When using [cluster version of VictoriaMetrics](https://docs.victoriametrics.com
## [v1.75.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.75.1)
Released at 28-03-2022
Released at 2022-03-28
* BUGFIX: update base image for VictoriaMetrics from `alpine-3.15.0` to `alpine-3.15.2`. This fixes [CVE-2022-0778](https://nvd.nist.gov/vuln/detail/CVE-2022-0778). See [alpine 3.15.2 release docs](https://alpinelinux.org/posts/Alpine-3.15.2-released.html).
## [v1.75.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.75.0)
Released at 18-03-2022
Released at 2022-03-18
**Update notes:** release contains breaking change to vmalert's API introduced in [ee396b5](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2320/commits/ee396b5750d0bcb98233d624f88fa6cf92a8253b).
It replaces the `api/v1/groups` API handler with `api/v1/rules` handler in order to become compatible
@ -737,7 +765,7 @@ See other changes introduced to vmalert [here](https://github.com/VictoriaMetric
## [v1.74.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.74.0)
Released at 03-03-2022
Released at 2022-03-03
**Update notes:** In this release VictoriaMetrics may use some extra memory due to issues [#2242](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2242) and [#2007](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2007). These issues were addressed in [v1.75.0](#v1750), so we recommend updating straight to it.
@ -768,7 +796,7 @@ This rule is equivalent to less clear traditional one:
## [v1.73.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.73.1)
Released at 22-02-2022
Released at 2022-02-22
**Update notes:** In this release VictoriaMetrics may use some extra memory due to issues [#2242](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2242) and [#2007](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2007). These issues were addressed in [v1.75.0](#v1750), so we recommend updating straight to it.
@ -789,7 +817,7 @@ Released at 22-02-2022
## [v1.73.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.73.0)
Released at 14-02-2022
Released at 2022-02-14
**Update notes:** In this release VictoriaMetrics may use some extra memory described in issues [#2242](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2242) and [#2007](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2007). These issues were addressed in [v1.75.0](#v1750), so we recommend updating straight to it.
@ -830,7 +858,7 @@ Released at 14-02-2022
## [v1.72.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.72.0)
Released at 18-01-2022
Released at 2022-01-18
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add support for `@` modifier, which is enabled by default in Prometheus starting from [Prometheus v2.33.0](https://github.com/prometheus/prometheus/pull/10121). See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1348). VictoriaMetrics extends `@` modifier with the following additional features:
* It can contain arbitrary expression. For example, `foo @ (end() - 1h)` would return `foo` value at `end - 1 hour` timestamp on the selected time range `[start ... end]`. Another example: `foo @ (now() - 10m)` would return `foo` value 10 minutes ago from the current time.
@ -871,7 +899,7 @@ Released at 18-01-2022
## [v1.71.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.71.0)
Released at 20-12-2021
Released at 2021-12-20
**Update notes:** deduplication logic was slightly changed on the release, which may cause extra
[background merges](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
@ -904,7 +932,7 @@ We recommend updating in "off-peak" time when load on the VictoriaMetrics is on
## [v1.70.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.70.0)
Released at 02-12-2021
Released at 2021-12-02
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add ability to pass arbitrary query args to `-datasource.url` on a per-group basis via `params` option. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1892).
* FEATURE: add `now()` function to MetricsQL. This function returns the current timestamp in seconds. See [these docs](https://docs.victoriametrics.com/MetricsQL.html#now).
@ -933,7 +961,7 @@ Released at 02-12-2021
## [v1.69.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.69.0)
Released at 08-11-2021
Released at 2021-11-08
* FEATURE: vmalert: allow groups with empty rules list like Prometheus does. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1742).
* FEATURE: vmalert: allow groups with default `tenant` in `-clusterMode`. Default `tenant` values can be specified via `-defaultTenant.prometheus` and `-defaultTenant.graphite`. See [these docs](https://docs.victoriametrics.com/vmalert.html#multitenancy).
@ -955,7 +983,7 @@ Released at 08-11-2021
## [v1.68.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.68.0)
Released at 22-10-2021
Released at 2021-10-22
* FEATURE: vmagent: expose `-promscrape.config` contents at `/config` page as Prometheus does. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1695).
* FEATURE: vmagent: add `show original labels` button per each scrape target displayed at `http://vmagent:8429/targets` page. This should improve debuggability for service discovery and relabeling issues similar to [this one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1664). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1698).
@ -983,7 +1011,7 @@ Released at 22-10-2021
## [v1.67.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.67.0)
Released at 08-10-2021
Released at 2021-10-08
* FEATURE: add ability to accept metrics from [DataDog agent](https://docs.datadoghq.com/agent/) and [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent). This option simplifies the migration path from DataDog to VictoriaMetrics. See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/206).
* FEATURE: vmagent [enterprise](https://docs.victoriametrics.com/enterprise.html): add support for data reading and writing from/to [Apache Kafka](https://kafka.apache.org/). See [these docs](https://docs.victoriametrics.com/vmagent.html#kafka-integration).
@ -1000,7 +1028,7 @@ Released at 08-10-2021
## [v1.66.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.66.2)
Released at 23-09-2021
Released at 2021-09-23
* FEATURE: vmagent: add `vm_promscrape_max_scrape_size_exceeded_errors_total` metric for counting of the failed scrapes due to the exceeded response size (the response size limit can be configured via `-promscrape.maxScrapeSize` command-line flag). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1639).
@ -1009,7 +1037,7 @@ Released at 23-09-2021
## [v1.66.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.66.1)
Released at 22-09-2021
Released at 2021-09-22
* FEATURE: add `-cluster` and/or `-enterprise` suffixes to `short_version` label at `vm_app_version` metric exposed at `/metrics` page of every VictoriaMetrics component. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1635).
@ -1018,7 +1046,7 @@ Released at 22-09-2021
## [v1.66.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.66.0)
Released at 20-09-2021
Released at 2021-09-20
* FEATURE: vmalert: add web UI with the list of alerting groups, alerts and alert statuses. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1602).
* FEATURE: vmalert: add `-rule.maxResolveDuration` command-line flag, which could be used for limiting the auto-resolve duration for the alerting rule. By default it is limited to 3x evaluation interval. This could be too high for big evaluation intervals. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1586).
@ -1055,7 +1083,7 @@ Released at 20-09-2021
## [v1.65.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.65.0)
Released at 01-09-2021
Released at 2021-09-01
* FEATURE: vmagent: add ability to read scrape configs from multiple files specified in `scrape_config_files` section. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1559).
* FEATURE: vmagent: reduce memory usage and CPU usage when [Prometheus staleness tracking](https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers) is enabled for metrics exported from the deleted or disappeared scrape targets.
@ -1079,7 +1107,7 @@ Released at 01-09-2021
## [v1.64.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.64.1)
Released at 19-08-2021
Released at 2021-08-19
* FEATURE: add `bitmap_and(q, mask)`, `bitmap_or(q, mask)` and `bitmak_xor(q, mask)` functions to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). These functions allow performing bitwise operations over data points in time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1541).
* FEATURE: vmalert: add `-remoteWrite.disablePathAppend` command-line flag, which can be used when custom `-remoteWrite.url` must be specified. For example, `./vmalert -disablePathAppend -remoteWrite.url='http://foo.bar/a/b/c?d=e'` would write data to `http://foo.bar/a/b/c?d=e` instead of `http://foo.bar/a/b/c?d=e/api/v1/write`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1536).
@ -1093,7 +1121,7 @@ Released at 19-08-2021
## [v1.64.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.64.0)
Released at 15-08-2021
Released at 2021-08-15
* FEATURE: add support for Prometheus staleness markers. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1526).
* FEATURE: vmagent: automatically generate Prometheus staleness markers for the scraped metrics when scrape targets disappear in the same way as Prometheus does. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1526).
@ -1119,7 +1147,7 @@ Released at 15-08-2021
## [v1.63.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.63.0)
Released at 15-07-2021
Released at 2021-07-15
* FEATURE: reduce memory usage by up to 30% on production workloads.
* FEATURE: vmselect: embed [vmui](https://github.com/VictoriaMetrics/vmui) into a single-node VictoriaMetrics and into `vmselect` component of cluster version. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1413). The web interface is available at the following paths:
@ -1140,7 +1168,7 @@ Released at 15-07-2021
## [v1.62.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.62.0)
Released at 25-06-2021
Released at 2021-06-25
* FEATURE: vmagent: add service discovery for Docker (aka [docker_sd_config](https://docs.victoriametrics.com/sd_configs.html#docker_sd_configs)). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1402).
* FEATURE: vmagent: add service discovery for DigitalOcean (aka [digitalocean_sd_config](https://docs.victoriametrics.com/sd_configs.html#digitalocean_sd_configs)). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1367).
@ -1163,14 +1191,14 @@ Released at 25-06-2021
## [v1.61.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.1)
Released at 11-06-2021
Released at 2021-06-11
* BUGFIX: vmalert: fix recording rules, which were broken in v1.61.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1369).
* BUGFIX: reset the on-disk cache for mapping from the full metric name to an internal metric id (e.g. `metric_name{labels} -> internal_metric_id`) after deleting metrics via [delete API](https://docs.victoriametrics.com/#how-to-delete-time-series). This should prevent from possible inconsistent state after unclean shutdown. This [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1347).
## [v1.61.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.0)
Released at 09-06-2021
Released at 2021-06-09
* FEATURE: vmalert: add support for backfilling (aka replay) of recording and alerting rules. See [these docs](https://docs.victoriametrics.com/vmalert.html#rules-backfilling) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/836).
* FEATURE: vmalert: add a command-line flag `-rule.configCheckInterval` for automatic re-reading of `-rule` files without the need to send SIGHUP signal. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/512).
@ -1188,7 +1216,7 @@ Released at 09-06-2021
## [v1.60.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.60.0)
Released at 24-05-2021
Released at 2021-05-24
* FEATURE: add ability to limit the number of unique time series, which can be added to storage per hour and per day. This can help dealing with high cardinality and high churn rate issues. See [these docs](https://docs.victoriametrics.com/#cardinality-limiter).
* FEATURE: vmagent: add ability to limit the number of unique time series, which can be sent to remote storage systems per hour and per day. This can help dealing with high cardinality and high churn rate issues. See [these docs](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter).
@ -1224,7 +1252,7 @@ Released at 24-05-2021
## [v1.59.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.59.0)
Released at 01-05-2021
Released at 2021-05-01
* FEATURE: improved new time series registration speed on systems with many CPU cores. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1244). Thanks to @waldoweng for the idea and [draft implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1243).
* FEATURE: vmalert: use the same technique as Grafana for determining evaluation timestamps for recording rules. This should make consistent graphs for series generated by recording rules compared to graphs generated for queries from recording rules in Grafana. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232).
@ -1241,7 +1269,7 @@ Thanks to @johnseekins!
## [v1.58.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.58.0)
Released at 08-04-2021
Released at 2021-04-08
* FEATURE: vminsert and vmagent: add `-sortLabels` command-line flag for sorting metric labels before pushing them to `vmstorage`. This should reduce the size of `MetricName -> internal_series_id` cache (aka `vm_cache_size_bytes{type="storage/tsid"}`) when ingesting samples for the same time series with distinct order of labels. For example, `foo{k1="v1",k2="v2"}` and `foo{k2="v2",k1="v1"}` represent a single time series. Labels sorting is disabled by default, since the majority of established exporters preserve the order of labels for the exported metrics.
* FEATURE: allow specifying label value alongside label name for the `others sum` time series returned from `topk_*` and `bottomk_*` functions from [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). For example, `topk_avg(3, max(process_resident_memory_bytes) by (instance), "instance=other_sum")` would return top 3 series from `max(process_resident_memory_bytes) by (instance)` plus a series containing the sum of other series. The `others sum` series will have `{instance="other_sum"}` label.
@ -1268,7 +1296,7 @@ Released at 08-04-2021
## [v1.57.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.57.1)
Released at 30-03-2021
Released at 2021-03-30
* FEATURE: publish vmutils for `GOOS=arm` on [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
@ -1278,7 +1306,7 @@ Released at 30-03-2021
## [v1.57.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.57.0)
Released at 29-03-2021
Released at 2021-03-29
* FEATURE: optimize query performance by up to 10x on systems with many CPU cores. See [this tweet](https://twitter.com/MetricsVictoria/status/1375064484860067840).
* FEATURE: add the following metrics at `/metrics` page for every VictoraMetrics app:
@ -1302,7 +1330,7 @@ Released at 29-03-2021
## [v1.56.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.56.0)
Released at 17-03-2021
Released at 2021-03-17
* FEATURE: add the following functions to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html):
* `histogram_avg(buckets)` - returns the average value for the given buckets.
@ -1332,14 +1360,14 @@ Released at 17-03-2021
## [v1.55.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.55.1)
Released at 03-03-2021
Released at 2021-03-03
* BUGFIX: vmagent: fix a panic in Kubernetes service discovery when a target is filtered out with relabeling. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1107>
* BUGFIX: vmagent: fix Kubernetes service discovery for `role: ingress`. See <https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1110>
## [v1.55.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.55.0)
Released at 02-03-2021
Released at 2021-03-02
* FEATURE: add `sign(q)` and `clamp(q, min, max)` functions, which are planned to be added in [the upcoming Prometheus release](https://twitter.com/roidelapluie/status/1363428376162295811) . The `last_over_time(m[d])` function is already supported in [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html).
* FEATURE: vmagent: add `scrape_align_interval` config option, which can be used for aligning scrapes to the beginning of the configured interval. See [these docs](https://docs.victoriametrics.com/vmagent.html#troubleshooting) for details.
@ -1375,13 +1403,13 @@ Released at 02-03-2021
## [v1.54.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.54.1)
Released at 18-02-2021
Released at 2021-08-18
* BUGFIX: properly handle queries containing a filter on metric name plus any number of negative filters and zero non-negative filters. For example, `node_cpu_seconds_total{mode!="idle"}`. The bug was introduced in [v1.54.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.54.0).
## [v1.54.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.54.0)
Released at 18-02-2021
Released at 2021-02-18
* FEATURE: optimize searching for matching metrics for `metric{<label_filters>}` queries if `<label_filters>` contains at least a single filter. For example, the query `up{job="foobar"}` should find the matching time series much faster than previously.
* FEATURE: reduce execution times for `q1 <binary_op> q2` queries by executing `q1` and `q2` in parallel.
@ -1403,13 +1431,13 @@ Released at 18-02-2021
## [v1.53.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.53.1)
Released at 03-02-2021
Released at 2021-02-03
* BUGFIX: vmselect: fix the bug peventing from proper searching by Graphite filter with wildcards such as `{__graphite__="foo.*.bar"}`.
## [v1.53.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.53.0)
Released at 03-02-2021
Released at 2021-02-03
* FEATURE: added [vmctl tool](https://docs.victoriametrics.com/vmctl.html) to VictoriaMetrics release process. Now it is packaged in `vmutils-*.tar.gz` archive on [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases). Source code for `vmctl` tool has been moved from [github.com/VictoriaMetrics/vmctl](https://github.com/VictoriaMetrics/vmctl) to [github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmctl).
* FEATURE: added `-loggerTimezone` command-line flag for adjusting time zone for timestamps in log messages. By default UTC is used.
@ -1435,7 +1463,7 @@ in front of VictoriaMetrics. [Contact us](mailto:sales@victoriametrics.com) if y
## [v1.52.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.52.0)
Released at 13-01-2021
Released at 2021-01-13
* FEATURE: provide a sample list of alerting rules for VictoriaMetrics components. It is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml).
* FEATURE: disable final merge for data for the previous month at the beginning of new month, since it may result in high disk IO and CPU usage. Final merge can be enabled by setting `-finalMergeDelay` command-line flag to positive duration.
@ -1456,7 +1484,7 @@ Released at 13-01-2021
## [v1.51.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.51.0)
Released at 27-12-2020
Released at 2020-12-27
* FEATURE: add `/api/v1/status/top_queries` handler, which returns the most frequently executed queries and queries that took the most time for execution. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/907>
* FEATURE: vmagent: add support for `proxy_url` config option in Prometheus scrape configs. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/503>
@ -1469,7 +1497,7 @@ Released at 27-12-2020
## [v1.50.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.50.2)
Released at 19-12-2020
Released at 2020-12-19
* FEATURE: do not publish duplicate Docker images with `-cluster` tag suffix for [vmagent](https://docs.victoriametrics.com/vmagent.html), [vmalert](https://docs.victoriametrics.com/vmalert.html), [vmauth](https://docs.victoriametrics.com/vmauth.html), [vmbackup](https://docs.victoriametrics.com/vmbackup.html) and [vmrestore](https://docs.victoriametrics.com/vmrestore.html), since they are identical to images without `-cluster` tag suffix.
@ -1479,7 +1507,7 @@ Released at 19-12-2020
## [v1.50.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.50.1)
Released at 15-12-2020
Released at 2020-12-15
* FEATURE: vmagent: export `vmagent_remotewrite_blocks_sent_total` and `vmagent_remotewrite_blocks_sent_total` metrics for each `-remoteWrite.url`.
@ -1487,7 +1515,7 @@ Released at 15-12-2020
## [v1.50.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.50.0)
Released at 15-12-2020
Released at 2020-12-15
* FEATURE: automatically reset response cache when samples with timestamps older than `now - search.cacheTimestampOffset` are ingested to VictoriaMetrics. This makes unnecessary disabling response cache during data backfilling or resetting it after backfilling is complete as described [in these docs](https://docs.victoriametrics.com/#backfilling). This feature applies only to single-node VictoriaMetrics. It doesn't apply to cluster version of VictoriaMetrics because `vminsert` nodes don't know about `vmselect` nodes where the response cache must be reset.
* FEATURE: vmalert: add `query`, `first` and `value` functions to alert templates. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/539>
@ -1513,7 +1541,7 @@ Released at 15-12-2020
## [v1.49.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.49.0)
Released at 05-12-2020
Released at 2020-12-05
* FEATURE: optimize Consul service discovery speed when discovering big number of services. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/574>
* FEATURE: add `label_uppercase(q, label1, ... labelN)` and `label_lowercase(q, label1, ... labelN)` function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html)
@ -1534,7 +1562,7 @@ Released at 05-12-2020
## [v1.48.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.48.0)
Released at 26-11-2020
Released at 2020-11-26
* FEATURE: added [Snap package for single-node VictoriaMetrics](https://snapcraft.io/victoriametrics). This simplifies installation under Ubuntu to a single command:
@ -1563,7 +1591,7 @@ Released at 26-11-2020
## [v1.47.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.47.0)
Released at 16-11-2020
Released at 2020-11-16
* FEATURE: vmselect: return the original error from `vmstorage` node in query response if `-search.denyPartialResponse` is set.
See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/891>
@ -1594,7 +1622,7 @@ Released at 16-11-2020
## [v1.46.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.46.0)
Released at 07-11-2020
Released at 2020-11-07
* FEATURE: optimize requests to `/api/v1/labels` and `/api/v1/label/<name>/values` when `start` and `end` args are set.
* FEATURE: reduce memory usage when query touches big number of time series.
@ -1613,7 +1641,7 @@ Released at 07-11-2020
## [v1.45.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.45.0)
Released at 02-11-2020
Released at 2020-11-02
* FEATURE: allow setting `-retentionPeriod` smaller than one month. I.e. `-retentionPeriod=3d`, `-retentionPeriod=2w`, etc. is supported now.
See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/173>
@ -1648,7 +1676,7 @@ Released at 02-11-2020
## [v1.44.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.44.0)
Released at 13-10-2020
Released at 2020-10-13
* FEATURE: automatically add missing label filters to binary operands as described at <https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization> .
This should improve performance for queries with missing label filters in binary operands. For example, the following query should work faster now, because it shouldn't
@ -1711,7 +1739,7 @@ Released at 13-10-2020
## [v1.43.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.43.0)
Released at 06-10-2020
Released at 2020-10-06
* FEATURE: reduce CPU usage for repeated queries over sliding time window when no new time series are added to the database.
Typical use cases: repeated evaluation of alerting rules in [vmalert](https://docs.victoriametrics.com/vmalert.html) or dashboard auto-refresh in Grafana.
@ -1731,7 +1759,7 @@ Released at 06-10-2020
## [v1.42.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.42.0)
Released at 30-09-2020
Released at 2020-09-30
* FEATURE: use all the available CPU cores when accepting data via a single TCP connection
for [all the supported protocols](https://docs.victoriametrics.com/#how-to-import-time-series-data).

View file

@ -1036,7 +1036,7 @@ Below is the output for `/path/to/vmselect -help`:
-search.graphiteStorageStep duration
The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overridden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 10s)
-search.latencyOffset duration
The time when data points become visible in query results after the collection. Too small value can result in incomplete last points for query results (default 30s)
The time when data points become visible in query results after the collection. It can be overridden on per-query basis via latency_offset arg. Too small value can result in incomplete last points for query results (default 30s)
-search.logSlowQueryDuration duration
Log queries with execution time exceeding this value. Zero disables slow query logging (default 5s)
-search.maxConcurrentRequests int

View file

@ -2327,7 +2327,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
-search.graphiteStorageStep duration
The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overridden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 10s)
-search.latencyOffset duration
The time when data points become visible in query results after the collection. Too small value can result in incomplete last points for query results (default 30s)
The time when data points become visible in query results after the collection. It can be overridden on per-query basis via latency_offset arg. Too small value can result in incomplete last points for query results (default 30s)
-search.logSlowQueryDuration duration
Log queries with execution time exceeding this value. Zero disables slow query logging (default 5s)
-search.maxConcurrentRequests int

View file

@ -2330,7 +2330,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
-search.graphiteStorageStep duration
The interval between datapoints stored in the database. It is used at Graphite Render API handler for normalizing the interval between datapoints in case it isn't normalized. It can be overridden by sending 'storage_step' query arg to /render API or by sending the desired interval via 'Storage-Step' http header during querying /render API. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default 10s)
-search.latencyOffset duration
The time when data points become visible in query results after the collection. Too small value can result in incomplete last points for query results (default 30s)
The time when data points become visible in query results after the collection. It can be overridden on per-query basis via latency_offset arg. Too small value can result in incomplete last points for query results (default 30s)
-search.logSlowQueryDuration duration
Log queries with execution time exceeding this value. Zero disables slow query logging (default 5s)
-search.maxConcurrentRequests int

View file

@ -327,6 +327,7 @@ for data ingestion (aka `push protocols`):
* [Prometheus remote write API](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#prometheus-setup).
* [Prometheus text exposition format](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format).
* [DataDog protocol](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent).
* [InfluxDB line protocol](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf)
over HTTP, TCP and UDP.
* [Graphite plaintext protocol](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
@ -335,7 +336,6 @@ for data ingestion (aka `push protocols`):
* [HTTP OpenTSDB /api/put requests](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#sending-opentsdb-data-via-http-apiput-requests).
* [JSON line format](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-json-line-format).
* [Arbitrary CSV data](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-csv-data).
* [Native binary format](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-native-format).
All the protocols are fully compatible with VictoriaMetrics [data model](#data-model) and can be used in production.
We recommend using the [github.com/VictoriaMetrics/metrics](https://github.com/VictoriaMetrics/metrics) package

View file

@ -140,6 +140,7 @@ scrape_configs:
# tag_separate is an optional string by which Consul tags are joined into the __meta_consul_tags label.
# By default "," is used as a tag separator.
# Individual tags are also available via __meta_consul_tag_<tagname> labels - see below.
# tag_separator: "..."
# allow_stale is an optional config, which allows stale Consul results.
@ -166,7 +167,9 @@ The following meta labels are available on discovered targets during [relabeling
* `__meta_consul_service_port`: the service port of the target
* `__meta_consul_service`: the name of the service the target belongs to
* `__meta_consul_tagged_address_<key>`: each node tagged address key value of the target
* `__meta_consul_tags`: the list of tags of the target joined by the tag separator
* `__meta_consul_tag_<tagname>`: the value for the given <tagname> tag of the target
* `__meta_consul_tagpresent_<tagname>`: "true" for every <tagname> tag of the target
* `__meta_consul_tags`: the list of tags of the target joined by the tag_separator
## digitalocean_sd_configs
@ -631,7 +634,7 @@ The following meta labels are available on discovered targets during [relabeling
* `__meta_gce_project`: the GCP project in which the instance is running
* `__meta_gce_public_ip`: the public IP address of the instance, if present
* `__meta_gce_subnetwork`: the subnetwork URL of the instance
* `__meta_gce_tags`: comma separated list of instance tags
* `__meta_gce_tags`: list of instance tags separated by tag_separator
* `__meta_gce_zone`: the GCE zone URL in which the instance is running

View file

@ -699,10 +699,10 @@ may get empty response from datasource and produce empty recording rules or rese
<img alt="vmalert evaluation when data is delayed" src="vmalert_ts_data_delay.gif">
_By default recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
This behavior is controlled by `-search.latencyOffset` command-line flag on vmselect. Usually, this results into
a 30s shift for recording rules results.
Note that too small value passed to `-search.latencyOffset` may lead to incomplete query results._
By default recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
This behavior is controlled by `-search.latencyOffset` command-line flag and the `latency_offset` query ag at `vmselect`.
Usually, this results into a 30s shift for recording rules results.
Note that too small value passed to `-search.latencyOffset` or to `latency_offest` query arg may lead to incomplete query results.
Try the following recommendations in such cases:

42
go.mod
View file

@ -13,10 +13,10 @@ require (
github.com/VictoriaMetrics/fasthttp v1.1.0
github.com/VictoriaMetrics/metrics v1.23.0
github.com/VictoriaMetrics/metricsql v0.50.0
github.com/aws/aws-sdk-go-v2 v1.17.2
github.com/aws/aws-sdk-go-v2/config v1.18.4
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.43
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.5
github.com/aws/aws-sdk-go-v2 v1.17.3
github.com/aws/aws-sdk-go-v2/config v1.18.6
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.45
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6
github.com/cespare/xxhash/v2 v2.2.0
github.com/cheggaaa/pb/v3 v3.1.0
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
@ -41,33 +41,33 @@ require (
golang.org/x/net v0.4.0
golang.org/x/oauth2 v0.3.0
golang.org/x/sys v0.3.0
google.golang.org/api v0.104.0
google.golang.org/api v0.105.0
gopkg.in/yaml.v2 v2.4.0
)
require (
cloud.google.com/go v0.107.0 // indirect
cloud.google.com/go/compute v1.14.0 // indirect
cloud.google.com/go/compute/metadata v0.2.2 // indirect
cloud.google.com/go/iam v0.8.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.9.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/aws/aws-sdk-go v1.44.160 // indirect
github.com/aws/aws-sdk-go v1.44.163 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.6 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.26 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.27 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
@ -107,7 +107,7 @@ require (
go.opentelemetry.io/otel/trace v1.11.2 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/goleak v1.2.0 // indirect
golang.org/x/exp v0.0.0-20221212164502-fae10dda9338 // indirect
golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/text v0.5.0 // indirect
golang.org/x/time v0.3.0 // indirect

84
go.sum
View file

@ -23,12 +23,12 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0=
cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo=
cloud.google.com/go/compute/metadata v0.2.2 h1:aWKAjYaBaOSrpKl57+jnS/3fJRQnxL7TvR/u1VVbt6k=
cloud.google.com/go/compute/metadata v0.2.2/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk=
cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE=
cloud.google.com/go/iam v0.9.0 h1:bK6Or6mxhuL8lnj1i9j0yMo2wE/IeTO2cWlfUrf/TZs=
cloud.google.com/go/iam v0.9.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM=
cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
@ -89,44 +89,44 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu
github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo=
github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.44.160 h1:F41sWUel1CJ69ezoBGCg8sDyu9kyeKEpwmDrLXbCuyA=
github.com/aws/aws-sdk-go v1.44.160/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v1.17.2 h1:r0yRZInwiPBNpQ4aDy/Ssh3ROWsGtKDwar2JS8Lm+N8=
github.com/aws/aws-sdk-go-v2 v1.17.2/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go v1.44.163 h1:XO1A/Laqf/l0IxVPghaQzdnVwxofVFv00IlX0BpmbhQ=
github.com/aws/aws-sdk-go v1.44.163/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY=
github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
github.com/aws/aws-sdk-go-v2/config v1.18.4 h1:VZKhr3uAADXHStS/Gf9xSYVmmaluTUfkc0dcbPiDsKE=
github.com/aws/aws-sdk-go-v2/config v1.18.4/go.mod h1:EZxMPLSdGAZ3eAmkqXfYbRppZJTzFTkv8VyEzJhKko4=
github.com/aws/aws-sdk-go-v2/credentials v1.13.4 h1:nEbHIyJy7mCvQ/kzGG7VWHSBpRB4H6sJy3bWierWUtg=
github.com/aws/aws-sdk-go-v2/credentials v1.13.4/go.mod h1:/Cj5w9LRsNTLSwexsohwDME32OzJ6U81Zs33zr2ZWOM=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20 h1:tpNOglTZ8kg9T38NpcGBxudqfUAwUzyUnLQ4XSd0CHE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20/go.mod h1:d9xFpWd3qYwdIXM0fvu7deD08vvdRXyc/ueV+0SqaWE=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.43 h1:+bkAMTd5OGyHu2nwNOangjEsP65fR0uhMbZJA52sZ64=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.43/go.mod h1:sS2tu0VEspKuY5eM1vQgy7P/hpZX8F62o6qsghZExWc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26 h1:5WU31cY7m0tG+AiaXuXGoMzo2GBQ1IixtWa8Yywsgco=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26/go.mod h1:2E0LdbJW6lbeU4uxjum99GZzI0ZjDpAb0CoSCM0oeEY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20 h1:WW0qSzDWoiWU2FS5DbKpxGilFVlCEJPwx4YtjdfI0Jw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20/go.mod h1:/+6lSiby8TBFpTVXZgKiN/rCfkYXEGvhlM4zCgPpt7w=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27 h1:N2eKFw2S+JWRCtTt0IhIX7uoGGQciD4p6ba+SJv4WEU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27/go.mod h1:RdwFVc7PBYWY33fa2+8T1mSqQ7ZEK4ILpM0wfioDC3w=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 h1:5tXbMJ7Jq0iG65oiMg6tCLsHkSaO2xLXa2EmZ29vaTA=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17/go.mod h1:twV0fKMQuqLY4klyFH56aXNq3AFiA5LO0/frTczEOFE=
github.com/aws/aws-sdk-go-v2/config v1.18.6 h1:iSuEAeervBWMHA7Aaq5hCNfwuN2m7x2VuQCnEbbQg68=
github.com/aws/aws-sdk-go-v2/config v1.18.6/go.mod h1:qyjgnyqpKnNGT+C62zMsrZ/Mn2OodYqwIH0DpXiW8f8=
github.com/aws/aws-sdk-go-v2/credentials v1.13.6 h1:BXOMvv3O82/4JLggIi67WKlTO56f0rliCKBT4CKyf0o=
github.com/aws/aws-sdk-go-v2/credentials v1.13.6/go.mod h1:VbnUvhw31DUu6aiubViixQwWCBNO/st84dhPeOkmdls=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.45 h1:ckFtXy51PT613d/KLKPxFiwRqgGIxDhVbNLof6x/XLo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.45/go.mod h1:xar61xizdVU4pQygvQrNdZY1VCLNcOIvm87KzdZmWrE=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 h1:H/mF2LNWwX00lD6FlYfKpLLZgUW7oIzCBkig78x4Xok=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18/go.mod h1:T2Ku+STrYQ1zIkL1wMvj8P3wWQaaCMKNdz70MT2FLfE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 h1:77b1GfaSuIok5yB/3HYbG+ypWvOJDQ2rVdq943D17R4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21/go.mod h1:sPOz31BVdqeeurKEuUpLNSve4tdCNPluE+070HNcEHI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20 h1:jlgyHbkZQAgAc7VIxJDmtouH8eNjOk2REVAQfVhdaiQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20/go.mod h1:Xs52xaLBqDEKRcAfX/hgjmD3YQ7c/W+BEyfamlO/W2E=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 h1:4K6dbmR0mlp3o4Bo78PnpvzHtYAqEeVMguvEenpMGsI=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20/go.mod h1:1XpDcReIEOHsjwNToDKhIAO3qwLo1BnfbtSqWJa8j7g=
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.5 h1:nRSEQj1JergKTVc8RGkhZvOEGgcvo4fWpDPwGDeg2ok=
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.5/go.mod h1:wcaJTmjKFDW0s+Se55HBNIds6ghdAGoDDw+SGUdrfAk=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.26 h1:ActQgdTNQej/RuUJjB9uxYVLDOvRGtUreXF8L3c8wyg=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.26/go.mod h1:uB9tV79ULEZUXc6Ob18A46KSQ0JDlrplPni9XW6Ot60=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 h1:wihKuqYUlA2T/Rx+yu2s6NDAns8B9DgnRooB1PVhY+Q=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9/go.mod h1:2E/3D/mB8/r2J7nK42daoKP/ooCwbf0q1PznNc+DZTU=
github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 h1:VQFOLQVL3BrKM/NLO/7FiS4vcp5bqK0mGMyk09xLoAY=
github.com/aws/aws-sdk-go-v2/service/sts v1.17.6/go.mod h1:Az3OXXYGyfNwQNsK/31L4R75qFYnO641RZGAoV3uH1c=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22 h1:kv5vRAl00tozRxSnI0IszPWGXsJOyA7hmEUHFYqsyvw=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22/go.mod h1:Od+GU5+Yx41gryN/ZGZzAJMZ9R1yn6lgA0fD5Lo5SkQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 h1:vY5siRXvW5TrOKm2qKEf9tliBfdLxdfy0i02LOcmqUo=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21/go.mod h1:WZvNXT1XuH8dnJM0HvOlvk+RNn7NbAPvA/ACO0QarSc=
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6 h1:W8pLcSn6Uy0eXgDBUUl8M8Kxv7JCoP68ZKTD04OXLEA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6/go.mod h1:L2l2/q76teehcW7YEsgsDjqdsDTERJeX3nOMIFlgGUE=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.27 h1:Nmvn0DJKg00TBmoBweK253Kdsuy4V5Rs68yL/H15uBQ=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.27/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 h1:KCacyVSs/wlcPGx37hcbT3IGYO8P8Jx+TgSDhAXtQMY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8=
github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 h1:9Mtq1KM6nD8/+HStvWcvYnixJ5N85DX+P+OY3kI3W2k=
github.com/aws/aws-sdk-go-v2/service/sts v1.17.7/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -488,8 +488,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20221212164502-fae10dda9338 h1:OvjRkcNHnf6/W5FZXSxODbxwD+X7fspczG7Jn/xQVD4=
golang.org/x/exp v0.0.0-20221212164502-fae10dda9338/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w=
golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -704,8 +704,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
google.golang.org/api v0.104.0 h1:KBfmLRqdZEbwQleFlSLnzpQJwhjpmNOk4cKQIBDZ9mg=
google.golang.org/api v0.104.0/go.mod h1:JCspTXJbBxa5ySXw4UgUqVer7DfVxbvc/CTUFqAED5U=
google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8=
google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

View file

@ -308,7 +308,7 @@ func (a *ArrayBytes) Set(value string) error {
}
// GetOptionalArgOrDefault returns optional arg under the given argIdx.
func (a *ArrayBytes) GetOptionalArgOrDefault(argIdx, defaultValue int) int {
func (a *ArrayBytes) GetOptionalArgOrDefault(argIdx int, defaultValue int64) int64 {
x := *a
if argIdx < len(x) {
return x[argIdx].N

View file

@ -265,8 +265,8 @@ func TestArrayInt_String(t *testing.T) {
}
func TestArrayBytes(t *testing.T) {
expected := []int{10000000, 23, 10240}
result := make([]int, len(fooFlagBytes))
expected := []int64{10000000, 23, 10240}
result := make([]int64, len(fooFlagBytes))
for i, b := range fooFlagBytes {
result[i] = b.N
}
@ -276,11 +276,11 @@ func TestArrayBytes(t *testing.T) {
}
func TestArrayBytes_Set(t *testing.T) {
f := func(s string, expectedValues []int) {
f := func(s string, expectedValues []int64) {
t.Helper()
var a ArrayBytes
_ = a.Set(s)
values := make([]int, len(a))
values := make([]int64, len(a))
for i, v := range a {
values[i] = v.N
}
@ -288,13 +288,13 @@ func TestArrayBytes_Set(t *testing.T) {
t.Fatalf("unexpected values parsed;\ngot\n%d\nwant\n%d", values, expectedValues)
}
}
f("", []int{})
f(`1`, []int{1})
f(`-2,3,10kb`, []int{-2, 3, 10000})
f("", []int64{})
f(`1`, []int64{1})
f(`-2,3,10kb`, []int64{-2, 3, 10000})
}
func TestArrayBytes_GetOptionalArg(t *testing.T) {
f := func(s string, argIdx, defaultValue, expectedValue int) {
f := func(s string, argIdx int, defaultValue, expectedValue int64) {
t.Helper()
var a ArrayBytes
_ = a.Set(s)

View file

@ -3,12 +3,13 @@ package flagutil
import (
"flag"
"fmt"
"math"
"strconv"
"strings"
)
// NewBytes returns new `bytes` flag with the given name, defaultValue and description.
func NewBytes(name string, defaultValue int, description string) *Bytes {
func NewBytes(name string, defaultValue int64, description string) *Bytes {
description += "\nSupports the following optional suffixes for `size` values: KB, MB, GB, TB, KiB, MiB, GiB, TiB"
b := Bytes{
N: defaultValue,
@ -23,11 +24,22 @@ func NewBytes(name string, defaultValue int, description string) *Bytes {
// It supports the following optional suffixes for values: KB, MB, GB, TB, KiB, MiB, GiB, TiB.
type Bytes struct {
// N contains parsed value for the given flag.
N int
N int64
valueString string
}
// IntN returns the stored value capped by int type.
func (b *Bytes) IntN() int {
if b.N > math.MaxInt {
return math.MaxInt
}
if b.N < math.MinInt {
return math.MinInt
}
return int(b.N)
}
// String implements flag.Value interface
func (b *Bytes) String() string {
return b.valueString
@ -42,7 +54,7 @@ func (b *Bytes) Set(value string) error {
if err != nil {
return err
}
b.N = int(f * 1000)
b.N = int64(f * 1000)
b.valueString = value
return nil
case strings.HasSuffix(value, "MB"):
@ -50,7 +62,7 @@ func (b *Bytes) Set(value string) error {
if err != nil {
return err
}
b.N = int(f * 1000 * 1000)
b.N = int64(f * 1000 * 1000)
b.valueString = value
return nil
case strings.HasSuffix(value, "GB"):
@ -58,7 +70,7 @@ func (b *Bytes) Set(value string) error {
if err != nil {
return err
}
b.N = int(f * 1000 * 1000 * 1000)
b.N = int64(f * 1000 * 1000 * 1000)
b.valueString = value
return nil
case strings.HasSuffix(value, "TB"):
@ -66,7 +78,7 @@ func (b *Bytes) Set(value string) error {
if err != nil {
return err
}
b.N = int(f * 1000 * 1000 * 1000 * 1000)
b.N = int64(f * 1000 * 1000 * 1000 * 1000)
b.valueString = value
return nil
case strings.HasSuffix(value, "KiB"):
@ -74,7 +86,7 @@ func (b *Bytes) Set(value string) error {
if err != nil {
return err
}
b.N = int(f * 1024)
b.N = int64(f * 1024)
b.valueString = value
return nil
case strings.HasSuffix(value, "MiB"):
@ -82,7 +94,7 @@ func (b *Bytes) Set(value string) error {
if err != nil {
return err
}
b.N = int(f * 1024 * 1024)
b.N = int64(f * 1024 * 1024)
b.valueString = value
return nil
case strings.HasSuffix(value, "GiB"):
@ -90,7 +102,7 @@ func (b *Bytes) Set(value string) error {
if err != nil {
return err
}
b.N = int(f * 1024 * 1024 * 1024)
b.N = int64(f * 1024 * 1024 * 1024)
b.valueString = value
return nil
case strings.HasSuffix(value, "TiB"):
@ -98,7 +110,7 @@ func (b *Bytes) Set(value string) error {
if err != nil {
return err
}
b.N = int(f * 1024 * 1024 * 1024 * 1024)
b.N = int64(f * 1024 * 1024 * 1024 * 1024)
b.valueString = value
return nil
default:
@ -106,7 +118,7 @@ func (b *Bytes) Set(value string) error {
if err != nil {
return err
}
b.N = int(f)
b.N = int64(f)
b.valueString = value
return nil
}

View file

@ -24,7 +24,7 @@ func TestBytesSetFailure(t *testing.T) {
}
func TestBytesSetSuccess(t *testing.T) {
f := func(value string, expectedResult int) {
f := func(value string, expectedResult int64) {
t.Helper()
var b Bytes
if err := b.Set(value); err != nil {

46
lib/logger/json_fields.go Normal file
View file

@ -0,0 +1,46 @@
package logger
import (
"flag"
"log"
"strings"
)
var loggerJSONFields = flag.String("loggerJSONFields", "", `Allows renaming fields in JSON formatted logs. `+
`Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". `+
"Supported fields: ts, level, caller, msg")
func setLoggerJSONFields() {
if *loggerJSONFields == "" {
return
}
fields := strings.Split(*loggerJSONFields, ",")
for _, f := range fields {
f = strings.TrimSpace(f)
v := strings.Split(f, ":")
if len(v) != 2 {
log.Fatalf("missing ':' delimiter in -loggerJSONFields=%q for %q item", *loggerJSONFields, f)
}
name, value := v[0], v[1]
switch name {
case "ts":
fieldTs = value
case "level":
fieldLevel = value
case "caller":
fieldCaller = value
case "msg":
fieldMsg = value
default:
log.Fatalf("unexpected json field name in -loggerJSONFields=%q: %q; supported values: ts, level, caller, msg", *loggerJSONFields, name)
}
}
}
var (
fieldTs = "ts"
fieldLevel = "level"
fieldCaller = "caller"
fieldMsg = "msg"
)

View file

@ -34,6 +34,7 @@ var (
//
// There is no need in calling Init from tests.
func Init() {
setLoggerJSONFields()
setLoggerOutput()
validateLoggerLevel()
validateLoggerFormat()
@ -239,9 +240,20 @@ func logMessage(level, msg string, skipframes int) {
switch *loggerFormat {
case "json":
if *disableTimestamps {
logMsg = fmt.Sprintf(`{"level":%q,"caller":%q,"msg":%q}`+"\n", levelLowercase, location, msg)
logMsg = fmt.Sprintf(
`{%q:%q,%q:%q,%q:%q}`+"\n",
fieldLevel, levelLowercase,
fieldCaller, location,
fieldMsg, msg,
)
} else {
logMsg = fmt.Sprintf(`{"ts":%q,"level":%q,"caller":%q,"msg":%q}`+"\n", timestamp, levelLowercase, location, msg)
logMsg = fmt.Sprintf(
`{%q:%q,%q:%q,%q:%q,%q:%q}`+"\n",
fieldTs, timestamp,
fieldLevel, levelLowercase,
fieldCaller, location,
fieldMsg, msg,
)
}
default:
if *disableTimestamps {

View file

@ -34,14 +34,14 @@ func initOnce() {
memoryLimit = sysTotalMemory()
if allowedBytes.N <= 0 {
if *allowedPercent < 1 || *allowedPercent > 200 {
logger.Panicf("FATAL: -memory.allowedPercent must be in the range [1...200]; got %g", *allowedPercent)
logger.Fatalf("FATAL: -memory.allowedPercent must be in the range [1...200]; got %g", *allowedPercent)
}
percent := *allowedPercent / 100
allowedMemory = int(float64(memoryLimit) * percent)
remainingMemory = memoryLimit - allowedMemory
logger.Infof("limiting caches to %d bytes, leaving %d bytes to the OS according to -memory.allowedPercent=%g", allowedMemory, remainingMemory, *allowedPercent)
} else {
allowedMemory = allowedBytes.N
allowedMemory = allowedBytes.IntN()
remainingMemory = memoryLimit - allowedMemory
logger.Infof("limiting caches to %d bytes, leaving %d bytes to the OS according to -memory.allowedBytes=%s", allowedMemory, remainingMemory, allowedBytes.String())
}

View file

@ -41,7 +41,7 @@ type FastQueue struct {
// if maxPendingBytes is 0, then the queue size is unlimited.
// Otherwise its size is limited by maxPendingBytes. The oldest data is dropped when the queue
// reaches maxPendingSize.
func MustOpenFastQueue(path, name string, maxInmemoryBlocks, maxPendingBytes int) *FastQueue {
func MustOpenFastQueue(path, name string, maxInmemoryBlocks int, maxPendingBytes int64) *FastQueue {
pq := mustOpen(path, name, maxPendingBytes)
fq := &FastQueue{
pq: pq,

View file

@ -123,7 +123,7 @@ func (q *queue) GetPendingBytes() uint64 {
//
// If maxPendingBytes is greater than 0, then the max queue size is limited by this value.
// The oldest data is deleted when queue size exceeds maxPendingBytes.
func mustOpen(path, name string, maxPendingBytes int) *queue {
func mustOpen(path, name string, maxPendingBytes int64) *queue {
if maxPendingBytes < 0 {
maxPendingBytes = 0
}

View file

@ -128,9 +128,9 @@ func newClient(sw *ScrapeWork) *client {
MaxIdleConnDuration: 2 * sw.ScrapeInterval,
ReadTimeout: sw.ScrapeTimeout,
WriteTimeout: 10 * time.Second,
MaxResponseBodySize: maxScrapeSize.N,
MaxResponseBodySize: maxScrapeSize.IntN(),
MaxIdempotentRequestAttempts: 1,
ReadBufferSize: maxResponseHeadersSize.N,
ReadBufferSize: maxResponseHeadersSize.IntN(),
}
var sc *http.Client
var proxyURLFunc func(*http.Request) (*url.URL, error)

View file

@ -95,6 +95,24 @@ func (sn *ServiceNode) appendTargetLabels(ms []*promutils.Labels, serviceName, t
// in relabeling rules don't have to consider tag positions.
m.Add("__meta_consul_tags", tagSeparator+strings.Join(sn.Service.Tags, tagSeparator)+tagSeparator)
// Expose individual tags via __meta_consul_tag_* labels, so users could move all the tags
// into the discovered scrape target with the following relabeling rule in the way similar to kubernetes_sd_configs:
//
// - action: labelmap
// regex: __meta_consul_tag_(.+)
//
// This solves https://stackoverflow.com/questions/44339461/relabeling-in-prometheus
for _, tag := range sn.Service.Tags {
k := tag
v := ""
if n := strings.IndexByte(tag, '='); n >= 0 {
k = tag[:n]
v = tag[n+1:]
}
m.Add(discoveryutils.SanitizeLabelName("__meta_consul_tag_"+k), v)
m.Add(discoveryutils.SanitizeLabelName("__meta_consul_tagpresent_"+k), "true")
}
for k, v := range sn.Node.Meta {
m.Add(discoveryutils.SanitizeLabelName("__meta_consul_metadata_"+k), v)
}

View file

@ -43,7 +43,7 @@ func TestParseServiceNodesSuccess(t *testing.T) {
"Service": {
"ID": "redis",
"Service": "redis",
"Tags": ["primary"],
"Tags": ["primary","foo=bar"],
"Address": "10.1.10.12",
"TaggedAddresses": {
"lan": {
@ -124,7 +124,11 @@ func TestParseServiceNodesSuccess(t *testing.T) {
"__meta_consul_service_port": "8000",
"__meta_consul_tagged_address_lan": "10.1.10.12",
"__meta_consul_tagged_address_wan": "10.1.10.12",
"__meta_consul_tags": ",primary,",
"__meta_consul_tag_foo": "bar",
"__meta_consul_tag_primary": "",
"__meta_consul_tagpresent_foo": "true",
"__meta_consul_tagpresent_primary": "true",
"__meta_consul_tags": ",primary,foo=bar,",
}),
}
discoveryutils.TestEqualLabelss(t, labelss, expectedLabelss)

View file

@ -90,31 +90,45 @@ func newDebugRelabelSteps(pcs *promrelabel.ParsedConfigs, labels *promutils.Labe
labels.Labels = labelsResult
outStr := promrelabel.LabelsToString(labels.GetLabels())
// Add missing instance label
if isTargetRelabel && labels.Get("instance") == "" {
address := labels.Get("__address__")
if address != "" {
inStr := outStr
labels.Add("instance", address)
outStr = promrelabel.LabelsToString(labels.GetLabels())
if isTargetRelabel {
// Add missing instance label
if labels.Get("instance") == "" {
address := labels.Get("__address__")
if address != "" {
inStr := outStr
labels.Add("instance", address)
outStr = promrelabel.LabelsToString(labels.GetLabels())
dss = append(dss, promrelabel.DebugStep{
Rule: "add missing instance label from __address__ label",
In: inStr,
Out: outStr,
})
}
}
// Remove labels with __ prefix
inStr := outStr
labels.RemoveLabelsWithDoubleUnderscorePrefix()
outStr = promrelabel.LabelsToString(labels.GetLabels())
if inStr != outStr {
dss = append(dss, promrelabel.DebugStep{
Rule: "add missing instance label from __address__ label",
Rule: "remove labels with __ prefix",
In: inStr,
Out: outStr,
})
}
} else {
// Remove labels with __ prefix except of __name__
inStr := outStr
labels.Labels = promrelabel.FinalizeLabels(labels.Labels[:0], labels.Labels)
outStr = promrelabel.LabelsToString(labels.GetLabels())
if inStr != outStr {
dss = append(dss, promrelabel.DebugStep{
Rule: "remove labels with __ prefix except of __name__",
In: inStr,
Out: outStr,
})
}
}
// Remove labels with __ prefix
inStr := outStr
labels.RemoveLabelsWithDoubleUnderscorePrefix()
outStr = promrelabel.LabelsToString(labels.GetLabels())
if inStr != outStr {
dss = append(dss, promrelabel.DebugStep{
Rule: "remove labels with __ prefix",
In: inStr,
Out: outStr,
})
}
// There is no need in labels' sorting, since promrelabel.LabelsToString() automatically sorts labels.

View file

@ -235,7 +235,7 @@ func (sw *scrapeWork) loadLastScrape() string {
}
func (sw *scrapeWork) storeLastScrape(lastScrape []byte) {
mustCompress := minResponseSizeForStreamParse.N > 0 && len(lastScrape) >= minResponseSizeForStreamParse.N
mustCompress := minResponseSizeForStreamParse.N > 0 && len(lastScrape) >= minResponseSizeForStreamParse.IntN()
if mustCompress {
sw.lastScrapeCompressed = encoding.CompressZSTDLevel(sw.lastScrapeCompressed[:0], lastScrape, 1)
sw.lastScrape = nil
@ -384,7 +384,7 @@ func (sw *scrapeWork) mustSwitchToStreamParseMode(responseSize int) bool {
if minResponseSizeForStreamParse.N <= 0 {
return false
}
return sw.Config.canSwitchToStreamParseMode() && responseSize >= minResponseSizeForStreamParse.N
return sw.Config.canSwitchToStreamParseMode() && responseSize >= minResponseSizeForStreamParse.IntN()
}
// getTargetResponse() fetches response from sw target in the same way as when scraping the target.

View file

@ -76,7 +76,7 @@ func (ctx *streamContext) Read() bool {
if ctx.err != nil || ctx.hasCallbackError() {
return false
}
ctx.reqBuf, ctx.tailBuf, ctx.err = common.ReadLinesBlockExt(ctx.br, ctx.reqBuf, ctx.tailBuf, maxLineSize.N)
ctx.reqBuf, ctx.tailBuf, ctx.err = common.ReadLinesBlockExt(ctx.br, ctx.reqBuf, ctx.tailBuf, maxLineSize.IntN())
if ctx.err != nil {
if ctx.err != io.EOF {
readErrors.Inc()

View file

@ -37,7 +37,7 @@ func ParseStream(r io.Reader, callback func(tss []prompb.TimeSeries) error) erro
if err != nil {
return fmt.Errorf("cannot decompress request with length %d: %w", len(ctx.reqBuf.B), err)
}
if len(bb.B) > maxInsertRequestSize.N {
if int64(len(bb.B)) > maxInsertRequestSize.N {
return fmt.Errorf("too big unpacked request; mustn't exceed `-maxInsertRequestSize=%d` bytes; got %d bytes", maxInsertRequestSize.N, len(bb.B))
}
wr := getWriteRequest()

View file

@ -52,7 +52,7 @@ func (ctx *streamContext) Read() bool {
if ctx.err != nil || ctx.hasCallbackError() {
return false
}
ctx.reqBuf, ctx.tailBuf, ctx.err = common.ReadLinesBlockExt(ctx.br, ctx.reqBuf, ctx.tailBuf, maxLineLen.N)
ctx.reqBuf, ctx.tailBuf, ctx.err = common.ReadLinesBlockExt(ctx.br, ctx.reqBuf, ctx.tailBuf, maxLineLen.IntN())
if ctx.err != nil {
if ctx.err != io.EOF {
readErrors.Inc()

View file

@ -443,8 +443,8 @@ func marshalMetricIDs(dst []byte, metricIDs []uint64) []byte {
var buf []byte
sh := (*reflect.SliceHeader)(unsafe.Pointer(&buf))
sh.Data = uintptr(unsafe.Pointer(&metricIDs[0]))
sh.Cap = sh.Len
sh.Len = 8 * len(metricIDs)
sh.Cap = 8 * len(metricIDs)
sh.Len = sh.Cap
dst = append(dst, buf...)
return dst
}
@ -741,7 +741,7 @@ func (db *indexDB) SearchLabelNamesWithFiltersOnTimeRange(qt *querytracer.Tracer
return nil, err
}
ok := db.doExtDB(func(extDB *indexDB) {
db.doExtDB(func(extDB *indexDB) {
qtChild := qt.NewChild("search for label names in the previous indexdb")
lnsLen := len(lns)
is := extDB.getIndexSearch(deadline)
@ -749,7 +749,7 @@ func (db *indexDB) SearchLabelNamesWithFiltersOnTimeRange(qt *querytracer.Tracer
extDB.putIndexSearch(is)
qtChild.Donef("found %d additional label names", len(lns)-lnsLen)
})
if ok && err != nil {
if err != nil {
return nil, err
}
@ -930,7 +930,7 @@ func (db *indexDB) SearchLabelValuesWithFiltersOnTimeRange(qt *querytracer.Trace
if err != nil {
return nil, err
}
ok := db.doExtDB(func(extDB *indexDB) {
db.doExtDB(func(extDB *indexDB) {
qtChild := qt.NewChild("search for label values in the previous indexdb")
lvsLen := len(lvs)
is := extDB.getIndexSearch(deadline)
@ -938,7 +938,7 @@ func (db *indexDB) SearchLabelValuesWithFiltersOnTimeRange(qt *querytracer.Trace
extDB.putIndexSearch(is)
qtChild.Donef("found %d additional label values", len(lvs)-lvsLen)
})
if ok && err != nil {
if err != nil {
return nil, err
}
@ -1132,14 +1132,14 @@ func (db *indexDB) SearchTagValueSuffixes(qt *querytracer.Tracer, tr TimeRange,
return nil, err
}
if len(tvss) < maxTagValueSuffixes {
ok := db.doExtDB(func(extDB *indexDB) {
db.doExtDB(func(extDB *indexDB) {
is := extDB.getIndexSearch(deadline)
qtChild := qt.NewChild("search tag value suffixes in the previous indexdb")
err = is.searchTagValueSuffixesForTimeRange(tvss, tr, tagKey, tagValuePrefix, delimiter, maxTagValueSuffixes)
qtChild.Done()
extDB.putIndexSearch(is)
})
if ok && err != nil {
if err != nil {
return nil, err
}
}
@ -1285,12 +1285,12 @@ func (db *indexDB) GetSeriesCount(deadline uint64) (uint64, error) {
}
var nExt uint64
ok := db.doExtDB(func(extDB *indexDB) {
db.doExtDB(func(extDB *indexDB) {
is := extDB.getIndexSearch(deadline)
nExt, err = is.getSeriesCount()
extDB.putIndexSearch(is)
})
if ok && err != nil {
if err != nil {
return 0, fmt.Errorf("error when searching in extDB: %w", err)
}
return n + nExt, nil
@ -1350,14 +1350,14 @@ func (db *indexDB) GetTSDBStatus(qt *querytracer.Tracer, tfss []*TagFilters, dat
if status.hasEntries() {
return status, nil
}
ok := db.doExtDB(func(extDB *indexDB) {
db.doExtDB(func(extDB *indexDB) {
qtChild := qt.NewChild("collect tsdb stats in the previous indexdb")
is := extDB.getIndexSearch(deadline)
status, err = is.getTSDBStatus(qtChild, tfss, date, focusLabel, topN, maxMetrics)
qtChild.Done()
extDB.putIndexSearch(is)
})
if ok && err != nil {
if err != nil {
return nil, fmt.Errorf("error when obtaining TSDB status from extDB: %w", err)
}
return status, nil
@ -1663,16 +1663,15 @@ func (db *indexDB) DeleteTSIDs(qt *querytracer.Tracer, tfss []*TagFilters) (int,
// Delete TSIDs in the extDB.
deletedCount := len(metricIDs)
if db.doExtDB(func(extDB *indexDB) {
db.doExtDB(func(extDB *indexDB) {
var n int
qtChild := qt.NewChild("deleting series from the previos indexdb")
n, err = extDB.DeleteTSIDs(qtChild, tfss)
qtChild.Donef("deleted %d series", n)
deletedCount += n
}) {
if err != nil {
return deletedCount, fmt.Errorf("cannot delete tsids in extDB: %w", err)
}
})
if err != nil {
return deletedCount, fmt.Errorf("cannot delete tsids in extDB: %w", err)
}
return deletedCount, nil
}
@ -1776,7 +1775,7 @@ func (db *indexDB) searchMetricIDs(qt *querytracer.Tracer, tfss []*TagFilters, t
qtChild.Done()
var extMetricIDs []uint64
if db.doExtDB(func(extDB *indexDB) {
db.doExtDB(func(extDB *indexDB) {
qtChild := qt.NewChild("search for metricIDs in the previous indexdb")
defer qtChild.Done()
@ -1794,10 +1793,9 @@ func (db *indexDB) searchMetricIDs(qt *querytracer.Tracer, tfss []*TagFilters, t
extMetricIDs, err = is.searchMetricIDs(qtChild, tfss, tr, maxMetrics)
extDB.putIndexSearch(is)
extDB.putMetricIDsToTagFiltersCache(qtChild, extMetricIDs, tfKeyExtBuf.B)
}) {
if err != nil {
return nil, fmt.Errorf("error when searching for metricIDs in the previous indexdb: %s", err)
}
})
if err != nil {
return nil, fmt.Errorf("error when searching for metricIDs in the previous indexdb: %s", err)
}
// Merge localMetricIDs with extMetricIDs.
@ -1847,43 +1845,83 @@ func (db *indexDB) getTSIDsFromMetricIDs(qt *querytracer.Tracer, metricIDs []uin
return nil, nil
}
tsids := make([]TSID, len(metricIDs))
is := db.getIndexSearch(deadline)
defer db.putIndexSearch(is)
var extMetricIDs []uint64
i := 0
for loopsPaceLimiter, metricID := range metricIDs {
if loopsPaceLimiter&paceLimiterSlowIterationsMask == 0 {
if err := checkSearchDeadlineAndPace(is.deadline); err != nil {
return nil, err
err := func() error {
is := db.getIndexSearch(deadline)
defer db.putIndexSearch(is)
for loopsPaceLimiter, metricID := range metricIDs {
if loopsPaceLimiter&paceLimiterSlowIterationsMask == 0 {
if err := checkSearchDeadlineAndPace(is.deadline); err != nil {
return err
}
}
}
// Try obtaining TSIDs from MetricID->TSID cache. This is much faster
// than scanning the mergeset if it contains a lot of metricIDs.
tsid := &tsids[i]
err := is.db.getFromMetricIDCache(tsid, metricID)
if err == nil {
// Fast path - the tsid for metricID is found in cache.
i++
continue
}
if err != io.EOF {
return nil, err
}
if err := is.getTSIDByMetricID(tsid, metricID); err != nil {
if err == io.EOF {
// Cannot find TSID for the given metricID.
// This may be the case on incomplete indexDB
// due to snapshot or due to unflushed entries.
// Just increment errors counter and skip it.
atomic.AddUint64(&is.db.missingTSIDsForMetricID, 1)
// Try obtaining TSIDs from MetricID->TSID cache. This is much faster
// than scanning the mergeset if it contains a lot of metricIDs.
tsid := &tsids[i]
err := is.db.getFromMetricIDCache(tsid, metricID)
if err == nil {
// Fast path - the tsid for metricID is found in cache.
i++
continue
}
return nil, fmt.Errorf("cannot find tsid %d out of %d for metricID %d: %w", i, len(metricIDs), metricID, err)
if err != io.EOF {
return err
}
if err := is.getTSIDByMetricID(tsid, metricID); err != nil {
if err == io.EOF {
// Postpone searching for the metricID in the extDB.
extMetricIDs = append(extMetricIDs, metricID)
continue
}
return fmt.Errorf("cannot find tsid %d out of %d for metricID %d: %w", i, len(metricIDs), metricID, err)
}
is.db.putToMetricIDCache(metricID, tsid)
i++
}
is.db.putToMetricIDCache(metricID, tsid)
i++
return nil
}()
if err != nil {
return nil, fmt.Errorf("error when searching for TISDs by metricIDs in the current indexdb: %w", err)
}
tsidsFound := i
qt.Printf("found %d tsids for %d metricIDs in the current indexdb", tsidsFound, len(metricIDs))
// Search for extMetricIDs in the extDB.
db.doExtDB(func(extDB *indexDB) {
is := extDB.getIndexSearch(deadline)
defer extDB.putIndexSearch(is)
for loopsPaceLimiter, metricID := range extMetricIDs {
if loopsPaceLimiter&paceLimiterSlowIterationsMask == 0 {
if err = checkSearchDeadlineAndPace(is.deadline); err != nil {
return
}
}
// There is no need in searching for TSIDs in MetricID->TSID cache, since
// this has been already done in the loop above (the MetricID->TSID cache is global).
tsid := &tsids[i]
if err = is.getTSIDByMetricID(tsid, metricID); err != nil {
if err == io.EOF {
// Cannot find TSID for the given metricID.
// This may be the case on incomplete indexDB
// due to snapshot or due to unflushed entries.
// Just increment errors counter and skip it.
atomic.AddUint64(&is.db.missingTSIDsForMetricID, 1)
continue
}
err = fmt.Errorf("cannot find tsid for metricID=%d: %w", metricID, err)
}
is.db.putToMetricIDCache(metricID, tsid)
i++
}
})
if err != nil {
return nil, fmt.Errorf("error when searching for TSIDs by metricIDs in the previous indexdb: %w", err)
}
qt.Printf("found %d tsids for %d metricIDs in the previous indexdb", i-tsidsFound, len(extMetricIDs))
tsids = tsids[:i]
qt.Printf("load %d tsids from %d metricIDs", len(tsids), len(metricIDs))
qt.Printf("load %d tsids for %d metricIDs from both current and previous indexdb", len(tsids), len(metricIDs))
// Sort the found tsids, since they must be passed to TSID search
// in the sorted order.

View file

@ -91,6 +91,10 @@ func TestMergeSortedMetricIDs(t *testing.T) {
f([]uint64{2, 3, 4, 6, 7, 8, 9}, []uint64{1, 2, 3, 4, 5, 6, 7})
f([]uint64{1, 2, 3, 4, 6, 7, 8, 9}, []uint64{1, 2, 3, 4, 5, 6, 7})
f([]uint64{1, 2, 3, 4, 6, 7, 8, 9}, []uint64{2, 3, 4, 5, 6, 7})
f([]uint64{}, []uint64{1, 2, 3})
f([]uint64{0}, []uint64{1, 2, 3})
f([]uint64{1}, []uint64{1, 2, 3})
f([]uint64{1, 2}, []uint64{3, 4})
}
func TestReverseBytes(t *testing.T) {

View file

@ -69,7 +69,7 @@ func (ps *partSearch) Init(p *part, tsids []TSID, tr TimeRange) {
if isInTest && !sort.SliceIsSorted(tsids, func(i, j int) bool { return tsids[i].Less(&tsids[j]) }) {
logger.Panicf("BUG: tsids must be sorted; got %+v", tsids)
}
// take ownership of of tsids.
// take ownership of tsids.
ps.tsids = tsids
}
ps.tr = tr
@ -120,14 +120,38 @@ func (ps *partSearch) nextTSID() bool {
return true
}
func (ps *partSearch) skipTSIDsSmallerThan(tsid *TSID) bool {
if !ps.BlockRef.bh.TSID.Less(tsid) {
return true
}
if !ps.nextTSID() {
return false
}
if !ps.BlockRef.bh.TSID.Less(tsid) {
// Fast path: the next TSID isn't smaller than the tsid.
return true
}
// Slower path - binary search for the next TSID, which isn't smaller than the tsid.
tsids := ps.tsids[ps.tsidIdx:]
ps.tsidIdx += sort.Search(len(tsids), func(i int) bool {
return !tsids[i].Less(tsid)
})
if ps.tsidIdx >= len(ps.tsids) {
ps.tsidIdx = len(ps.tsids)
ps.err = io.EOF
return false
}
ps.BlockRef.bh.TSID = ps.tsids[ps.tsidIdx]
ps.tsidIdx++
return true
}
func (ps *partSearch) nextBHS() bool {
for len(ps.metaindex) > 0 {
// Optimization: skip tsid values smaller than the minimum value
// from ps.metaindex.
for ps.BlockRef.bh.TSID.Less(&ps.metaindex[0].TSID) {
if !ps.nextTSID() {
return false
}
// Optimization: skip tsid values smaller than the minimum value from ps.metaindex.
if !ps.skipTSIDsSmallerThan(&ps.metaindex[0].TSID) {
return false
}
// Invariant: ps.BlockRef.bh.TSID >= ps.metaindex[0].TSID
@ -232,22 +256,24 @@ func (ps *partSearch) searchBHS() bool {
for len(bhs) > 0 {
// Skip block headers with tsids smaller than the given tsid.
tsid := &ps.BlockRef.bh.TSID
n := sort.Search(len(bhs), func(i int) bool {
return !bhs[i].TSID.Less(tsid)
})
if n == len(bhs) {
// Nothing found.
break
if bhs[0].TSID.Less(tsid) {
n := sort.Search(len(bhs), func(i int) bool {
return !bhs[i].TSID.Less(tsid)
})
if n == len(bhs) {
// Nothing found.
break
}
bhs = bhs[n:]
}
bhs = bhs[n:]
bh := &bhs[0]
// Invariant: tsid <= bh.TSID
bh := &bhs[0]
if bh.TSID.MetricID != tsid.MetricID {
// tsid < bh.TSID: no more blocks with the given tsid.
// Proceed to the next (bigger) tsid.
if !ps.nextTSID() {
if !ps.skipTSIDsSmallerThan(&bh.TSID) {
return false
}
continue

View file

@ -0,0 +1,61 @@
package storage
import (
"fmt"
"testing"
)
func BenchmarkPartSearch(b *testing.B) {
for _, sparseness := range []int{1, 2, 10, 100} {
b.Run(fmt.Sprintf("sparseness-%d", sparseness), func(b *testing.B) {
benchmarkPartSearchWithSparseness(b, sparseness)
})
}
}
func benchmarkPartSearchWithSparseness(b *testing.B, sparseness int) {
blocksCount := 100000
rows := make([]rawRow, blocksCount)
for i := 0; i < blocksCount; i++ {
r := &rows[i]
r.PrecisionBits = defaultPrecisionBits
r.TSID.MetricID = uint64(i * sparseness)
r.Timestamp = int64(i) * 1000
r.Value = float64(i)
}
tr := TimeRange{
MinTimestamp: rows[0].Timestamp,
MaxTimestamp: rows[len(rows)-1].Timestamp,
}
p := newTestPart(rows)
for _, tsidsCount := range []int{100, 1000, 10000, 100000} {
b.Run(fmt.Sprintf("tsids-%d", tsidsCount), func(b *testing.B) {
tsids := make([]TSID, tsidsCount)
for i := 0; i < tsidsCount; i++ {
tsids[i].MetricID = uint64(i)
}
benchmarkPartSearch(b, p, tsids, tr, sparseness)
})
}
}
func benchmarkPartSearch(b *testing.B, p *part, tsids []TSID, tr TimeRange, sparseness int) {
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
var ps partSearch
for pb.Next() {
blocksRead := 0
ps.Init(p, tsids, tr)
for ps.NextBlock() {
blocksRead++
}
if err := ps.Error(); err != nil {
panic(fmt.Errorf("BUG: unexpected error: %s", err))
}
blocksWant := len(tsids) / sparseness
if blocksRead != blocksWant {
panic(fmt.Errorf("BUG: unexpected blocks read; got %d; want %d", blocksRead, blocksWant))
}
}
})
}

View file

@ -613,7 +613,7 @@ func (s *Storage) UpdateMetrics(m *Metrics) {
// SetFreeDiskSpaceLimit sets the minimum free disk space size of current storage path
//
// The function must be called before opening or creating any storage.
func SetFreeDiskSpaceLimit(bytes int) {
func SetFreeDiskSpaceLimit(bytes int64) {
freeDiskSpaceLimitBytes = uint64(bytes)
}

View file

@ -1,5 +1,12 @@
# Changes
## [0.2.3](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.2...compute/metadata/v0.2.3) (2022-12-15)
### Bug Fixes
* **compute/metadata:** Switch DNS lookup to an absolute lookup ([119b410](https://github.com/googleapis/google-cloud-go/commit/119b41060c7895e45e48aee5621ad35607c4d021)), refs [#7165](https://github.com/googleapis/google-cloud-go/issues/7165)
## [0.2.2](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.1...compute/metadata/v0.2.2) (2022-12-01)

View file

@ -147,7 +147,7 @@ func testOnGCE() bool {
go func() {
resolver := &net.Resolver{}
addrs, err := resolver.LookupHost(ctx, "metadata.google.internal")
addrs, err := resolver.LookupHost(ctx, "metadata.google.internal.")
if err != nil || len(addrs) == 0 {
resc <- false
return

View file

@ -1,5 +1,12 @@
# Changes
## [0.9.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.8.0...iam/v0.9.0) (2022-12-15)
### Features
* **iam:** Rewrite iam sigs and update proto import ([#7137](https://github.com/googleapis/google-cloud-go/issues/7137)) ([ad67fa3](https://github.com/googleapis/google-cloud-go/commit/ad67fa36c263c161226f7fecbab5221592374dca))
## [0.8.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.7.0...iam/v0.8.0) (2022-12-05)

View file

@ -15,7 +15,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.21.5
// protoc v3.21.9
// source: google/iam/v1/iam_policy.proto
package iampb

View file

@ -15,7 +15,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.21.5
// protoc v3.21.9
// source: google/iam/v1/options.proto
package iampb

View file

@ -15,7 +15,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.21.5
// protoc v3.21.9
// source: google/iam/v1/policy.proto
package iampb

View file

@ -10,3 +10,5 @@ Gemfile.lock
.gradle/
build/
.idea/
bin/
.vscode/

View file

@ -1,3 +1,145 @@
# Release (2022-12-15)
## General Highlights
* **Dependency Update**: Updated to the latest SDK module versions
## Module Highlights
* `github.com/aws/aws-sdk-go-v2`: v1.17.3
* **Bug Fix**: Unify logic between shared config and in finding home directory
* `github.com/aws/aws-sdk-go-v2/config`: [v1.18.5](config/CHANGELOG.md#v1185-2022-12-15)
* **Bug Fix**: Unify logic between shared config and in finding home directory
* `github.com/aws/aws-sdk-go-v2/credentials`: [v1.13.5](credentials/CHANGELOG.md#v1135-2022-12-15)
* **Bug Fix**: Unify logic between shared config and in finding home directory
* `github.com/aws/aws-sdk-go-v2/service/backupgateway`: [v1.8.0](service/backupgateway/CHANGELOG.md#v180-2022-12-15)
* **Feature**: This release adds support for VMware vSphere tags, enabling customer to protect VMware virtual machines using tag-based policies for AWS tags mapped from vSphere tags. This release also adds support for customer-accessible gateway-hypervisor interaction log and upload bandwidth rate limit schedule.
* `github.com/aws/aws-sdk-go-v2/service/connect`: [v1.40.0](service/connect/CHANGELOG.md#v1400-2022-12-15)
* **Feature**: Added support for "English - New Zealand" and "English - South African" to be used with Amazon Connect Custom Vocabulary APIs.
* `github.com/aws/aws-sdk-go-v2/service/ecs`: [v1.21.0](service/ecs/CHANGELOG.md#v1210-2022-12-15)
* **Feature**: This release adds support for container port ranges in ECS, a new capability that allows customers to provide container port ranges to simplify use cases where multiple ports are in use in a container. This release updates TaskDefinition mutation APIs and the Task description APIs.
* `github.com/aws/aws-sdk-go-v2/service/eks`: [v1.26.0](service/eks/CHANGELOG.md#v1260-2022-12-15)
* **Feature**: Add support for Windows managed nodes groups.
* `github.com/aws/aws-sdk-go-v2/service/glue`: [v1.38.0](service/glue/CHANGELOG.md#v1380-2022-12-15)
* **Feature**: This release adds support for AWS Glue Crawler with native DeltaLake tables, allowing Crawlers to classify Delta Lake format tables and catalog them for query engines to query against.
* `github.com/aws/aws-sdk-go-v2/service/kinesis`: [v1.16.0](service/kinesis/CHANGELOG.md#v1160-2022-12-15)
* **Feature**: Added StreamARN parameter for Kinesis Data Streams APIs. Added a new opaque pagination token for ListStreams. SDKs will auto-generate Account Endpoint when accessing Kinesis Data Streams.
* `github.com/aws/aws-sdk-go-v2/service/location`: [v1.19.5](service/location/CHANGELOG.md#v1195-2022-12-15)
* **Documentation**: This release adds support for a new style, "VectorOpenDataStandardLight" which can be used with the new data source, "Open Data Maps (Preview)".
* `github.com/aws/aws-sdk-go-v2/service/m2`: [v1.2.0](service/m2/CHANGELOG.md#v120-2022-12-15)
* **Feature**: Adds an optional create-only `KmsKeyId` property to Environment and Application resources.
* `github.com/aws/aws-sdk-go-v2/service/sagemaker`: [v1.57.0](service/sagemaker/CHANGELOG.md#v1570-2022-12-15)
* **Feature**: SageMaker Inference Recommender now allows customers to load tests their models on various instance types using private VPC.
* `github.com/aws/aws-sdk-go-v2/service/securityhub`: [v1.26.0](service/securityhub/CHANGELOG.md#v1260-2022-12-15)
* **Feature**: Added new resource details objects to ASFF, including resources for AwsEc2LaunchTemplate, AwsSageMakerNotebookInstance, AwsWafv2WebAcl and AwsWafv2RuleGroup.
* `github.com/aws/aws-sdk-go-v2/service/translate`: [v1.16.0](service/translate/CHANGELOG.md#v1160-2022-12-15)
* **Feature**: Raised the input byte size limit of the Text field in the TranslateText API to 10000 bytes.
# Release (2022-12-14)
## Module Highlights
* `github.com/aws/aws-sdk-go-v2/service/cloudwatch`: [v1.23.0](service/cloudwatch/CHANGELOG.md#v1230-2022-12-14)
* **Feature**: Adding support for Metrics Insights Alarms
* `github.com/aws/aws-sdk-go-v2/service/costexplorer`: [v1.24.0](service/costexplorer/CHANGELOG.md#v1240-2022-12-14)
* **Feature**: This release supports percentage-based thresholds on Cost Anomaly Detection alert subscriptions.
* `github.com/aws/aws-sdk-go-v2/service/networkmanager`: [v1.16.0](service/networkmanager/CHANGELOG.md#v1160-2022-12-14)
* **Feature**: Appliance Mode support for AWS Cloud WAN.
* `github.com/aws/aws-sdk-go-v2/service/redshiftdata`: [v1.17.0](service/redshiftdata/CHANGELOG.md#v1170-2022-12-14)
* **Feature**: This release adds a new --client-token field to ExecuteStatement and BatchExecuteStatement operations. Customers can now run queries with the additional client token parameter to ensures idempotency.
* `github.com/aws/aws-sdk-go-v2/service/sagemakermetrics`: [v1.0.1](service/sagemakermetrics/CHANGELOG.md#v101-2022-12-14)
* **Documentation**: Update SageMaker Metrics documentation.
# Release (2022-12-13)
## Module Highlights
* `github.com/aws/aws-sdk-go-v2/service/cloudtrail`: [v1.21.0](service/cloudtrail/CHANGELOG.md#v1210-2022-12-13)
* **Feature**: Merging mainline branch for service model into mainline release branch. There are no new APIs.
* `github.com/aws/aws-sdk-go-v2/service/marketplaceentitlementservice`: [v1.11.21](service/marketplaceentitlementservice/CHANGELOG.md#v11121-2022-12-13)
* **Bug Fix**: Fixing a shape type in the marketplaceentitlementservice client
* `github.com/aws/aws-sdk-go-v2/service/rds`: [v1.35.0](service/rds/CHANGELOG.md#v1350-2022-12-13)
* **Feature**: This deployment adds ClientPasswordAuthType field to the Auth structure of the DBProxy.
# Release (2022-12-12)
## Module Highlights
* `github.com/aws/aws-sdk-go-v2/service/customerprofiles`: [v1.21.0](service/customerprofiles/CHANGELOG.md#v1210-2022-12-12)
* **Feature**: This release allows custom strings in PartyType and Gender through 2 new attributes in the CreateProfile and UpdateProfile APIs: PartyTypeString and GenderString.
* `github.com/aws/aws-sdk-go-v2/service/ec2`: [v1.76.0](service/ec2/CHANGELOG.md#v1760-2022-12-12)
* **Feature**: This release updates DescribeFpgaImages to show supported instance types of AFIs in its response.
* `github.com/aws/aws-sdk-go-v2/service/kinesisvideo`: [v1.13.0](service/kinesisvideo/CHANGELOG.md#v1130-2022-12-12)
* **Feature**: This release adds support for public preview of Kinesis Video Stream at Edge enabling customers to provide configuration for the Kinesis Video Stream EdgeAgent running on an on-premise IoT device. Customers can now locally record from cameras and stream videos to the cloud on configured schedule.
* `github.com/aws/aws-sdk-go-v2/service/lookoutvision`: [v1.14.13](service/lookoutvision/CHANGELOG.md#v11413-2022-12-12)
* **Documentation**: This documentation update adds kms:GenerateDataKey as a required permission to StartModelPackagingJob.
* `github.com/aws/aws-sdk-go-v2/service/migrationhubrefactorspaces`: [v1.7.0](service/migrationhubrefactorspaces/CHANGELOG.md#v170-2022-12-12)
* **Feature**: This release adds support for Lambda alias service endpoints. Lambda alias ARNs can now be passed into CreateService.
* `github.com/aws/aws-sdk-go-v2/service/rds`: [v1.34.0](service/rds/CHANGELOG.md#v1340-2022-12-12)
* **Feature**: Update the RDS API model to support copying option groups during the CopyDBSnapshot operation
* `github.com/aws/aws-sdk-go-v2/service/rekognition`: [v1.22.0](service/rekognition/CHANGELOG.md#v1220-2022-12-12)
* **Feature**: Adds support for "aliases" and "categories", inclusion and exclusion filters for labels and label categories, and aggregating labels by video segment timestamps for Stored Video Label Detection APIs.
* `github.com/aws/aws-sdk-go-v2/service/sagemakermetrics`: [v1.0.0](service/sagemakermetrics/CHANGELOG.md#v100-2022-12-12)
* **Release**: New AWS service client module
* **Feature**: This release introduces support SageMaker Metrics APIs.
* `github.com/aws/aws-sdk-go-v2/service/wafv2`: [v1.23.3](service/wafv2/CHANGELOG.md#v1233-2022-12-12)
* **Documentation**: Documents the naming requirement for logging destinations that you use with web ACLs.
# Release (2022-12-09)
## Module Highlights
* `github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs`: [v1.17.2](service/cloudwatchlogs/CHANGELOG.md#v1172-2022-12-09)
* **Documentation**: Doc-only update for CloudWatch Logs, for Tagging Permissions clarifications
* `github.com/aws/aws-sdk-go-v2/service/iotfleetwise`: [v1.1.0](service/iotfleetwise/CHANGELOG.md#v110-2022-12-09)
* **Feature**: Deprecated assignedValue property for actuators and attributes. Added a message to invalid nodes and invalid decoder manifest exceptions.
* `github.com/aws/aws-sdk-go-v2/service/medialive`: [v1.26.0](service/medialive/CHANGELOG.md#v1260-2022-12-09)
* **Feature**: Link devices now support buffer size (latency) configuration. A higher latency value means a longer delay in transmitting from the device to MediaLive, but improved resiliency. A lower latency value means a shorter delay, but less resiliency.
* `github.com/aws/aws-sdk-go-v2/service/mediapackagevod`: [v1.20.0](service/mediapackagevod/CHANGELOG.md#v1200-2022-12-09)
* **Feature**: This release provides the approximate number of assets in a packaging group.
# Release (2022-12-08)
## Module Highlights
* `github.com/aws/aws-sdk-go-v2/service/autoscaling`: [v1.25.0](service/autoscaling/CHANGELOG.md#v1250-2022-12-08)
* **Feature**: Adds support for metric math for target tracking scaling policies, saving you the cost and effort of publishing a custom metric to CloudWatch. Also adds support for VPC Lattice by adding the Attach/Detach/DescribeTrafficSources APIs and a new health check type to the CreateAutoScalingGroup API.
* `github.com/aws/aws-sdk-go-v2/service/iottwinmaker`: [v1.9.0](service/iottwinmaker/CHANGELOG.md#v190-2022-12-08)
* **Feature**: This release adds the following new features: 1) New APIs for managing a continuous sync of assets and asset models from AWS IoT SiteWise. 2) Support user friendly names for component types (ComponentTypeName) and properties (DisplayName).
* `github.com/aws/aws-sdk-go-v2/service/migrationhubstrategy`: [v1.6.0](service/migrationhubstrategy/CHANGELOG.md#v160-2022-12-08)
* **Feature**: This release adds known application filtering, server selection for assessments, support for potential recommendations, and indications for configuration and assessment status. For more information, see the AWS Migration Hub documentation at https://docs.aws.amazon.com/migrationhub/index.html
# Release (2022-12-07)
## Module Highlights
* `github.com/aws/aws-sdk-go-v2/service/cloudfront`: [v1.22.0](service/cloudfront/CHANGELOG.md#v1220-2022-12-07)
* **Feature**: Introducing UpdateDistributionWithStagingConfig that can be used to promote the staging configuration to the production.
* `github.com/aws/aws-sdk-go-v2/service/costexplorer`: [v1.23.0](service/costexplorer/CHANGELOG.md#v1230-2022-12-07)
* **Feature**: This release adds the LinkedAccountName field to the GetAnomalies API response under RootCause
* `github.com/aws/aws-sdk-go-v2/service/eks`: [v1.25.0](service/eks/CHANGELOG.md#v1250-2022-12-07)
* **Feature**: Adds support for EKS add-ons configurationValues fields and DescribeAddonConfiguration function
* `github.com/aws/aws-sdk-go-v2/service/kms`: [v1.19.2](service/kms/CHANGELOG.md#v1192-2022-12-07)
* **Documentation**: Updated examples and exceptions for External Key Store (XKS).
# Release (2022-12-06)
## Module Highlights
* `github.com/aws/aws-sdk-go-v2/service/billingconductor`: [v1.3.0](service/billingconductor/CHANGELOG.md#v130-2022-12-06)
* **Feature**: This release adds the Tiering Pricing Rule feature.
* `github.com/aws/aws-sdk-go-v2/service/connect`: [v1.39.0](service/connect/CHANGELOG.md#v1390-2022-12-06)
* **Feature**: This release provides APIs that enable you to programmatically manage rules for Contact Lens conversational analytics and third party applications. For more information, see https://docs.aws.amazon.com/connect/latest/APIReference/rules-api.html
* `github.com/aws/aws-sdk-go-v2/service/rds`: [v1.33.0](service/rds/CHANGELOG.md#v1330-2022-12-06)
* **Feature**: This release adds the BlueGreenDeploymentNotFoundFault to the AddTagsToResource, ListTagsForResource, and RemoveTagsFromResource operations.
* `github.com/aws/aws-sdk-go-v2/service/sagemakerfeaturestoreruntime`: [v1.12.0](service/sagemakerfeaturestoreruntime/CHANGELOG.md#v1120-2022-12-06)
* **Feature**: For online + offline Feature Groups, added ability to target PutRecord and DeleteRecord actions to only online store, or only offline store. If target store parameter is not specified, actions will apply to both stores.
# Release (2022-12-05)
## Module Highlights
* `github.com/aws/aws-sdk-go-v2/service/costexplorer`: [v1.22.0](service/costexplorer/CHANGELOG.md#v1220-2022-12-05)
* **Feature**: This release introduces two new APIs that offer a 1-click experience to refresh Savings Plans recommendations. The two APIs are StartSavingsPlansPurchaseRecommendationGeneration and ListSavingsPlansPurchaseRecommendationGeneration.
* `github.com/aws/aws-sdk-go-v2/service/ec2`: [v1.75.0](service/ec2/CHANGELOG.md#v1750-2022-12-05)
* **Feature**: Documentation updates for EC2.
* `github.com/aws/aws-sdk-go-v2/service/ivschat`: [v1.2.0](service/ivschat/CHANGELOG.md#v120-2022-12-05)
* **Feature**: Adds PendingVerification error type to messaging APIs to block the resource usage for accounts identified as being fraudulent.
* `github.com/aws/aws-sdk-go-v2/service/rds`: [v1.32.0](service/rds/CHANGELOG.md#v1320-2022-12-05)
* **Feature**: This release adds the InvalidDBInstanceStateFault to the RestoreDBClusterFromSnapshot operation.
* `github.com/aws/aws-sdk-go-v2/service/transcribe`: [v1.23.0](service/transcribe/CHANGELOG.md#v1230-2022-12-05)
* **Feature**: Amazon Transcribe now supports creating custom language models in the following languages: Japanese (ja-JP) and German (de-DE).
# Release (2022-12-02)
## General Highlights

View file

@ -3,4 +3,4 @@
package aws
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.17.2"
const goModuleVersion = "1.17.3"

View file

@ -1,3 +1,12 @@
# v1.18.6 (2022-12-19)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.18.5 (2022-12-15)
* **Bug Fix**: Unify logic between shared config and in finding home directory
* **Dependency Update**: Updated to the latest SDK module versions
# v1.18.4 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package config
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.18.4"
const goModuleVersion = "1.18.6"

View file

@ -8,7 +8,6 @@ import (
"io"
"io/ioutil"
"os"
"os/user"
"path/filepath"
"strings"
"time"
@ -16,6 +15,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/aws/aws-sdk-go-v2/internal/ini"
"github.com/aws/aws-sdk-go-v2/internal/shareddefaults"
"github.com/aws/smithy-go/logging"
)
@ -108,7 +108,7 @@ var defaultSharedConfigProfile = DefaultSharedConfigProfile
// - Linux/Unix: $HOME/.aws/credentials
// - Windows: %USERPROFILE%\.aws\credentials
func DefaultSharedCredentialsFilename() string {
return filepath.Join(userHomeDir(), ".aws", "credentials")
return filepath.Join(shareddefaults.UserHomeDir(), ".aws", "credentials")
}
// DefaultSharedConfigFilename returns the SDK's default file path for
@ -119,7 +119,7 @@ func DefaultSharedCredentialsFilename() string {
// - Linux/Unix: $HOME/.aws/config
// - Windows: %USERPROFILE%\.aws\config
func DefaultSharedConfigFilename() string {
return filepath.Join(userHomeDir(), ".aws", "config")
return filepath.Join(shareddefaults.UserHomeDir(), ".aws", "config")
}
// DefaultSharedConfigFiles is a slice of the default shared config files that
@ -1268,22 +1268,6 @@ func (e CredentialRequiresARNError) Error() string {
)
}
func userHomeDir() string {
// Ignore errors since we only care about Windows and *nix.
home, _ := os.UserHomeDir()
if len(home) > 0 {
return home
}
currUser, _ := user.Current()
if currUser != nil {
home = currUser.HomeDir
}
return home
}
func oneOrNone(bs ...bool) bool {
var count int

View file

@ -1,3 +1,12 @@
# v1.13.6 (2022-12-19)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.13.5 (2022-12-15)
* **Bug Fix**: Unify logic between shared config and in finding home directory
* **Dependency Update**: Updated to the latest SDK module versions
# v1.13.4 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package credentials
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.13.4"
const goModuleVersion = "1.13.6"

View file

@ -13,9 +13,10 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/internal/sdk"
"github.com/aws/aws-sdk-go-v2/internal/shareddefaults"
)
var osUserHomeDur = os.UserHomeDir
var osUserHomeDur = shareddefaults.UserHomeDir
// StandardCachedTokenFilepath returns the filepath for the cached SSO token file, or
// error if unable get derive the path. Key that will be used to compute a SHA1
@ -25,13 +26,12 @@ var osUserHomeDur = os.UserHomeDir
//
// ~/.aws/sso/cache/<sha1-hex-encoded-key>.json
func StandardCachedTokenFilepath(key string) (string, error) {
homeDir, err := osUserHomeDur()
if err != nil {
return "", fmt.Errorf("unable to get USER's home directory for cached token, %w", err)
homeDir := osUserHomeDur()
if len(homeDir) == 0 {
return "", fmt.Errorf("unable to get USER's home directory for cached token")
}
hash := sha1.New()
if _, err = hash.Write([]byte(key)); err != nil {
if _, err := hash.Write([]byte(key)); err != nil {
return "", fmt.Errorf("unable to compute cached token filepath key SHA1 hash, %w", err)
}

View file

@ -1,3 +1,7 @@
# v1.12.21 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.12.20 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package imds
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.12.20"
const goModuleVersion = "1.12.21"

View file

@ -1,3 +1,11 @@
# v1.11.45 (2022-12-19)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.11.44 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.11.43 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package manager
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.11.43"
const goModuleVersion = "1.11.45"

View file

@ -1,3 +1,7 @@
# v1.1.27 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.1.26 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package configsources
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.1.26"
const goModuleVersion = "1.1.27"

View file

@ -1,3 +1,7 @@
# v2.4.21 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v2.4.20 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package endpoints
// goModuleVersion is the tagged release for this module
const goModuleVersion = "2.4.20"
const goModuleVersion = "2.4.21"

View file

@ -1,3 +1,7 @@
# v1.3.28 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.3.27 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package ini
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.3.27"
const goModuleVersion = "1.3.28"

View file

@ -0,0 +1,47 @@
package shareddefaults
import (
"os"
"os/user"
"path/filepath"
)
// SharedCredentialsFilename returns the SDK's default file path
// for the shared credentials file.
//
// Builds the shared config file path based on the OS's platform.
//
// - Linux/Unix: $HOME/.aws/credentials
// - Windows: %USERPROFILE%\.aws\credentials
func SharedCredentialsFilename() string {
return filepath.Join(UserHomeDir(), ".aws", "credentials")
}
// SharedConfigFilename returns the SDK's default file path for
// the shared config file.
//
// Builds the shared config file path based on the OS's platform.
//
// - Linux/Unix: $HOME/.aws/config
// - Windows: %USERPROFILE%\.aws\config
func SharedConfigFilename() string {
return filepath.Join(UserHomeDir(), ".aws", "config")
}
// UserHomeDir returns the home directory for the user the process is
// running under.
func UserHomeDir() string {
// Ignore errors since we only care about Windows and *nix.
home, _ := os.UserHomeDir()
if len(home) > 0 {
return home
}
currUser, _ := user.Current()
if currUser != nil {
home = currUser.HomeDir
}
return home
}

View file

@ -1,3 +1,7 @@
# v1.0.18 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.0.17 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package v4a
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.0.17"
const goModuleVersion = "1.0.18"

View file

@ -1,3 +1,7 @@
# v1.1.22 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.1.21 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package checksum
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.1.21"
const goModuleVersion = "1.1.22"

View file

@ -1,3 +1,7 @@
# v1.9.21 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.9.20 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package presignedurl
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.9.20"
const goModuleVersion = "1.9.21"

View file

@ -1,3 +1,7 @@
# v1.13.21 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.13.20 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package s3shared
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.13.20"
const goModuleVersion = "1.13.21"

View file

@ -1,3 +1,7 @@
# v1.29.6 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.29.5 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package s3
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.29.5"
const goModuleVersion = "1.29.6"

View file

@ -1,3 +1,7 @@
# v1.11.27 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.11.26 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package sso
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.11.26"
const goModuleVersion = "1.11.27"

View file

@ -1,3 +1,11 @@
# v1.13.11 (2022-12-19)
* No change notes available for this release.
# v1.13.10 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.13.9 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package ssooidc
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.13.9"
const goModuleVersion = "1.13.11"

View file

@ -135,6 +135,14 @@ var defaultPartitions = endpoints.Partitions{
RegionRegex: partitionRegexp.Aws,
IsRegionalized: true,
Endpoints: endpoints.Endpoints{
endpoints.EndpointKey{
Region: "af-south-1",
}: endpoints.Endpoint{
Hostname: "oidc.af-south-1.amazonaws.com",
CredentialScope: endpoints.CredentialScope{
Region: "af-south-1",
},
},
endpoints.EndpointKey{
Region: "ap-east-1",
}: endpoints.Endpoint{
@ -191,6 +199,14 @@ var defaultPartitions = endpoints.Partitions{
Region: "ap-southeast-2",
},
},
endpoints.EndpointKey{
Region: "ap-southeast-3",
}: endpoints.Endpoint{
Hostname: "oidc.ap-southeast-3.amazonaws.com",
CredentialScope: endpoints.CredentialScope{
Region: "ap-southeast-3",
},
},
endpoints.EndpointKey{
Region: "ca-central-1",
}: endpoints.Endpoint{
@ -279,6 +295,14 @@ var defaultPartitions = endpoints.Partitions{
Region: "us-east-2",
},
},
endpoints.EndpointKey{
Region: "us-west-1",
}: endpoints.Endpoint{
Hostname: "oidc.us-west-1.amazonaws.com",
CredentialScope: endpoints.CredentialScope{
Region: "us-west-1",
},
},
endpoints.EndpointKey{
Region: "us-west-2",
}: endpoints.Endpoint{

View file

@ -1,3 +1,7 @@
# v1.17.7 (2022-12-15)
* **Dependency Update**: Updated to the latest SDK module versions
# v1.17.6 (2022-12-02)
* **Dependency Update**: Updated to the latest SDK module versions

View file

@ -3,4 +3,4 @@
package sts
// goModuleVersion is the tagged release for this module
const goModuleVersion = "1.17.6"
const goModuleVersion = "1.17.7"

View file

@ -9300,6 +9300,15 @@ var awsPartition = partition{
}: endpoint{
Hostname: "elasticfilesystem-fips.eu-central-1.amazonaws.com",
},
endpointKey{
Region: "eu-central-2",
}: endpoint{},
endpointKey{
Region: "eu-central-2",
Variant: fipsVariant,
}: endpoint{
Hostname: "elasticfilesystem-fips.eu-central-2.amazonaws.com",
},
endpointKey{
Region: "eu-north-1",
}: endpoint{},
@ -9444,6 +9453,15 @@ var awsPartition = partition{
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-eu-central-2",
}: endpoint{
Hostname: "elasticfilesystem-fips.eu-central-2.amazonaws.com",
CredentialScope: credentialScope{
Region: "eu-central-2",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-eu-north-1",
}: endpoint{
@ -11737,12 +11755,18 @@ var awsPartition = partition{
endpointKey{
Region: "ap-northeast-1",
}: endpoint{},
endpointKey{
Region: "ap-south-1",
}: endpoint{},
endpointKey{
Region: "ap-southeast-1",
}: endpoint{},
endpointKey{
Region: "ap-southeast-2",
}: endpoint{},
endpointKey{
Region: "ca-central-1",
}: endpoint{},
endpointKey{
Region: "eu-central-1",
}: endpoint{},
@ -11752,6 +11776,12 @@ var awsPartition = partition{
endpointKey{
Region: "eu-west-1",
}: endpoint{},
endpointKey{
Region: "eu-west-2",
}: endpoint{},
endpointKey{
Region: "sa-east-1",
}: endpoint{},
endpointKey{
Region: "us-east-1",
}: endpoint{},
@ -17595,6 +17625,14 @@ var awsPartition = partition{
},
"oidc": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "af-south-1",
}: endpoint{
Hostname: "oidc.af-south-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "af-south-1",
},
},
endpointKey{
Region: "ap-east-1",
}: endpoint{
@ -17651,6 +17689,14 @@ var awsPartition = partition{
Region: "ap-southeast-2",
},
},
endpointKey{
Region: "ap-southeast-3",
}: endpoint{
Hostname: "oidc.ap-southeast-3.amazonaws.com",
CredentialScope: credentialScope{
Region: "ap-southeast-3",
},
},
endpointKey{
Region: "ca-central-1",
}: endpoint{
@ -17739,6 +17785,14 @@ var awsPartition = partition{
Region: "us-east-2",
},
},
endpointKey{
Region: "us-west-1",
}: endpoint{
Hostname: "oidc.us-west-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-west-1",
},
},
endpointKey{
Region: "us-west-2",
}: endpoint{
@ -18897,12 +18951,18 @@ var awsPartition = partition{
endpointKey{
Region: "eu-central-1",
}: endpoint{},
endpointKey{
Region: "eu-north-1",
}: endpoint{},
endpointKey{
Region: "eu-west-1",
}: endpoint{},
endpointKey{
Region: "eu-west-2",
}: endpoint{},
endpointKey{
Region: "eu-west-3",
}: endpoint{},
endpointKey{
Region: "sa-east-1",
}: endpoint{},
@ -18937,6 +18997,9 @@ var awsPartition = partition{
endpointKey{
Region: "ap-south-1",
}: endpoint{},
endpointKey{
Region: "ap-south-2",
}: endpoint{},
endpointKey{
Region: "ap-southeast-1",
}: endpoint{},
@ -18958,12 +19021,18 @@ var awsPartition = partition{
endpointKey{
Region: "eu-central-1",
}: endpoint{},
endpointKey{
Region: "eu-central-2",
}: endpoint{},
endpointKey{
Region: "eu-north-1",
}: endpoint{},
endpointKey{
Region: "eu-south-1",
}: endpoint{},
endpointKey{
Region: "eu-south-2",
}: endpoint{},
endpointKey{
Region: "eu-west-1",
}: endpoint{},
@ -22991,6 +23060,9 @@ var awsPartition = partition{
endpointKey{
Region: "eu-west-3",
}: endpoint{},
endpointKey{
Region: "me-central-1",
}: endpoint{},
endpointKey{
Region: "me-south-1",
}: endpoint{},
@ -28068,6 +28140,16 @@ var awscnPartition = partition{
}: endpoint{},
},
},
"datasync": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "cn-north-1",
}: endpoint{},
endpointKey{
Region: "cn-northwest-1",
}: endpoint{},
},
},
"dax": service{
Endpoints: serviceEndpoints{
endpointKey{
@ -32413,6 +32495,9 @@ var awsusgovPartition = partition{
},
"metrics.sagemaker": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "us-gov-east-1",
}: endpoint{},
endpointKey{
Region: "us-gov-west-1",
}: endpoint{},
@ -35018,6 +35103,13 @@ var awsisoPartition = partition{
}: endpoint{},
},
},
"glue": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "us-iso-east-1",
}: endpoint{},
},
},
"health": service{
Endpoints: serviceEndpoints{
endpointKey{
@ -35400,6 +35492,9 @@ var awsisoPartition = partition{
endpointKey{
Region: "us-iso-east-1",
}: endpoint{},
endpointKey{
Region: "us-iso-west-1",
}: endpoint{},
},
},
},

View file

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.44.160"
const SDKVersion = "1.44.163"

View file

@ -199,6 +199,9 @@ func Clone[S ~[]E, E any](s S) S {
// Compact replaces consecutive runs of equal elements with a single copy.
// This is like the uniq command found on Unix.
// Compact modifies the contents of the slice s; it does not create a new slice.
// When Compact discards m elements in total, it might not modify the elements
// s[len(s)-m:len(s)]. If those elements contain pointers you might consider
// zeroing those elements so that objects they reference can be garbage collected.
func Compact[S ~[]E, E comparable](s S) S {
if len(s) < 2 {
return s

View file

@ -186,8 +186,9 @@ func CheckMediaResponse(res *http.Response) error {
}
slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
return &Error{
Code: res.StatusCode,
Body: string(slurp),
Code: res.StatusCode,
Body: string(slurp),
Header: res.Header,
}
}

View file

@ -86,7 +86,12 @@ func schemaToMap(schema interface{}, mustInclude, useNull map[string]bool, useNu
if f.Type.Kind() == reflect.Map && useNullMaps[f.Name] != nil {
ms, ok := v.Interface().(map[string]string)
if !ok {
return nil, fmt.Errorf("field %q has keys in NullFields but is not a map[string]string", f.Name)
mi, err := initMapSlow(v, f.Name, useNullMaps)
if err != nil {
return nil, err
}
m[tag.apiName] = mi
continue
}
mi := map[string]interface{}{}
for k, v := range ms {
@ -120,6 +125,25 @@ func schemaToMap(schema interface{}, mustInclude, useNull map[string]bool, useNu
return m, nil
}
// initMapSlow uses reflection to build up a map object. This is slower than
// the default behavior so it should be used only as a fallback.
func initMapSlow(rv reflect.Value, fieldName string, useNullMaps map[string]map[string]bool) (map[string]interface{}, error) {
mi := map[string]interface{}{}
iter := rv.MapRange()
for iter.Next() {
k, ok := iter.Key().Interface().(string)
if !ok {
return nil, fmt.Errorf("field %q has keys in NullFields but is not a map[string]any", fieldName)
}
v := iter.Value().Interface()
mi[k] = v
}
for k := range useNullMaps[fieldName] {
mi[k] = nil
}
return mi, nil
}
// formatAsString returns a string representation of v, dereferencing it first if possible.
func formatAsString(v reflect.Value, kind reflect.Kind) string {
if kind == reflect.Ptr && !v.IsNil() {

View file

@ -5,4 +5,4 @@
package internal
// Version is the current tagged release of the library.
const Version = "0.104.0"
const Version = "0.105.0"

43
vendor/modules.txt vendored
View file

@ -7,10 +7,10 @@ cloud.google.com/go/internal/version
# cloud.google.com/go/compute v1.14.0
## explicit; go 1.19
cloud.google.com/go/compute/internal
# cloud.google.com/go/compute/metadata v0.2.2
# cloud.google.com/go/compute/metadata v0.2.3
## explicit; go 1.19
cloud.google.com/go/compute/metadata
# cloud.google.com/go/iam v0.8.0
# cloud.google.com/go/iam v0.9.0
## explicit; go 1.19
cloud.google.com/go/iam
cloud.google.com/go/iam/apiv1/iampb
@ -80,7 +80,7 @@ github.com/VividCortex/ewma
# github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
## explicit; go 1.15
github.com/alecthomas/units
# github.com/aws/aws-sdk-go v1.44.160
# github.com/aws/aws-sdk-go v1.44.163
## explicit; go 1.11
github.com/aws/aws-sdk-go/aws
github.com/aws/aws-sdk-go/aws/awserr
@ -122,7 +122,7 @@ github.com/aws/aws-sdk-go/service/sso
github.com/aws/aws-sdk-go/service/sso/ssoiface
github.com/aws/aws-sdk-go/service/sts
github.com/aws/aws-sdk-go/service/sts/stsiface
# github.com/aws/aws-sdk-go-v2 v1.17.2
# github.com/aws/aws-sdk-go-v2 v1.17.3
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2
github.com/aws/aws-sdk-go-v2/aws
@ -141,6 +141,7 @@ github.com/aws/aws-sdk-go-v2/internal/awsutil
github.com/aws/aws-sdk-go-v2/internal/rand
github.com/aws/aws-sdk-go-v2/internal/sdk
github.com/aws/aws-sdk-go-v2/internal/sdkio
github.com/aws/aws-sdk-go-v2/internal/shareddefaults
github.com/aws/aws-sdk-go-v2/internal/strings
github.com/aws/aws-sdk-go-v2/internal/sync/singleflight
github.com/aws/aws-sdk-go-v2/internal/timeconv
@ -148,10 +149,10 @@ github.com/aws/aws-sdk-go-v2/internal/timeconv
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream/eventstreamapi
# github.com/aws/aws-sdk-go-v2/config v1.18.4
# github.com/aws/aws-sdk-go-v2/config v1.18.6
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/config
# github.com/aws/aws-sdk-go-v2/credentials v1.13.4
# github.com/aws/aws-sdk-go-v2/credentials v1.13.6
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/credentials
github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds
@ -160,23 +161,23 @@ github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client
github.com/aws/aws-sdk-go-v2/credentials/processcreds
github.com/aws/aws-sdk-go-v2/credentials/ssocreds
github.com/aws/aws-sdk-go-v2/credentials/stscreds
# github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20
# github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/feature/ec2/imds
github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config
# github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.43
# github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.45
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/feature/s3/manager
# github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26
# github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/internal/configsources
# github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20
# github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2
# github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27
# github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/internal/ini
# github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17
# github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/internal/v4a
github.com/aws/aws-sdk-go-v2/internal/v4a/internal/crypto
@ -184,35 +185,35 @@ github.com/aws/aws-sdk-go-v2/internal/v4a/internal/v4
# github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding
# github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21
# github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/service/internal/checksum
# github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20
# github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url
# github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20
# github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/service/internal/s3shared
github.com/aws/aws-sdk-go-v2/service/internal/s3shared/arn
github.com/aws/aws-sdk-go-v2/service/internal/s3shared/config
# github.com/aws/aws-sdk-go-v2/service/s3 v1.29.5
# github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/service/s3
github.com/aws/aws-sdk-go-v2/service/s3/internal/arn
github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations
github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints
github.com/aws/aws-sdk-go-v2/service/s3/types
# github.com/aws/aws-sdk-go-v2/service/sso v1.11.26
# github.com/aws/aws-sdk-go-v2/service/sso v1.11.27
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/service/sso
github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints
github.com/aws/aws-sdk-go-v2/service/sso/types
# github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9
# github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/service/ssooidc
github.com/aws/aws-sdk-go-v2/service/ssooidc/internal/endpoints
github.com/aws/aws-sdk-go-v2/service/ssooidc/types
# github.com/aws/aws-sdk-go-v2/service/sts v1.17.6
# github.com/aws/aws-sdk-go-v2/service/sts v1.17.7
## explicit; go 1.15
github.com/aws/aws-sdk-go-v2/service/sts
github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints
@ -527,7 +528,7 @@ go.uber.org/atomic
## explicit; go 1.18
go.uber.org/goleak
go.uber.org/goleak/internal/stack
# golang.org/x/exp v0.0.0-20221212164502-fae10dda9338
# golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15
## explicit; go 1.18
golang.org/x/exp/constraints
golang.org/x/exp/slices
@ -574,7 +575,7 @@ golang.org/x/time/rate
## explicit; go 1.17
golang.org/x/xerrors
golang.org/x/xerrors/internal
# google.golang.org/api v0.104.0
# google.golang.org/api v0.105.0
## explicit; go 1.19
google.golang.org/api/googleapi
google.golang.org/api/googleapi/transport