Merge branch 'public-single-node' into victorialogs-wip

This commit is contained in:
Aliaksandr Valialkin 2024-05-22 20:55:30 +02:00
commit b25ad7feab
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
41 changed files with 823 additions and 295 deletions

View file

@ -15,6 +15,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@ -119,7 +120,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
}
tr := &http.Transport{
DialContext: statDial,
DialContext: httputils.GetStatDialFunc("vmagent_remotewrite"),
TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
MaxConnsPerHost: 2 * concurrency,
MaxIdleConnsPerHost: 2 * concurrency,

View file

@ -377,8 +377,10 @@ func Stop() {
configReloaderWG.Wait()
sasGlobal.Load().MustStop()
deduplicatorGlobal.MustStop()
deduplicatorGlobal = nil
if deduplicatorGlobal != nil {
deduplicatorGlobal.MustStop()
deduplicatorGlobal = nil
}
for _, rwctx := range rwctxs {
rwctx.MustStop()
@ -492,7 +494,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
}
sortLabelsIfNeeded(tssBlock)
tssBlock = limitSeriesCardinality(tssBlock)
if sas != nil {
if sas.IsEnabled() {
matchIdxs := matchIdxsPool.Get()
matchIdxs.B = sas.Push(tssBlock, matchIdxs.B)
if !*streamAggrGlobalKeepInput {
@ -899,7 +901,7 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
// Apply stream aggregation or deduplication if they are configured
sas := rwctx.sas.Load()
if sas != nil {
if sas.IsEnabled() {
matchIdxs := matchIdxsPool.Get()
matchIdxs.B = sas.Push(tss, matchIdxs.B)
if !rwctx.streamAggrKeepInput {

View file

@ -1,73 +0,0 @@
package remotewrite
import (
"context"
"net"
"sync/atomic"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
"github.com/VictoriaMetrics/metrics"
)
func statDial(ctx context.Context, _, addr string) (conn net.Conn, err error) {
network := netutil.GetTCPNetwork()
conn, err = netutil.DialMaybeSRV(ctx, network, addr)
dialsTotal.Inc()
if err != nil {
dialErrors.Inc()
return nil, err
}
conns.Inc()
sc := &statConn{
Conn: conn,
}
return sc, nil
}
var (
dialsTotal = metrics.NewCounter(`vmagent_remotewrite_dials_total`)
dialErrors = metrics.NewCounter(`vmagent_remotewrite_dial_errors_total`)
conns = metrics.NewCounter(`vmagent_remotewrite_conns`)
)
type statConn struct {
closed atomic.Int32
net.Conn
}
func (sc *statConn) Read(p []byte) (int, error) {
n, err := sc.Conn.Read(p)
connReadsTotal.Inc()
if err != nil {
connReadErrors.Inc()
}
connBytesRead.Add(n)
return n, err
}
func (sc *statConn) Write(p []byte) (int, error) {
n, err := sc.Conn.Write(p)
connWritesTotal.Inc()
if err != nil {
connWriteErrors.Inc()
}
connBytesWritten.Add(n)
return n, err
}
func (sc *statConn) Close() error {
err := sc.Conn.Close()
if sc.closed.Add(1) == 1 {
conns.Dec()
}
return err
}
var (
connReadsTotal = metrics.NewCounter(`vmagent_remotewrite_conn_reads_total`)
connWritesTotal = metrics.NewCounter(`vmagent_remotewrite_conn_writes_total`)
connReadErrors = metrics.NewCounter(`vmagent_remotewrite_conn_read_errors_total`)
connWriteErrors = metrics.NewCounter(`vmagent_remotewrite_conn_write_errors_total`)
connBytesRead = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_read_total`)
connBytesWritten = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_written_total`)
)

View file

@ -15,8 +15,9 @@ import (
)
var (
addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. "+
"E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend and -datasource.showURL")
addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
"See also -remoteRead.disablePathAppend and -datasource.showURL")
appendTypePrefix = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
"It is hidden by default, since it can contain sensitive info such as auth key")
@ -98,6 +99,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
if err != nil {
return nil, fmt.Errorf("failed to create transport: %w", err)
}
tr.DialContext = httputils.GetStatDialFunc("vmalert_datasource")
tr.DisableKeepAlives = *disableKeepAlive
tr.MaxIdleConnsPerHost = *maxIdleConnections
if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {

View file

@ -15,7 +15,8 @@ var (
addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
"Remote read is used to restore alerts state."+
"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
"E.g. http://127.0.0.1:8428. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
"See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
"It is hidden by default, since it can contain sensitive info such as auth key")
@ -65,6 +66,7 @@ func Init() (datasource.QuerierBuilder, error) {
if err != nil {
return nil, fmt.Errorf("failed to create transport: %w", err)
}
tr.DialContext = httputils.GetStatDialFunc("vmalert_remoteread")
endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
if err != nil {

View file

@ -13,7 +13,9 @@ import (
var (
addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
"and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
"and recording rules results in form of timeseries. "+
"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
"It is hidden by default, since it can contain sensitive info such as auth key")
@ -69,6 +71,7 @@ func Init(ctx context.Context) (*Client, error) {
if err != nil {
return nil, fmt.Errorf("failed to create transport: %w", err)
}
t.DialContext = httputils.GetStatDialFunc("vmalert_remotewrite")
endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
if err != nil {

View file

@ -140,7 +140,7 @@ func (ctx *InsertCtx) ApplyRelabeling() {
// FlushBufs flushes buffered rows to the underlying storage.
func (ctx *InsertCtx) FlushBufs() error {
sas := sasGlobal.Load()
if (sas != nil || deduplicator != nil) && !ctx.skipStreamAggr {
if (sas.IsEnabled() || deduplicator != nil) && !ctx.skipStreamAggr {
matchIdxs := matchIdxsPool.Get()
matchIdxs.B = ctx.streamAggrCtx.push(ctx.mrs, matchIdxs.B)
if !*streamAggrKeepInput {

View file

@ -242,7 +242,7 @@ func (ctx *streamAggrCtx) push(mrs []storage.MetricRow, matchIdxs []byte) []byte
tss = tss[tssLen:]
sas := sasGlobal.Load()
if sas != nil {
if sas.IsEnabled() {
matchIdxs = sas.Push(tss, matchIdxs)
} else if deduplicator != nil {
matchIdxs = bytesutil.ResizeNoCopyMayOverallocate(matchIdxs, len(tss))

View file

@ -178,6 +178,10 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
}
}, [stateQuery, awaitStateQuery]);
useEffect(() => {
setStateQuery(query || []);
}, [query]);
return <div
className={classNames({
"vm-query-configurator": true,

View file

@ -1,12 +1,18 @@
import { useEffect } from "react";
import { useTimeState } from "../../../state/time/TimeStateContext";
import { useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
import { useAppState } from "../../../state/common/StateContext";
import { useQueryState } from "../../../state/query/QueryStateContext";
import { useEffect, useState } from "react";
import { useTimeDispatch, useTimeState } from "../../../state/time/TimeStateContext";
import { useCustomPanelDispatch, useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
import { useAppDispatch, useAppState } from "../../../state/common/StateContext";
import { useQueryDispatch, useQueryState } from "../../../state/query/QueryStateContext";
import { displayTypeTabs } from "../DisplayTypeSwitch";
import { compactObject } from "../../../utils/object";
import { useGraphState } from "../../../state/graph/GraphStateContext";
import { useGraphDispatch, useGraphState } from "../../../state/graph/GraphStateContext";
import { useSearchParams } from "react-router-dom";
import { useCallback } from "preact/compat";
import { getInitialDisplayType } from "../../../state/customPanel/reducer";
import { getInitialTimeState } from "../../../state/time/reducer";
import useEventListener from "../../../hooks/useEventListener";
import { getQueryArray } from "../../../utils/query-string";
import { arrayEquals } from "../../../utils/array";
import { isEqualURLSearchParams } from "../../../utils/url";
export const useSetQueryParams = () => {
const { tenantId } = useAppState();
@ -14,25 +20,108 @@ export const useSetQueryParams = () => {
const { query } = useQueryState();
const { duration, relativeTime, period: { date, step } } = useTimeState();
const { customStep } = useGraphState();
const [, setSearchParams] = useSearchParams();
const [searchParams, setSearchParams] = useSearchParams();
const dispatch = useAppDispatch();
const timeDispatch = useTimeDispatch();
const graphDispatch = useGraphDispatch();
const queryDispatch = useQueryDispatch();
const customPanelDispatch = useCustomPanelDispatch();
const [isPopstate, setIsPopstate] = useState(false);
const setterSearchParams = useCallback(() => {
if (isPopstate) {
// After the popstate event, the states synchronizes with the searchParams,
// so there's no need to refresh the searchParams again.
setIsPopstate(false);
return;
}
const newSearchParams = new URLSearchParams(searchParams);
const setSearchParamsFromState = () => {
const params: Record<string, unknown> = {};
query.forEach((q, i) => {
const group = `g${i}`;
params[`${group}.expr`] = q;
params[`${group}.range_input`] = duration;
params[`${group}.end_input`] = date;
params[`${group}.tab`] = displayTypeTabs.find(t => t.value === displayType)?.prometheusCode || 0;
params[`${group}.relative_time`] = relativeTime;
params[`${group}.tenantID`] = tenantId;
if ((searchParams.get(`${group}.expr`) !== q) && q) {
newSearchParams.set(`${group}.expr`, q);
}
if ((step !== customStep) && customStep) params[`${group}.step_input`] = customStep;
if (searchParams.get(`${group}.range_input`) !== duration) {
newSearchParams.set(`${group}.range_input`, duration);
}
if (searchParams.get(`${group}.end_input`) !== date) {
newSearchParams.set(`${group}.end_input`, date);
}
if (searchParams.get(`${group}.relative_time`) !== relativeTime) {
newSearchParams.set(`${group}.relative_time`, relativeTime || "none");
}
const stepFromUrl = searchParams.get(`${group}.step_input`) || step;
if (stepFromUrl && (stepFromUrl !== customStep)) {
newSearchParams.set(`${group}.step_input`, customStep);
}
const displayTypeCode = `${displayTypeTabs.find(t => t.value === displayType)?.prometheusCode || 0}`;
if (searchParams.get(`${group}.tab`) !== displayTypeCode) {
newSearchParams.set(`${group}.tab`, `${displayTypeCode}`);
}
if (searchParams.get(`${group}.tenantID`) !== tenantId && tenantId) {
newSearchParams.set(`${group}.tenantID`, tenantId);
}
});
if (isEqualURLSearchParams(newSearchParams, searchParams) || !newSearchParams.size) return;
setSearchParams(newSearchParams);
}, [tenantId, displayType, query, duration, relativeTime, date, step, customStep]);
setSearchParams(compactObject(params) as Record<string, string>);
};
useEffect(() => {
const timer = setTimeout(setterSearchParams, 200);
return () => clearTimeout(timer);
}, [setterSearchParams]);
useEffect(setSearchParamsFromState, [tenantId, displayType, query, duration, relativeTime, date, step, customStep]);
useEffect(setSearchParamsFromState, []);
useEffect(() => {
// Synchronize the states with searchParams only after the popstate event.
if (!isPopstate) return;
const timeFromUrl = getInitialTimeState();
const isDurationDifferent = (timeFromUrl.duration !== duration);
const isRelativeTimeDifferent = timeFromUrl.relativeTime !== relativeTime;
const isDateDifferent = timeFromUrl.relativeTime === "none" && timeFromUrl.period.date !== date;
const someNotEqual = isDurationDifferent || isRelativeTimeDifferent || isDateDifferent;
if (someNotEqual) {
timeDispatch({ type: "SET_TIME_STATE", payload: timeFromUrl });
}
const displayTypeFromUrl = getInitialDisplayType();
if (displayTypeFromUrl !== displayType) {
customPanelDispatch({ type: "SET_DISPLAY_TYPE", payload: displayTypeFromUrl });
}
const tenantIdFromUrl = searchParams.get("g0.tenantID") || "";
if (tenantIdFromUrl !== tenantId) {
dispatch({ type: "SET_TENANT_ID", payload: tenantIdFromUrl });
}
const queryFromUrl = getQueryArray();
if (!arrayEquals(queryFromUrl, query)) {
queryDispatch({ type: "SET_QUERY", payload: queryFromUrl });
timeDispatch({ type: "RUN_QUERY" });
}
// Timer prevents customStep reset on time range change.
const timer = setTimeout(() => {
const customStepFromUrl = searchParams.get("g0.step_input") || step;
if (customStepFromUrl && customStepFromUrl !== customStep) {
graphDispatch({ type: "SET_CUSTOM_STEP", payload: customStepFromUrl });
}
}, 50);
return () => clearTimeout(timer);
}, [searchParams, isPopstate]);
useEventListener("popstate", () => {
setIsPopstate(true);
});
};

View file

@ -12,7 +12,6 @@ import Alert from "../../components/Main/Alert/Alert";
import classNames from "classnames";
import useDeviceDetect from "../../hooks/useDeviceDetect";
import InstantQueryTip from "./InstantQueryTip/InstantQueryTip";
import useEventListener from "../../hooks/useEventListener";
import { useRef } from "react";
import CustomPanelTraces from "./CustomPanelTraces/CustomPanelTraces";
import WarningLimitSeries from "./WarningLimitSeries/WarningLimitSeries";
@ -65,9 +64,6 @@ const CustomPanel: FC = () => {
setHideError(false);
};
const handleChangePopstate = () => window.location.reload();
useEventListener("popstate", handleChangePopstate);
useEffect(() => {
graphDispatch({ type: "SET_IS_HISTOGRAM", payload: isHistogram });
}, [graphData]);

View file

@ -13,27 +13,18 @@ export const useFetchLogs = (server: string, query: string, limit: number) => {
const url = useMemo(() => getLogsUrl(server), [server]);
// include time range in query if not already present
const queryWithTime = useMemo(() => {
if (!/_time/.test(query)) {
const start = dayjs(period.start * 1000).tz().toISOString();
const end = dayjs(period.end * 1000).tz().toISOString();
const timerange = `_time:[${start}, ${end}]`;
return `${timerange} AND (${query})`;
}
return query;
}, [query, period]);
const options = useMemo(() => ({
method: "POST",
headers: {
"Accept": "application/stream+json",
},
body: new URLSearchParams({
query: queryWithTime.trim(),
limit: `${limit}`
query: query.trim(),
limit: `${limit}`,
start: dayjs(period.start * 1000).tz().toISOString(),
end: dayjs(period.end * 1000).tz().toISOString()
})
}), [queryWithTime, limit]);
}), [query, limit, period]);
const parseLineToJSON = (line: string): Logs | null => {
try {

View file

@ -19,12 +19,16 @@ export type CustomPanelAction =
| { type: "TOGGLE_QUERY_TRACING" }
| { type: "TOGGLE_TABLE_COMPACT" }
const queryTab = getQueryStringValue("g0.tab", 0) as string;
const displayType = displayTypeTabs.find(t => t.prometheusCode === +queryTab || t.value === queryTab);
export const getInitialDisplayType = () => {
const queryTab = getQueryStringValue("g0.tab", 0) as string;
const displayType = displayTypeTabs.find(t => t.prometheusCode === +queryTab || t.value === queryTab);
return displayType?.value || DisplayType.chart;
};
const limitsStorage = getFromStorage("SERIES_LIMITS") as string;
export const initialCustomPanelState: CustomPanelState = {
displayType: (displayType?.value || DisplayType.chart),
displayType: getInitialDisplayType(),
nocache: false,
isTracingEnabled: false,
seriesLimits: limitsStorage ? JSON.parse(limitsStorage) : DEFAULT_MAX_SERIES,

View file

@ -21,6 +21,7 @@ export interface TimeState {
}
export type TimeAction =
| { type: "SET_TIME_STATE", payload: { duration: string, period: TimeParams, relativeTime?: string; } }
| { type: "SET_DURATION", payload: string }
| { type: "SET_RELATIVE_TIME", payload: {id: string, duration: string, until: Date} }
| { type: "SET_PERIOD", payload: TimePeriod }
@ -32,24 +33,35 @@ export type TimeAction =
const timezone = getFromStorage("TIMEZONE") as string || getBrowserTimezone().region;
setTimezone(timezone);
const defaultDuration = getQueryStringValue("g0.range_input") as string;
export const getInitialTimeState = () => {
const defaultDuration = getQueryStringValue("g0.range_input") as string;
const { duration, endInput, relativeTimeId } = getRelativeTime({
defaultDuration: defaultDuration || "1h",
defaultEndInput: formatDateToLocal(getQueryStringValue("g0.end_input", getDateNowUTC()) as string),
relativeTimeId: defaultDuration ? getQueryStringValue("g0.relative_time", "none") as string : undefined
});
const { duration, endInput, relativeTimeId } = getRelativeTime({
defaultDuration: defaultDuration || "1h",
defaultEndInput: formatDateToLocal(getQueryStringValue("g0.end_input", getDateNowUTC()) as string),
relativeTimeId: defaultDuration ? getQueryStringValue("g0.relative_time", "none") as string : undefined
});
return {
duration,
period: getTimeperiodForDuration(duration, endInput),
relativeTime: relativeTimeId,
};
};
export const initialTimeState: TimeState = {
duration,
period: getTimeperiodForDuration(duration, endInput),
relativeTime: relativeTimeId,
...getInitialTimeState(),
timezone,
};
export function reducer(state: TimeState, action: TimeAction): TimeState {
switch (action.type) {
case "SET_TIME_STATE":
return {
...state,
...action.payload
};
case "SET_DURATION":
return {
...state,

View file

@ -11,3 +11,17 @@ export const isValidHttpUrl = (str: string): boolean => {
};
export const removeTrailingSlash = (url: string) => url.replace(/\/$/, "");
export const isEqualURLSearchParams = (params1: URLSearchParams, params2: URLSearchParams): boolean => {
if (Array.from(params1.entries()).length !== Array.from(params2.entries()).length) {
return false;
}
for (const [key, value] of params1) {
if (params2.get(key) !== value) {
return false;
}
}
return true;
};

View file

@ -53,7 +53,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset 20m) by(short_version))",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))",
"hide": true,
"iconColor": "blue",
"name": "version change",
@ -66,7 +66,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"})) by(job)",
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)",
"hide": true,
"iconColor": "orange",
"name": "restarts",

View file

@ -76,7 +76,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset 20m) by(version))",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))",
"hide": true,
"iconColor": "dark-blue",
"name": "version change",
@ -89,8 +89,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"})) by(job)",
"hide": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)",
"iconColor": "dark-yellow",
"name": "restarts",
"textFormat": "{{job}} restarted"

View file

@ -76,7 +76,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset 20m) by(version))",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))",
"hide": true,
"iconColor": "dark-blue",
"name": "version",

View file

@ -77,7 +77,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset 20m) by(version))",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))",
"hide": true,
"iconColor": "dark-blue",
"name": "version change",
@ -90,8 +90,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"})) by(job)",
"hide": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)",
"iconColor": "dark-yellow",
"name": "restarts",
"textFormat": "{{job}} restarted"

View file

@ -77,7 +77,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset 20m) by(version))",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))",
"hide": true,
"iconColor": "dark-blue",
"name": "version",

View file

@ -66,7 +66,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset 20m) by(short_version))",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))",
"hide": true,
"iconColor": "dark-blue",
"name": "version",
@ -79,8 +79,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"})) by(job, instance)",
"hide": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job, instance)",
"iconColor": "dark-yellow",
"name": "restarts",
"textFormat": "{{job}}:{{instance}} restarted"

View file

@ -60,7 +60,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset 20m) by(short_version))",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))",
"hide": true,
"iconColor": "dark-blue",
"name": "version",
@ -73,15 +73,14 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"})) by(job, instance)",
"hide": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job, instance)",
"iconColor": "dark-yellow",
"name": "restarts",
"textFormat": "{{job}}:{{instance}} restarted"
}
]
},
"description": "Overview for VictoriaMetrics vmalert v1.96.0 or higher",
"description": "Overview for VictoriaMetrics vmalert v1.102.0 or higher",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
@ -3237,6 +3236,213 @@
],
"title": "Datapoints drop rate ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"description": "Shows current number of established connections to remote write endpoints.\n\n",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 44
},
"id": 54,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum(max_over_time(vmalert_remotewrite_conns{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)",
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Connections ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"description": "Shows the global rate for number of written bytes via remote write connections.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 44
},
"id": 55,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum(rate(vmalert_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0",
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Bytes write rate ($instance)",
"type": "timeseries"
}
],
"title": "Remote write",

View file

@ -65,7 +65,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset 20m) by(short_version))",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))",
"hide": true,
"iconColor": "dark-blue",
"name": "version",
@ -78,8 +78,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"})) by(job, instance)",
"hide": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job, instance)",
"iconColor": "dark-yellow",
"name": "restarts",
"textFormat": "{{job}}:{{instance}} restarted"

View file

@ -59,7 +59,7 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset 20m) by(short_version))",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))",
"hide": true,
"iconColor": "dark-blue",
"name": "version",
@ -72,15 +72,14 @@
"uid": "$ds"
},
"enable": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"})) by(job, instance)",
"hide": true,
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job, instance)",
"iconColor": "dark-yellow",
"name": "restarts",
"textFormat": "{{job}}:{{instance}} restarted"
}
]
},
"description": "Overview for VictoriaMetrics vmalert v1.96.0 or higher",
"description": "Overview for VictoriaMetrics vmalert v1.102.0 or higher",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
@ -3236,6 +3235,213 @@
],
"title": "Datapoints drop rate ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "Shows current number of established connections to remote write endpoints.\n\n",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 44
},
"id": 54,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum(max_over_time(vmalert_remotewrite_conns{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)",
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Connections ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "Shows the global rate for number of written bytes via remote write connections.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 44
},
"id": 55,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum(rate(vmalert_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0",
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Bytes write rate ($instance)",
"type": "timeseries"
}
],
"title": "Remote write",

View file

@ -31,6 +31,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
## tip
**Update note 1: the `-remoteWrite.multitenantURL` command-line flag at `vmagent` was removed starting from this release. This flag was deprecated since [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0). Use `-enableMultitenantHandlers` instead, as it is easier to use and combine with [multitenant URL at vminsert](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels). See these [docs for details](https://docs.victoriametrics.com/vmagent.html#multitenancy).**
**Update note 2: the `-streamAggr.dropInputLabels` command-line flag at `vmagent` was renamed to `-remoteWrite.streamAggr.dropInputLabels`. `-streamAggr.dropInputLabels` is now used for global streaming aggregation.**
* SECURITY: upgrade Go builder from Go1.22.2 to Go1.22.3. See [the list of issues addressed in Go1.22.3](https://github.com/golang/go/issues?q=milestone%3AGo1.22.3+label%3ACherryPickApproved).
@ -46,16 +47,23 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add labels `path` and `url` to metrics `vmagent_remotewrite_push_failures_total` and `vmagent_remotewrite_samples_dropped_total`. Now number of failed pushes and dropped samples can be tracked per `-remoteWrite.url`.
* FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): add [rate_sum](https://docs.victoriametrics.com/stream-aggregation/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/stream-aggregation/#rate_avg) aggregation outputs.
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): reduce CPU usage when evaluating high number of alerting and recording rules.
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): speed up retrieving rules files from object storages by skipping unchanged objects during reloading. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6210).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): support reading [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) records in `-datasource.url`, `-remoteWrite.url` and `-remoteRead.url` command-line option. For example, `-remoteWrite.url=http://srv+victoria-metrics` automatically resolves the `victoria-metrics` DNS SRV to a list of hostnames with TCP ports and then sends data to one of the addresses. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6053).
* FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): add `-s3TLSInsecureSkipVerify` command-line flag for skipping TLS certificates verification when connecting to S3 endpoint.
* FEATURE: [dashboards](https://grafana.com/orgs/victoriametrics): use `$__interval` variable for offsets and look-behind windows in annotations. This should improve precision of `restarts` and `version change` annotations when zooming-in/zooming-out on the dashboards.
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix bug that prevents the first query trace from expanding on click event. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6186). The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/changelog/#v11000) release.
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix calendar display when `UTC+00:00` timezone is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6239).
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): remove redundant requests on the `Explore Cardinality` page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6240).
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix handling of URL params for browser history navigation (back and forward buttons). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6126) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5516#issuecomment-1867507232).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): prevent potential panic during [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) if more than one `--remoteWrite.streamAggr.dedupInterval` is configured. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6205).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): skip empty data blocks before sending to the remote write destination. Thanks to @viperstars for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6241).
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): set correct suffix `<output>_prometheus` for aggregation outputs [increase_prometheus](https://docs.victoriametrics.com/stream-aggregation/#increase_prometheus) and [total_prometheus](https://docs.victoriametrics.com/stream-aggregation/#total_prometheus). Before, outputs `total` and `total_prometheus` or `increase` and `increase_prometheus` had the same suffix.
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): prevent from excessive resource usage when stream aggregation config file is empty.
* BUGFIX: properly estimate the needed memory for query execution if it has the format [`aggr_func`](https://docs.victoriametrics.com/metricsql/#aggregate-functions)([`rollup_func[d]`](https://docs.victoriametrics.com/metricsql/#rollup-functions) (for example, `sum(rate(request_duration_seconds_bucket[5m]))`). This should allow performing aggregations over bigger number of time series when VictoriaMetrics runs in environments with small amounts of available memory. The issue has been introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/5138eaeea0791caa34bcfab410e0ca9cd253cd8f) in [v1.83.0](https://docs.victoriametrics.com/changelog_2022/#v1830).
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): correctly apply `-inmemoryDataFlushInterval` when it's set to minimum supported value 1s.
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly release memory used for metrics during config reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6247).
* BUGFIX: [dashboards](https://grafana.com/orgs/victoriametrics): fix `AnnotationQueryRunner` error in Grafana when executing annotations query against Prometheus backend. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6309) for details.
* DEPRECATION: [vmagent](https://docs.victoriametrics.com/vmagent/): removed deprecated `-remoteWrite.multitenantURL` flag from vmagent. This flag was deprecated since [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0). Use `-enableMultitenantHandlers` instead, as it is easier to use and combine with [multitenant URL at vminsert](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels). See these [docs for details](https://docs.victoriametrics.com/vmagent.html#multitenancy).

View file

@ -30,6 +30,7 @@ according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/QuickSta
* FEATURE: add `/select/logsql/streams` HTTP endpoint for returning [streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) from results of the given query. See [these docs](https://docs.victoriametrics.com/victorialogs/querying/#querying-streams) for details.
* FEATURE: add `/select/logsql/stream_label_names` HTTP endpoint for returning [stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) label names from results of the given query. See [these docs](https://docs.victoriametrics.com/victorialogs/querying/#querying-stream-label-names) for details.
* FEATURE: add `/select/logsql/stream_label_values` HTTP endpoint for returning [stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) label values for the given label from results of the given query. See [these docs](https://docs.victoriametrics.com/victorialogs/querying/#querying-stream-label-values) for details.
* FEATURE: [web UI](https://docs.victoriametrics.com/VictoriaLogs/querying/#web-ui): change time range limitation from `_time` in the expression to `start` and `end` query args.
* BUGFIX: fix `invalid memory address or nil pointer dereference` panic when using [`extract`](https://docs.victoriametrics.com/victorialogs/logsql/#extract-pipe), [`unpack_json`](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_json-pipe) or [`unpack_logfmt`](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_logfmt-pipe) pipes. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6306).

View file

@ -127,7 +127,7 @@ VictoriaLogs doesn't perform per-tenant authorization. Use [vmauth](https://docs
## Benchmarks
Here is a [benchmark suite](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/logs-benchmark) for comparing data ingestion performance
and resource usage between VictoriaLogs and Elasticsearch.
and resource usage between VictoriaLogs and Elasticsearch or Loki.
It is recommended [setting up VictoriaLogs](https://docs.victoriametrics.com/VictoriaLogs/QuickStart.html) in production alongside the existing
log management systems and comparing resource usage + query performance between VictoriaLogs and your system such as Elasticsearch or Grafana Loki.

View file

@ -376,8 +376,8 @@ In `BacktestingScheduler`, the inference window is *implicitly* defined as a per
```yaml
scheduler:
class: "scheduler.backtesting.BacktestingScheduler"
from_start_iso: '2021-01-01T00:00:00Z'
to_end_iso: '2021-01-14T00:00:00Z'
from_iso: '2021-01-01T00:00:00Z'
to_iso: '2021-01-14T00:00:00Z'
fit_window: 'P14D'
fit_every: 'PT1H'
```
@ -386,8 +386,8 @@ scheduler:
```yaml
scheduler:
class: "scheduler.backtesting.BacktestingScheduler"
from_start_s: 167253120
to_end_s: 167443200
from_s: 167253120
to_s: 167443200
fit_window: '14d'
fit_every: '1h'
```

View file

@ -17,6 +17,8 @@ aliases:
## Next release
- [vmalertmanager](./api.md#vmalertmanager): ignores content of `cr.spec.configSecret` if it's name clashes with secret used by operator for storing alertmanager config. See this [issue](https://github.com/VictoriaMetrics/operator/issues/954) for details.
- [operator](./README.md): remove finalizer for child objects with non-empty `DeletetionTimestamp`. See this [issue](https://github.com/VictoriaMetrics/operator/issues/953) for details.
- [operator](./README.md): skip storageClass check if there is no PVC size change. See this [issue](https://github.com/VictoriaMetrics/operator/issues/957) for details.
## [v0.44.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.44.0) - 9 May 2024

View file

@ -10,7 +10,7 @@ menu:
<!-- this doc autogenerated - don't edit it manually -->
# Auto Generated vars for package config
updated at Mon May 20 07:54:43 UTC 2024
updated at Tue May 21 11:53:40 UTC 2024
| varible name | variable default value | variable required | variable description |

View file

@ -1042,7 +1042,7 @@ The shortlist of configuration flags is the following:
-datasource.tlsServerName string
Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
-datasource.url string
Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend and -datasource.showURL
Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. See also -remoteRead.disablePathAppend and -datasource.showURL
-defaultTenant.graphite string
Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert/#multitenancy .This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/enterprise/
-defaultTenant.prometheus string
@ -1306,7 +1306,7 @@ The shortlist of configuration flags is the following:
-remoteRead.tlsServerName string
Optional TLS server name to use for connections to -remoteRead.url. By default, the server name from -remoteRead.url is used
-remoteRead.url vmalert
Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.
Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.
-remoteWrite.basicAuth.password string
Optional basic auth password for -remoteWrite.url
-remoteWrite.basicAuth.passwordFile string
@ -1360,7 +1360,7 @@ The shortlist of configuration flags is the following:
-remoteWrite.tlsServerName string
Optional TLS server name to use for connections to -remoteWrite.url. By default, the server name from -remoteWrite.url is used
-remoteWrite.url string
Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
-replay.disableProgressBar
Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
-replay.maxDatapointsPerQuery /query_range

View file

@ -439,6 +439,8 @@ Run `vmbackup -help` in order to see all the available options:
-s3StorageClass string
The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.
See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html
-s3TLSInsecureSkipVerify
Whether to skip TLS verification when connecting to the S3 endpoint.
-snapshot.createURL string
VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set
-snapshot.deleteURL string

View file

@ -577,6 +577,8 @@ command-line flags:
-s3StorageClass string
The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.
See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html
-s3TLSInsecureSkipVerify
Whether to skip TLS verification when connecting to the S3 endpoint.
-snapshot.createURL string
VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup.Example: http://victoriametrics:8428/snapshot/create
-snapshot.deleteURL string

View file

@ -221,6 +221,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
-s3StorageClass string
The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.
See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html
-s3TLSInsecureSkipVerify
Whether to skip TLS verification when connecting to the S3 endpoint.
-skipBackupCompleteCheck
Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file
-src string

View file

@ -27,6 +27,7 @@ var (
s3StorageClass = flag.String("s3StorageClass", "", "The Storage Class applied to objects uploaded to AWS S3. Supported values are: GLACIER, "+
"DEEP_ARCHIVE, GLACIER_IR, INTELLIGENT_TIERING, ONEZONE_IA, OUTPOSTS, REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.\n"+
"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html")
s3TLSInsecureSkipVerify = flag.Bool("s3TLSInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to the S3 endpoint.")
)
func runParallel(concurrency int, parts []common.Part, f func(p common.Part) error, progress func(elapsed time.Duration)) error {
@ -240,14 +241,15 @@ func NewRemoteFS(path string) (common.RemoteFS, error) {
bucket := dir[:n]
dir = dir[n:]
fs := &s3remote.FS{
CredsFilePath: *credsFilePath,
ConfigFilePath: *configFilePath,
CustomEndpoint: *customS3Endpoint,
StorageClass: s3remote.StringToS3StorageClass(*s3StorageClass),
S3ForcePathStyle: *s3ForcePathStyle,
ProfileName: *configProfile,
Bucket: bucket,
Dir: dir,
CredsFilePath: *credsFilePath,
ConfigFilePath: *configFilePath,
CustomEndpoint: *customS3Endpoint,
TLSInsecureSkipVerify: *s3TLSInsecureSkipVerify,
StorageClass: s3remote.StringToS3StorageClass(*s3StorageClass),
S3ForcePathStyle: *s3ForcePathStyle,
ProfileName: *configProfile,
Bucket: bucket,
Dir: dir,
}
if err := fs.Init(); err != nil {
return nil, fmt.Errorf("cannot initialize connection to s3: %w", err)

View file

@ -3,8 +3,10 @@ package s3remote
import (
"bytes"
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
"path"
"strings"
@ -72,6 +74,9 @@ type FS struct {
// The name of S3 config profile to use.
ProfileName string
// Whether to use HTTP client with tls.InsecureSkipVerify setting
TLSInsecureSkipVerify bool
s3 *s3.Client
uploader *manager.Uploader
}
@ -112,12 +117,19 @@ func (fs *FS) Init() error {
return err
}
if fs.TLSInsecureSkipVerify {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
cfg.HTTPClient = &http.Client{Transport: tr}
}
var outerErr error
fs.s3 = s3.NewFromConfig(cfg, func(o *s3.Options) {
if len(fs.CustomEndpoint) > 0 {
logger.Infof("Using provided custom S3 endpoint: %q", fs.CustomEndpoint)
o.UsePathStyle = fs.S3ForcePathStyle
o.EndpointResolver = s3.EndpointResolverFromURL(fs.CustomEndpoint)
o.BaseEndpoint = &fs.CustomEndpoint
} else {
region, err := manager.GetBucketRegion(context.Background(), s3.NewFromConfig(cfg), fs.Bucket)
if err != nil {

146
lib/httputils/statconn.go Normal file
View file

@ -0,0 +1,146 @@
package httputils
import (
"context"
"fmt"
"net"
"strconv"
"strings"
"sync"
"sync/atomic"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
"github.com/VictoriaMetrics/metrics"
)
var statConnMetricsRegistry sync.Map
type statConnMetrics struct {
dialsTotal *metrics.Counter
dialErrors *metrics.Counter
conns *metrics.Counter
connReadsTotal *metrics.Counter
connWritesTotal *metrics.Counter
connReadErrors *metrics.Counter
connWriteErrors *metrics.Counter
connBytesRead *metrics.Counter
connBytesWritten *metrics.Counter
}
func newStatConnMetrics(metricPrefix string) statConnMetrics {
scm := statConnMetrics{}
scm.dialsTotal = metrics.NewCounter(fmt.Sprintf(`%s_dials_total`, metricPrefix))
scm.dialErrors = metrics.NewCounter(fmt.Sprintf(`%s_dial_errors_total`, metricPrefix))
scm.conns = metrics.NewCounter(fmt.Sprintf(`%s_conns`, metricPrefix))
scm.connReadsTotal = metrics.NewCounter(fmt.Sprintf(`%s_conn_reads_total`, metricPrefix))
scm.connWritesTotal = metrics.NewCounter(fmt.Sprintf(`%s_conn_writes_total`, metricPrefix))
scm.connReadErrors = metrics.NewCounter(fmt.Sprintf(`%s_conn_read_errors_total`, metricPrefix))
scm.connWriteErrors = metrics.NewCounter(fmt.Sprintf(`%s_conn_write_errors_total`, metricPrefix))
scm.connBytesRead = metrics.NewCounter(fmt.Sprintf(`%s_conn_bytes_read_total`, metricPrefix))
scm.connBytesWritten = metrics.NewCounter(fmt.Sprintf(`%s_conn_bytes_written_total`, metricPrefix))
return scm
}
// GetStatDialFunc returns dial function that supports DNS SRV records,
// and register stats metrics for conns.
func GetStatDialFunc(metricPrefix string) func(ctx context.Context, network, addr string) (net.Conn, error) {
v, ok := statConnMetricsRegistry.Load(metricPrefix)
if !ok {
v = newStatConnMetrics(metricPrefix)
statConnMetricsRegistry.Store(metricPrefix, v)
}
sm := v.(statConnMetrics)
return func(ctx context.Context, _, addr string) (net.Conn, error) {
network := netutil.GetTCPNetwork()
conn, err := netutil.DialMaybeSRV(ctx, network, addr)
sm.dialsTotal.Inc()
if err != nil {
sm.dialErrors.Inc()
if !netutil.TCP6Enabled() && !isTCPv4Addr(addr) {
err = fmt.Errorf("%w; try -enableTCP6 command-line flag for dialing ipv6 addresses", err)
}
return nil, err
}
sm.conns.Inc()
sc := &statConn{
Conn: conn,
statConnMetrics: sm,
}
return sc, nil
}
}
type statConn struct {
closed atomic.Int32
net.Conn
statConnMetrics
}
func (sc *statConn) Read(p []byte) (int, error) {
n, err := sc.Conn.Read(p)
sc.connReadsTotal.Inc()
if err != nil {
sc.connReadErrors.Inc()
}
sc.connBytesRead.Add(n)
return n, err
}
func (sc *statConn) Write(p []byte) (int, error) {
n, err := sc.Conn.Write(p)
sc.connWritesTotal.Inc()
if err != nil {
sc.connWriteErrors.Inc()
}
sc.connBytesWritten.Add(n)
return n, err
}
func (sc *statConn) Close() error {
err := sc.Conn.Close()
if sc.closed.Add(1) == 1 {
sc.conns.Dec()
}
return err
}
func isTCPv4Addr(addr string) bool {
s := addr
for i := 0; i < 3; i++ {
n := strings.IndexByte(s, '.')
if n < 0 {
return false
}
if !isUint8NumString(s[:n]) {
return false
}
s = s[n+1:]
}
n := strings.IndexByte(s, ':')
if n < 0 {
return false
}
if !isUint8NumString(s[:n]) {
return false
}
s = s[n+1:]
// Verify TCP port
n, err := strconv.Atoi(s)
if err != nil {
return false
}
return n >= 0 && n < (1<<16)
}
func isUint8NumString(s string) bool {
n, err := strconv.Atoi(s)
if err != nil {
return false
}
return n >= 0 && n < (1<<8)
}

View file

@ -1,4 +1,4 @@
package promscrape
package httputils
import (
"testing"

View file

@ -14,6 +14,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
)
var (
@ -71,7 +72,7 @@ func newClient(ctx context.Context, sw *ScrapeWork) (*client, error) {
IdleConnTimeout: 2 * sw.ScrapeInterval,
DisableCompression: *disableCompression || sw.DisableCompression,
DisableKeepAlives: *disableKeepAlive || sw.DisableKeepAlive,
DialContext: statStdDial,
DialContext: httputils.GetStatDialFunc("vm_promscrape"),
MaxIdleConnsPerHost: 100,
MaxResponseHeaderBytes: int64(maxResponseHeadersSize.N),
}),

View file

@ -1,116 +0,0 @@
package promscrape
import (
"context"
"fmt"
"net"
"strconv"
"strings"
"sync/atomic"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
"github.com/VictoriaMetrics/metrics"
)
func statStdDial(ctx context.Context, _, addr string) (net.Conn, error) {
network := netutil.GetTCPNetwork()
conn, err := netutil.DialMaybeSRV(ctx, network, addr)
dialsTotal.Inc()
if err != nil {
dialErrors.Inc()
if !netutil.TCP6Enabled() && !isTCPv4Addr(addr) {
err = fmt.Errorf("%w; try -enableTCP6 command-line flag if you scrape ipv6 addresses", err)
}
return nil, err
}
conns.Inc()
sc := &statConn{
Conn: conn,
}
return sc, nil
}
var (
dialsTotal = metrics.NewCounter(`vm_promscrape_dials_total`)
dialErrors = metrics.NewCounter(`vm_promscrape_dial_errors_total`)
conns = metrics.NewCounter(`vm_promscrape_conns`)
)
type statConn struct {
closed atomic.Int32
net.Conn
}
func (sc *statConn) Read(p []byte) (int, error) {
n, err := sc.Conn.Read(p)
connReadsTotal.Inc()
if err != nil {
connReadErrors.Inc()
}
connBytesRead.Add(n)
return n, err
}
func (sc *statConn) Write(p []byte) (int, error) {
n, err := sc.Conn.Write(p)
connWritesTotal.Inc()
if err != nil {
connWriteErrors.Inc()
}
connBytesWritten.Add(n)
return n, err
}
func (sc *statConn) Close() error {
err := sc.Conn.Close()
if sc.closed.Add(1) == 1 {
conns.Dec()
}
return err
}
var (
connReadsTotal = metrics.NewCounter(`vm_promscrape_conn_reads_total`)
connWritesTotal = metrics.NewCounter(`vm_promscrape_conn_writes_total`)
connReadErrors = metrics.NewCounter(`vm_promscrape_conn_read_errors_total`)
connWriteErrors = metrics.NewCounter(`vm_promscrape_conn_write_errors_total`)
connBytesRead = metrics.NewCounter(`vm_promscrape_conn_bytes_read_total`)
connBytesWritten = metrics.NewCounter(`vm_promscrape_conn_bytes_written_total`)
)
func isTCPv4Addr(addr string) bool {
s := addr
for i := 0; i < 3; i++ {
n := strings.IndexByte(s, '.')
if n < 0 {
return false
}
if !isUint8NumString(s[:n]) {
return false
}
s = s[n+1:]
}
n := strings.IndexByte(s, ':')
if n < 0 {
return false
}
if !isUint8NumString(s[:n]) {
return false
}
s = s[n+1:]
// Verify TCP port
n, err := strconv.Atoi(s)
if err != nil {
return false
}
return n >= 0 && n < (1<<16)
}
func isUint8NumString(s string) bool {
n, err := strconv.Atoi(s)
if err != nil {
return false
}
return n >= 0 && n < (1<<8)
}

View file

@ -294,6 +294,17 @@ func newAggregatorsFromData(data []byte, pushFunc PushFunc, opts *Options) (*Agg
}, nil
}
// IsEnabled returns true if Aggregators has at least one configured aggregator
func (a *Aggregators) IsEnabled() bool {
if a == nil {
return false
}
if len(a.as) == 0 {
return false
}
return true
}
// MustStop stops a.
func (a *Aggregators) MustStop() {
if a == nil {