github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

Browse source
This commit is contained in:
Aliaksandr Valialkin 2023-01-17 21:49:03 -08:00
commit a2ab1f0ec9
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
76 changed files with 906 additions and 657 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/codeql-analysis.yml vendored
View file

@ -32,7 +32,7 @@ jobs:
strategy:
fail-fast: false
matrix:
language: [ 'go', 'javascript' ]
language: [ 'go' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://git.io/codeql-language-support

9
app/vminsert/main.go
View file

@ -128,7 +128,14 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
httpserver.Errorf(w, r, "%s", err)
return true
}
w.WriteHeader(http.StatusNoContent)
statusCode := http.StatusNoContent
if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus/metrics/job/") ||
strings.HasPrefix(path, "/api/v1/import/prometheus/metrics/job/") {
// Return 200 status code for pushgateway requests.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
statusCode = http.StatusOK
}
w.WriteHeader(statusCode)
return true
}
if strings.HasPrefix(path, "/datadog/") {

2
app/vmselect/netstorage/netstorage.go
View file

@ -486,7 +486,7 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
upw.br = br
upw.tr = tr
}
if gomaxprocs == 1 || upwsLen <= 100 {
if gomaxprocs == 1 || upwsLen <= 1000 {
// It is faster to unpack all the data in the current goroutine.
upw := getUnpackWork()
samples := 0

7
app/vmselect/promql/binary_op.go
View file

@ -5,6 +5,7 @@ import (
"math"
"strings"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/metricsql"
@ -248,7 +249,8 @@ func groupJoin(singleTimeseriesSide string, be *metricsql.BinaryOpExpr, rvsLeft,
bb.B = marshalMetricTagsSorted(bb.B[:0], &tsCopy.MetricName)
pair, ok := m[string(bb.B)]
if !ok {
m[string(bb.B)] = &tsPair{
k := bytesutil.InternBytes(bb.B)
m[k] = &tsPair{
left: &tsCopy,
right: tsRight,
}
@ -504,7 +506,8 @@ func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*time
logger.Panicf("BUG: unexpected binary op modifier %q", groupOp)
}
bb.B = marshalMetricTagsSorted(bb.B[:0], mn)
m[string(bb.B)] = append(m[string(bb.B)], ts)
k := bytesutil.InternBytes(bb.B)
m[k] = append(m[k], ts)
}
storage.PutMetricName(mn)
bbPool.Put(bb)

60
app/vmselect/promql/eval.go
View file

@ -505,26 +505,54 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
}
func getCommonLabelFilters(tss []*timeseries) []metricsql.LabelFilter {
m := make(map[string][]string)
if len(tss) == 0 {
return nil
}
type valuesCounter struct {
values map[string]struct{}
count int
}
m := make(map[string]*valuesCounter, len(tss[0].MetricName.Tags))
for _, ts := range tss {
for _, tag := range ts.MetricName.Tags {
m[string(tag.Key)] = append(m[string(tag.Key)], string(tag.Value))
vc, ok := m[string(tag.Key)]
if !ok {
k := bytesutil.InternBytes(tag.Key)
v := bytesutil.InternBytes(tag.Value)
m[k] = &valuesCounter{
values: map[string]struct{}{
v: {},
},
count: 1,
}
continue
}
if len(vc.values) > 100 {
// Too many unique values found for the given tag.
// Do not make a filter on such values, since it may slow down
// search for matching time series.
continue
}
vc.count++
if _, ok := vc.values[string(tag.Value)]; !ok {
v := bytesutil.InternBytes(tag.Value)
vc.values[v] = struct{}{}
}
}
}
lfs := make([]metricsql.LabelFilter, 0, len(m))
for key, values := range m {
if len(values) != len(tss) {
var values []string
for k, vc := range m {
if vc.count != len(tss) {
// Skip the tag, since it doesn't belong to all the time series.
continue
}
values = getUniqueValues(values)
if len(values) > 1000 {
// Skip the filter on the given tag, since it needs to enumerate too many unique values.
// This may slow down the search for matching time series.
continue
values = values[:0]
for s := range vc.values {
values = append(values, s)
}
lf := metricsql.LabelFilter{
Label: key,
Label: k,
}
if len(values) == 1 {
lf.Value = values[0]
@ -541,18 +569,6 @@ func getCommonLabelFilters(tss []*timeseries) []metricsql.LabelFilter {
return lfs
}
func getUniqueValues(a []string) []string {
m := make(map[string]struct{}, len(a))
results := make([]string, 0, len(a))
for _, s := range a {
if _, ok := m[s]; !ok {
results = append(results, s)
m[s] = struct{}{}
}
}
return results
}
func joinRegexpValues(a []string) string {
var b []byte
for i, s := range a {

8
app/vmselect/vmui/asset-manifest.json
View file

@ -1,12 +1,12 @@
{
"files": {
"main.css": "./static/css/main.7672c15c.css",
"main.js": "./static/js/main.84759f8d.js",
"main.css": "./static/css/main.e8e53cbf.css",
"main.js": "./static/js/main.b53fb552.js",
"static/js/27.c1ccfd29.chunk.js": "./static/js/27.c1ccfd29.chunk.js",
"index.html": "./index.html"
},
"entrypoints": [
"static/css/main.7672c15c.css",
"static/js/main.84759f8d.js"
"static/css/main.e8e53cbf.css",
"static/js/main.b53fb552.js"
]
}

2
app/vmselect/vmui/index.html
View file

@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono&family=Lato:wght@300;400;700&display=swap" rel="stylesheet"><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.84759f8d.js"></script><link href="./static/css/main.7672c15c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono&family=Lato:wght@300;400;700&display=swap" rel="stylesheet"><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.b53fb552.js"></script><link href="./static/css/main.e8e53cbf.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

1
app/vmselect/vmui/static/css/main.7672c15c.css
View file

File diff suppressed because one or more lines are too long

1
app/vmselect/vmui/static/css/main.e8e53cbf.css Normal file
View file

File diff suppressed because one or more lines are too long

2
app/vmselect/vmui/static/js/main.84759f8d.js
View file

File diff suppressed because one or more lines are too long

2
app/vmselect/vmui/static/js/main.b53fb552.js Normal file
View file

File diff suppressed because one or more lines are too long

11
app/vmselect/vmui/static/js/main.84759f8d.js.LICENSE.txt → app/vmselect/vmui/static/js/main.b53fb552.js.LICENSE.txt
View file

@ -17,6 +17,17 @@
* @license MIT
*/
/**
* React Router DOM v6.4.5
*
* Copyright (c) Remix Software Inc.
*
* This source code is licensed under the MIT license found in the
* LICENSE.md file in the root directory of this source tree.
*
* @license MIT
*/
/**
* React Router v6.4.5
*

4
app/vmstorage/main.go
View file

@ -650,10 +650,6 @@ func registerStorageMetrics(strg *storage.Storage) {
return float64(m().TooSmallTimestampRows)
})
metrics.NewGauge(`vm_search_delays_total`, func() float64 {
return float64(m().SearchDelays)
})
metrics.NewGauge(`vm_slow_row_inserts_total`, func() float64 {
return float64(m().SlowRowInserts)
})

3
app/vmui/packages/vmui/src/api/types.ts
View file

@ -11,7 +11,8 @@ export interface MetricResult extends MetricBase {
export interface InstantMetricResult extends MetricBase {
value: [number, string]
value?: [number, string]
values?: [number, string][]
}
export interface TracingData {

7
app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx
View file

@ -55,9 +55,8 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
const color = series[seriesIdx]?.stroke+"";
const groups = new Set();
metrics.forEach(m => groups.add(m.group));
const groupsSize = groups.size;
const groups = new Set(metrics.map(m => m.group));
const showQueryNum = groups.size > 1;
const group = metrics[seriesIdx-1]?.group || 0;
const metric = metrics[seriesIdx-1]?.metric || {};
@ -141,7 +140,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
>
<div className="vm-chart-tooltip-header">
<div className="vm-chart-tooltip-header__date">
{groupsSize > 1 && (
{showQueryNum && (
<div>Query {group}</div>
)}
{date}

5
app/vmui/packages/vmui/src/components/Chart/Legend/Legend.tsx
View file

@ -13,6 +13,7 @@ const Legend: FC<LegendProps> = ({ labels, query, onChange }) => {
const groups = useMemo(() => {
return Array.from(new Set(labels.map(l => l.group)));
}, [labels]);
const showQueryNum = groups.length > 1;
return <>
<div className="vm-legend">
@ -21,7 +22,9 @@ const Legend: FC<LegendProps> = ({ labels, query, onChange }) => {
key={group}
>
<div className="vm-legend-group-title">
<span className="vm-legend-group-title__count">Query {group}: </span>
{showQueryNum && (
<span className="vm-legend-group-title__count">Query {group}: </span>
)}
<span className="vm-legend-group-title__query">{query[group - 1]}</span>
</div>
<div>

32
app/vmui/packages/vmui/src/components/Configurators/AdditionalSettings/AdditionalSettings.tsx
View file

@ -1,20 +1,13 @@
import React, { FC, useEffect } from "preact/compat";
import StepConfigurator from "../StepConfigurator/StepConfigurator";
import { useGraphDispatch, useGraphState } from "../../../state/graph/GraphStateContext";
import React, { FC } from "preact/compat";
import { getAppModeParams } from "../../../utils/app-mode";
import TenantsConfiguration from "../TenantsConfiguration/TenantsConfiguration";
import { useCustomPanelDispatch, useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
import { useTimeState } from "../../../state/time/TimeStateContext";
import { useQueryDispatch, useQueryState } from "../../../state/query/QueryStateContext";
import "./style.scss";
import Switch from "../../Main/Switch/Switch";
import usePrevious from "../../../hooks/usePrevious";
const AdditionalSettings: FC = () => {
const { customStep } = useGraphState();
const graphDispatch = useGraphDispatch();
const { inputTenantID } = getAppModeParams();
const { autocomplete } = useQueryState();
@ -23,9 +16,6 @@ const AdditionalSettings: FC = () => {
const { nocache, isTracingEnabled } = useCustomPanelState();
const customPanelDispatch = useCustomPanelDispatch();
const { period: { step }, duration } = useTimeState();
const prevDuration = usePrevious(duration);
const onChangeCache = () => {
customPanelDispatch({ type: "TOGGLE_NO_CACHE" });
};
@ -38,19 +28,6 @@ const AdditionalSettings: FC = () => {
queryDispatch({ type: "TOGGLE_AUTOCOMPLETE" });
};
const onChangeStep = (value: string) => {
graphDispatch({ type: "SET_CUSTOM_STEP", payload: value });
};
useEffect(() => {
if (!customStep && step) onChangeStep(step);
}, [step]);
useEffect(() => {
if (duration === prevDuration || !prevDuration) return;
if (step) onChangeStep(step);
}, [duration, prevDuration]);
return <div className="vm-additional-settings">
<Switch
label={"Autocomplete"}
@ -67,13 +44,6 @@ const AdditionalSettings: FC = () => {
value={isTracingEnabled}
onChange={onChangeQueryTracing}
/>
<div className="vm-additional-settings__input">
<StepConfigurator
defaultStep={step}
setStep={onChangeStep}
value={customStep}
/>
</div>
{!!inputTenantID && (
<div className="vm-additional-settings__input">
<TenantsConfiguration/>

153
app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/StepConfigurator.tsx
View file

@ -1,26 +1,59 @@
import React, { FC, useEffect, useState } from "preact/compat";
import { RestartIcon } from "../../Main/Icons";
import React, { FC, useEffect, useRef, useState } from "preact/compat";
import { RestartIcon, TimelineIcon } from "../../Main/Icons";
import TextField from "../../Main/TextField/TextField";
import Button from "../../Main/Button/Button";
import Tooltip from "../../Main/Tooltip/Tooltip";
import { ErrorTypes } from "../../../types";
import { supportedDurations } from "../../../utils/time";
import { useTimeState } from "../../../state/time/TimeStateContext";
import { useGraphDispatch, useGraphState } from "../../../state/graph/GraphStateContext";
import usePrevious from "../../../hooks/usePrevious";
import "./style.scss";
import { getAppModeEnable } from "../../../utils/app-mode";
import Popper from "../../Main/Popper/Popper";
interface StepConfiguratorProps {
defaultStep?: string,
value?: string,
setStep: (step: string) => void,
}
const StepConfigurator: FC = () => {
const appModeEnable = getAppModeEnable();
const StepConfigurator: FC<StepConfiguratorProps> = ({ value, defaultStep, setStep }) => {
const { customStep: value } = useGraphState();
const { period: { step: defaultStep } } = useTimeState();
const graphDispatch = useGraphDispatch();
const { period: duration } = useTimeState();
const prevDuration = usePrevious(duration.end - duration.start);
const [openOptions, setOpenOptions] = useState(false);
const [customStep, setCustomStep] = useState(value || defaultStep);
const [error, setError] = useState("");
const buttonRef = useRef<HTMLDivElement>(null);
const toggleOpenOptions = () => {
setOpenOptions(prev => !prev);
};
const handleCloseOptions = () => {
setOpenOptions(false);
};
const handleApply = (value?: string) => {
const step = value || customStep || defaultStep || "1s";
const durations = step.match(/[a-zA-Z]+/g) || [];
setStep(!durations.length ? `${step}s` : step);
const stepDur = !durations.length ? `${step}s` : step;
graphDispatch({ type: "SET_CUSTOM_STEP", payload: stepDur });
setCustomStep(stepDur);
setError("");
};
const handleFocus = () => {
if (document.activeElement instanceof HTMLInputElement) {
document.activeElement.select();
}
};
const handleEnter = () => {
handleApply();
handleCloseOptions();
};
const handleChangeStep = (value: string) => {
@ -46,28 +79,94 @@ const StepConfigurator: FC<StepConfiguratorProps> = ({ value, defaultStep, setSt
};
useEffect(() => {
if (value) handleChangeStep(value);
if (value) {
handleApply(value);
}
}, [value]);
useEffect(() => {
if (!value && defaultStep) {
handleApply(defaultStep);
}
}, [defaultStep]);
useEffect(() => {
const dur = duration.end - duration.start;
if (dur === prevDuration || !prevDuration) return;
if (defaultStep) {
handleApply(defaultStep);
}
}, [duration, prevDuration, defaultStep]);
return (
<TextField
label="Step value"
value={customStep}
error={error}
onChange={handleChangeStep}
onEnter={handleApply}
onBlur={handleApply}
endIcon={(
<Tooltip title="Reset step to default">
<Button
variant={"text"}
size={"small"}
startIcon={<RestartIcon/>}
onClick={handleReset}
<div
className="vm-step-control"
ref={buttonRef}
>
<Tooltip title="Query resolution step width">
<Button
className={appModeEnable ? "" : "vm-header-button"}
variant="contained"
color="primary"
startIcon={<TimelineIcon/>}
onClick={toggleOpenOptions}
>
STEP {customStep}
</Button>
</Tooltip>
<Popper
open={openOptions}
placement="bottom-right"
onClose={handleCloseOptions}
buttonRef={buttonRef}
>
<div className="vm-step-control-popper">
<TextField
autofocus
label="Step value"
value={customStep}
error={error}
onChange={handleChangeStep}
onEnter={handleEnter}
onFocus={handleFocus}
onBlur={handleApply}
endIcon={(
<Tooltip title={`Set default step value: ${defaultStep}`}>
<Button
size="small"
variant="text"
color="primary"
startIcon={<RestartIcon/>}
onClick={handleReset}
/>
</Tooltip>
)}
/>
</Tooltip>
)}
/>
<div className="vm-step-control-popper-info">
<code>step</code> - the <a
className="vm-link vm-link_colored"
href="https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations"
target="_blank"
rel="noreferrer"
>
interval
</a>
between datapoints, which must be returned from the range query.
The <code>query</code> is executed at
<code>start</code>, <code>start+step</code>, <code>start+2*step</code>, …, <code>end</code> timestamps.
<a
className="vm-link vm-link_colored"
href="https://docs.victoriametrics.com/keyConcepts.html#range-query"
target="_blank"
rel="noreferrer"
>
Read more about Range query
</a>
</div>
</div>
</Popper>
</div>
);
};

37
app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/style.scss Normal file
View file

@ -0,0 +1,37 @@
@use "src/styles/variables" as *;
@use "src/components/Main/Button/style" as *;
.vm-step-control {
display: inline-flex;
button {
text-transform: none;
}
&-popper {
display: grid;
gap: $padding-small;
max-width: 300px;
max-height: 208px;
overflow: auto;
padding: $padding-global;
font-size: $font-size;
&-info {
font-size: $font-size-small;
line-height: 1.6;
a {
margin: 0 0.2em;
}
code {
padding: 0.2em 0.4em;
margin: 0 0.2em;
font-size: 85%;
background-color: rgba($color-black, 0.05);
border-radius: 6px;
}
}
}
}

32
app/vmui/packages/vmui/src/components/ExploreMetrics/ExploreMetricsHeader/ExploreMetricsHeader.tsx
View file

@ -1,10 +1,6 @@
import React, { FC, useEffect, useMemo } from "preact/compat";
import React, { FC, useMemo } from "preact/compat";
import Select from "../../Main/Select/Select";
import StepConfigurator from "../../Configurators/StepConfigurator/StepConfigurator";
import "./style.scss";
import { useTimeState } from "../../../state/time/TimeStateContext";
import { useGraphDispatch, useGraphState } from "../../../state/graph/GraphStateContext";
import usePrevious from "../../../hooks/usePrevious";
import { GRAPH_SIZES } from "../../../constants/graph";
interface ExploreMetricsHeaderProps {
@ -36,28 +32,9 @@ const ExploreMetricsHeader: FC<ExploreMetricsHeaderProps> = ({
onToggleMetric,
onChangeSize
}) => {
const { period: { step }, duration } = useTimeState();
const { customStep } = useGraphState();
const graphDispatch = useGraphDispatch();
const prevDuration = usePrevious(duration);
const noInstanceText = useMemo(() => job ? "" : "No instances. Please select job", [job]);
const noMetricsText = useMemo(() => job ? "" : "No metric names. Please select job", [job]);
const handleChangeStep = (value: string) => {
graphDispatch({ type: "SET_CUSTOM_STEP", payload: value });
};
useEffect(() => {
if (duration === prevDuration || !prevDuration) return;
if (customStep) handleChangeStep(step || "1s");
}, [duration, prevDuration]);
useEffect(() => {
if (!customStep && step) handleChangeStep(step);
}, [step]);
return (
<div className="vm-explore-metrics-header vm-block">
<div className="vm-explore-metrics-header__job">
@ -81,13 +58,6 @@ const ExploreMetricsHeader: FC<ExploreMetricsHeaderProps> = ({
clearable
/>
</div>
<div className="vm-explore-metrics-header__step">
<StepConfigurator
defaultStep={step}
setStep={handleChangeStep}
value={customStep}
/>
</div>
<div className="vm-explore-metrics-header__size">
<Select
label="Size graphs"

4
app/vmui/packages/vmui/src/components/Layout/Footer/Footer.tsx
View file

@ -8,7 +8,7 @@ const Footer: FC = () => {
return <footer className="vm-footer">
<a
className="vm__link vm-footer__website"
className="vm-link vm-footer__website"
target="_blank"
href="https://victoriametrics.com/"
rel="noreferrer"
@ -17,7 +17,7 @@ const Footer: FC = () => {
victoriametrics.com
</a>
<a
className="vm__link"
className="vm-link"
target="_blank"
href="https://github.com/VictoriaMetrics/VictoriaMetrics/issues/new/choose"
rel="noreferrer"

11
app/vmui/packages/vmui/src/components/Layout/Header/Header.tsx
View file

@ -15,6 +15,7 @@ import Tabs from "../../Main/Tabs/Tabs";
import "./style.scss";
import classNames from "classnames";
import { useDashboardsState } from "../../../state/dashboards/DashboardsStateContext";
import StepConfigurator from "../../Configurators/StepConfigurator/StepConfigurator";
const Header: FC = () => {
const primaryColor = getCssVariable("color-primary");
@ -58,11 +59,6 @@ const Header: FC = () => {
const [activeMenu, setActiveMenu] = useState(pathname);
const handleChangeTab = (value: string) => {
setActiveMenu(value);
navigate(value);
};
const headerSetup = useMemo(() => {
return ((routerOptions[pathname] || {}) as RouterOptions).header || {};
}, [pathname]);
@ -90,7 +86,7 @@ const Header: FC = () => {
>
{!appModeEnable && (
<div
className="vm-header__logo"
className="vm-header-logo"
onClick={onClickLogo}
style={{ color }}
>
@ -99,13 +95,14 @@ const Header: FC = () => {
)}
<div className="vm-header-nav">
<Tabs
isNavLink
activeItem={activeMenu}
items={routes.filter(r => !r.hide)}
color={color}
onChange={handleChangeTab}
/>
</div>
<div className="vm-header__settings">
{headerSetup?.stepControl && <StepConfigurator/>}
{headerSetup?.timeSelector && <TimeSelector/>}
{headerSetup?.cardinalityDatePicker && <CardinalityDatePicker/>}
{headerSetup?.executionControls && <ExecutionControls/>}

25
app/vmui/packages/vmui/src/components/Layout/Header/style.scss
View file

@ -2,6 +2,7 @@
.vm-header {
display: flex;
flex-wrap: wrap;
align-items: center;
justify-content: flex-start;
padding: $padding-small $padding-medium;
@ -11,15 +12,35 @@
padding: $padding-small 0;
}
&__logo {
@media (max-width: 1200px) {
gap: $padding-global;
.vm-tabs {
gap: 0;
}
}
&-logo {
position: relative;
display: flex;
align-items: center;
justify-content: center;
justify-content: flex-start;
cursor: pointer;
width: 100%;
max-width: 65px;
min-width: 65px;
margin-bottom: 2px;
overflow: hidden;
@media (max-width: 1200px) {
min-width: 14px;
max-width: 14px;
}
svg {
max-width: 65px;
min-width: 65px;
}
}
&-nav {

11
app/vmui/packages/vmui/src/components/Main/Icons/index.tsx
View file

@ -333,3 +333,14 @@ export const ResizeIcon = () => (
<path d="M21 11V3h-8l3.29 3.29-10 10L3 13v8h8l-3.29-3.29 10-10z"></path>
</svg>
);
export const TimelineIcon = () => (
<svg
viewBox="0 0 24 24"
fill="currentColor"
>
<path
d="M23 8c0 1.1-.9 2-2 2-.18 0-.35-.02-.51-.07l-3.56 3.55c.05.16.07.34.07.52 0 1.1-.9 2-2 2s-2-.9-2-2c0-.18.02-.36.07-.52l-2.55-2.55c-.16.05-.34.07-.52.07s-.36-.02-.52-.07l-4.55 4.56c.05.16.07.33.07.51 0 1.1-.9 2-2 2s-2-.9-2-2 .9-2 2-2c.18 0 .35.02.51.07l4.56-4.55C8.02 9.36 8 9.18 8 9c0-1.1.9-2 2-2s2 .9 2 2c0 .18-.02.36-.07.52l2.55 2.55c.16-.05.34-.07.52-.07s.36.02.52.07l3.55-3.56C19.02 8.35 19 8.18 19 8c0-1.1.9-2 2-2s2 .9 2 2z"
></path>
</svg>
);

57
app/vmui/packages/vmui/src/components/Main/Tabs/TabItem.tsx Normal file
View file

@ -0,0 +1,57 @@
import React, { Component, FC, Ref } from "preact/compat";
import classNames from "classnames";
import { getCssVariable } from "../../../utils/theme";
import { TabItemType } from "./Tabs";
import TabItemWrapper from "./TabItemWrapper";
import "./style.scss";
interface TabItemProps {
activeItem: string
item: TabItemType
color?: string
onChange?: (value: string) => void
activeNavRef: Ref<Component>
isNavLink?: boolean
}
const TabItem: FC<TabItemProps> = ({
activeItem,
item,
color = getCssVariable("color-primary"),
activeNavRef,
onChange,
isNavLink
}) => {
const createHandlerClickTab = (value: string) => () => {
onChange && onChange(value);
};
return (
<TabItemWrapper
className={classNames({
"vm-tabs-item": true,
"vm-tabs-item_active": activeItem === item.value,
[item.className || ""]: item.className
})}
isNavLink={isNavLink}
to={item.value}
style={{ color: color }}
onClick={createHandlerClickTab(item.value)}
ref={activeItem === item.value ? activeNavRef : undefined}
>
{item.icon && (
<div
className={classNames({
"vm-tabs-item__icon": true,
"vm-tabs-item__icon_single": !item.label
})}
>
{item.icon}
</div>
)}
{item.label}
</TabItemWrapper>
);
};
export default TabItem;

29
app/vmui/packages/vmui/src/components/Main/Tabs/TabItemWrapper.tsx Normal file
View file

@ -0,0 +1,29 @@
import { ReactNode } from "react";
import React, { FC } from "preact/compat";
import { NavLink } from "react-router-dom";
interface TabItemWrapperProps {
to: string
isNavLink?: boolean
className: string
style: { color: string }
children: ReactNode
onClick: () => void
}
const TabItemWrapper: FC<TabItemWrapperProps> = ({ to, isNavLink, children, ...props }) => {
if (isNavLink) {
return (
<NavLink
to={to}
{...props}
>
{children}
</NavLink>
);
}
return <div {...props}>{children}</div>;
};
export default TabItemWrapper;

61
app/vmui/packages/vmui/src/components/Main/Tabs/Tabs.tsx
View file

@ -1,16 +1,24 @@
import React, { FC, useRef, useState } from "preact/compat";
import React, { Component, FC, useRef, useState } from "preact/compat";
import { ReactNode, useEffect } from "react";
import "./style.scss";
import classNames from "classnames";
import { getCssVariable } from "../../../utils/theme";
import useResize from "../../../hooks/useResize";
import TabItem from "./TabItem";
import "./style.scss";
export interface TabItemType {
value: string
label?: string
icon?: ReactNode
className?: string
}
interface TabsProps {
activeItem: string
items: {value: string, label?: string, icon?: ReactNode, className?: string}[]
items: TabItemType[]
color?: string
onChange: (value: string) => void
onChange?: (value: string) => void
indicatorPlacement?: "bottom" | "top"
isNavLink?: boolean
}
const Tabs: FC<TabsProps> = ({
@ -18,19 +26,16 @@ const Tabs: FC<TabsProps> = ({
items,
color = getCssVariable("color-primary"),
onChange,
indicatorPlacement = "bottom"
indicatorPlacement = "bottom",
isNavLink,
}) => {
const windowSize = useResize(document.body);
const activeNavRef = useRef<HTMLDivElement>(null);
const activeNavRef = useRef<Component>(null);
const [indicatorPosition, setIndicatorPosition] = useState({ left: 0, width: 0, bottom: 0 });
const createHandlerClickTab = (value: string) => () => {
onChange(value);
};
useEffect(() => {
if(activeNavRef.current) {
const { offsetLeft: left, offsetWidth: width, offsetHeight: height } = activeNavRef.current;
if(activeNavRef.current?.base instanceof HTMLElement) {
const { offsetLeft: left, offsetWidth: width, offsetHeight: height } = activeNavRef.current.base;
const positionTop = indicatorPlacement === "top";
setIndicatorPosition({ left, width, bottom: positionTop ? height - 2 : 0 });
}
@ -38,29 +43,15 @@ const Tabs: FC<TabsProps> = ({
return <div className="vm-tabs">
{items.map(item => (
<div
className={classNames({
"vm-tabs-item": true,
"vm-tabs-item_active": activeItem === item.value,
[item.className || ""]: item.className
})}
ref={activeItem === item.value ? activeNavRef : undefined}
<TabItem
key={item.value}
style={{ color: color }}
onClick={createHandlerClickTab(item.value)}
>
{item.icon && (
<div
className={classNames({
"vm-tabs-item__icon": true,
"vm-tabs-item__icon_single": !item.label
})}
>
{item.icon}
</div>
)}
{item.label}
</div>
activeItem={activeItem}
item={item}
onChange={onChange}
color={color}
activeNavRef={activeNavRef}
isNavLink={isNavLink}
/>
))}
<div
className="vm-tabs__indicator"

4
app/vmui/packages/vmui/src/components/Main/Tabs/style.scss
View file

@ -27,6 +27,10 @@
opacity: 1;
}
&:hover {
opacity: 0.8;
}
&__icon {
display: grid;
width: 15px;

5
app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx
View file

@ -1,9 +1,9 @@
import React, { FC, useEffect, useMemo, useRef, useState } from "preact/compat";
import React, { FC, useCallback, useEffect, useMemo, useRef, useState } from "preact/compat";
import { MetricResult } from "../../../api/types";
import LineChart from "../../Chart/LineChart/LineChart";
import { AlignedData as uPlotData, Series as uPlotSeries } from "uplot";
import Legend from "../../Chart/Legend/Legend";
import { getHideSeries, getLegendItem, getSeriesItem } from "../../../utils/uplot/series";
import { getHideSeries, getLegendItem, getSeriesItemContext } from "../../../utils/uplot/series";
import { getLimitsYAxis, getMinMaxBuffer, getTimeSeries } from "../../../utils/uplot/axes";
import { LegendItemType } from "../../../utils/uplot/types";
import { TimeParams } from "../../../types";
@ -59,6 +59,7 @@ const GraphView: FC<GraphViewProps> = ({
}) => {
const { timezone } = useTimeState();
const currentStep = useMemo(() => customStep || period.step || "1s", [period.step, customStep]);
const getSeriesItem = useCallback(getSeriesItemContext(), [data]);
const [dataChart, setDataChart] = useState<uPlotData>([[]]);
const [series, setSeries] = useState<uPlotSeries[]>([]);

10
app/vmui/packages/vmui/src/components/Views/TableView/TableView.tsx
View file

@ -41,13 +41,17 @@ const TableView: FC<GraphViewProps> = ({ data, displayColumns }) => {
return `${__name__} ${JSON.stringify(fields)}`;
};
const groups = new Set(data?.map(d => d.group));
const showQueryName = groups.size > 1;
const rows: InstantDataSeries[] = useMemo(() => {
const rows = data?.map(d => ({
metadata: sortedColumns.map(c => (tableCompact
? getNameForMetric(d)
? getNameForMetric(d, "", showQueryName)
: (d.metric[c.key] || "-")
)),
value: d.value ? d.value[1] : "-",
values: d.values ? d.values.map(([time, val]) => `${val} @${time}`) : [],
copyValue: getCopyValue(d.metric)
}));
const orderByValue = orderBy === "Value";
@ -171,8 +175,8 @@ const TableView: FC<GraphViewProps> = ({ data, displayColumns }) => {
{rowMeta}
</td>
))}
<td className="vm-table-cell vm-table-cell_right">
{row.value}
<td className="vm-table-cell vm-table-cell_right vm-table-cell_no-wrap">
{!row.values.length ? row.value : row.values.map(val => <p key={val}>{val}</p>)}
</td>
{hasCopyValue && (
<td className="vm-table-cell vm-table-cell_right">

8
app/vmui/packages/vmui/src/pages/ExploreMetrics/hooks/useSetQueryParams.ts
View file

@ -2,6 +2,7 @@ import { useEffect } from "react";
import { compactObject } from "../../../utils/object";
import { useTimeState } from "../../../state/time/TimeStateContext";
import { setQueryStringWithoutPageReload } from "../../../utils/query-string";
import { useGraphState } from "../../../state/graph/GraphStateContext";
interface queryProps {
job: string
@ -11,13 +12,14 @@ interface queryProps {
}
export const useSetQueryParams = ({ job, instance, metrics, size }: queryProps) => {
const { duration, relativeTime, period: { date, step } } = useTimeState();
const { duration, relativeTime, period: { date } } = useTimeState();
const { customStep } = useGraphState();
const setSearchParamsFromState = () => {
const params = compactObject({
["g0.range_input"]: duration,
["g0.end_input"]: date,
["g0.step_input"]: step,
["g0.step_input"]: customStep,
["g0.relative_time"]: relativeTime,
size,
job,
@ -28,6 +30,6 @@ export const useSetQueryParams = ({ job, instance, metrics, size }: queryProps)
setQueryStringWithoutPageReload(params);
};
useEffect(setSearchParamsFromState, [duration, relativeTime, date, step, job, instance, metrics, size]);
useEffect(setSearchParamsFromState, [duration, relativeTime, date, customStep, job, instance, metrics, size]);
useEffect(setSearchParamsFromState, []);
};

20
app/vmui/packages/vmui/src/pages/PredefinedPanels/PredefinedPanel/PredefinedPanel.tsx
View file

@ -4,7 +4,6 @@ import { AxisRange, YaxisState } from "../../../state/graph/reducer";
import GraphView from "../../../components/Views/GraphView/GraphView";
import { useFetchQuery } from "../../../hooks/useFetchQuery";
import Spinner from "../../../components/Main/Spinner/Spinner";
import StepConfigurator from "../../../components/Configurators/StepConfigurator/StepConfigurator";
import GraphSettings from "../../../components/Configurators/GraphSettings/GraphSettings";
import { marked } from "marked";
import { useTimeDispatch, useTimeState } from "../../../state/time/TimeStateContext";
@ -12,7 +11,7 @@ import { InfoIcon } from "../../../components/Main/Icons";
import "./style.scss";
import Alert from "../../../components/Main/Alert/Alert";
import Tooltip from "../../../components/Main/Tooltip/Tooltip";
import usePrevious from "../../../hooks/usePrevious";
import { useGraphState } from "../../../state/graph/GraphStateContext";
export interface PredefinedPanelsProps extends PanelSettings {
filename: string;
@ -28,13 +27,12 @@ const PredefinedPanel: FC<PredefinedPanelsProps> = ({
alias
}) => {
const { period, duration } = useTimeState();
const { period } = useTimeState();
const { customStep } = useGraphState();
const dispatch = useTimeDispatch();
const prevDuration = usePrevious(duration);
const containerRef = useRef<HTMLDivElement>(null);
const [visible, setVisible] = useState(true);
const [customStep, setCustomStep] = useState(period.step || "1s");
const [yaxis, setYaxis] = useState<YaxisState>({
limits: {
enable: false,
@ -77,11 +75,6 @@ const PredefinedPanel: FC<PredefinedPanelsProps> = ({
};
}, []);
useEffect(() => {
if (duration === prevDuration || !prevDuration) return;
if (customStep) setCustomStep(period.step || "1s");
}, [duration, prevDuration]);
if (!validExpr) return (
<Alert variant="error">
<code>&quot;expr&quot;</code> not found. Check the configuration file <b>{filename}</b>.
@ -123,13 +116,6 @@ const PredefinedPanel: FC<PredefinedPanelsProps> = ({
<h3 className="vm-predefined-panel-header__title">
{title || ""}
</h3>
<div className="vm-predefined-panel-header__step">
<StepConfigurator
defaultStep={period.step}
value={customStep}
setStep={setCustomStep}
/>
</div>
<GraphSettings
yaxis={yaxis}
setYaxisLimits={setYaxisLimits}

2
app/vmui/packages/vmui/src/pages/PredefinedPanels/PredefinedPanel/style.scss
View file

@ -4,7 +4,7 @@
&-header {
display: grid;
grid-template-columns: auto 1fr 160px auto;
grid-template-columns: auto 1fr auto;
align-items: center;
justify-content: flex-start;
gap: $padding-small;

8
app/vmui/packages/vmui/src/pages/PredefinedPanels/hooks/useSetQueryParams.ts
View file

@ -2,21 +2,23 @@ import { useEffect } from "react";
import { compactObject } from "../../../utils/object";
import { useTimeState } from "../../../state/time/TimeStateContext";
import { setQueryStringWithoutPageReload } from "../../../utils/query-string";
import { useGraphState } from "../../../state/graph/GraphStateContext";
export const useSetQueryParams = () => {
const { duration, relativeTime, period: { date, step } } = useTimeState();
const { duration, relativeTime, period: { date } } = useTimeState();
const { customStep } = useGraphState();
const setSearchParamsFromState = () => {
const params = compactObject({
["g0.range_input"]: duration,
["g0.end_input"]: date,
["g0.step_input"]: step,
["g0.step_input"]: customStep,
["g0.relative_time"]: relativeTime
});
setQueryStringWithoutPageReload(params);
};
useEffect(setSearchParamsFromState, [duration, relativeTime, date, step]);
useEffect(setSearchParamsFromState, [duration, relativeTime, date, customStep]);
useEffect(setSearchParamsFromState, []);
};

2
app/vmui/packages/vmui/src/pages/TracePage/index.tsx
View file

@ -148,7 +148,7 @@ const TracePage: FC = () => {
{"\n"}
In order to use tracing please refer to the doc:&nbsp;
<a
className="vm__link vm__link_colored"
className="vm-link vm-link_colored"
href="https://docs.victoriametrics.com/#query-tracing"
target="_blank"
rel="noreferrer"

3
app/vmui/packages/vmui/src/router/index.ts
View file

@ -11,6 +11,7 @@ const router = {
export interface RouterOptions {
title?: string,
header: {
stepControl?: boolean,
timeSelector?: boolean,
executionControls?: boolean,
globalSettings?: boolean,
@ -20,6 +21,7 @@ export interface RouterOptions {
const routerOptionsDefault = {
header: {
stepControl: true,
timeSelector: true,
executionControls: true,
}
@ -33,6 +35,7 @@ export const routerOptions: {[key: string]: RouterOptions} = {
[router.metrics]: {
title: "Explore metrics",
header: {
stepControl: true,
timeSelector: true,
}
},

2
app/vmui/packages/vmui/src/styles/components/link.scss
View file

@ -1,6 +1,6 @@
@use "src/styles/variables" as *;
.vm__link {
.vm-link {
transition: color 200ms ease;
cursor: pointer;

3
app/vmui/packages/vmui/src/styles/components/table.scss
View file

@ -30,7 +30,8 @@
padding: $padding-small;
border-bottom: $border-divider;
height: 40px;
vertical-align: middle;
vertical-align: top;
line-height: 25px;
&__content {
display: flex;

1
app/vmui/packages/vmui/src/types/index.ts
View file

@ -35,6 +35,7 @@ export interface DataSeries extends MetricBase{
export interface InstantDataSeries {
metadata: string[]; // just ordered columns
value: string;
values: string[]
copyValue: string;
}

41
app/vmui/packages/vmui/src/utils/color.ts
View file

@ -1,14 +1,35 @@
export const getColorFromString = (str: string): string => {
let hash = 0;
for (let i = 0; i < str.length; i++) {
hash = str.charCodeAt(i) + ((hash << 5) - hash);
export const baseContrastColors = [
"#e54040",
"#32a9dc",
"#2ee329",
"#7126a1",
"#e38f0f",
"#3d811a",
"#ffea00",
"#2d2d2d",
"#da42a6",
"#a44e0c",
];
export const getColorFromString = (text: string): string => {
const SEED = 16777215;
const FACTOR = 49979693;
let b = 1;
let d = 0;
let f = 1;
if (text.length > 0) {
for (let i = 0; i < text.length; i++) {
text[i].charCodeAt(0) > d && (d = text[i].charCodeAt(0));
f = parseInt(String(SEED / d));
b = (b + text[i].charCodeAt(0) * f * FACTOR) % SEED;
}
}
let colour = "#";
for (let i = 0; i < 3; i++) {
const value = (hash >> (i * 8)) & 0xFF;
colour += ("00" + value.toString(16)).substr(-2);
}
return colour;
let hex = ((b * text.length) % SEED).toString(16);
hex = hex.padEnd(6, hex);
return `#${hex}`;
};
export const hexToRGB = (hex: string): string => {

7
app/vmui/packages/vmui/src/utils/metric.ts
View file

@ -1,9 +1,12 @@
import { MetricBase } from "../api/types";
export const getNameForMetric = (result: MetricBase, alias?: string): string => {
export const getNameForMetric = (result: MetricBase, alias?: string, showQueryNum = true): string => {
const { __name__, ...freeFormFields } = result.metric;
const name = alias || `[Query ${result.group}] ${__name__ || ""}`;
const name = alias || `${showQueryNum ? `[Query ${result.group}] ` : ""}${__name__ || ""}`;
if (Object.keys(freeFormFields).length == 0) {
if (!name) {
return "value";
}
return name;
}
return `${name}{${Object.entries(freeFormFields).map(e =>

16
app/vmui/packages/vmui/src/utils/uplot/helpers.ts
View file

@ -1,5 +1,4 @@
import uPlot, { Axis } from "uplot";
import { getColorFromString } from "../color";
export const defaultOptions = {
legend: {
@ -37,7 +36,18 @@ export const formatPrettyNumber = (n: number | null | undefined, min = 0, max =
if (n === undefined || n === null) {
return "";
}
let digits = 3 + Math.floor(1 + Math.log10(Math.max(Math.abs(min), Math.abs(max))) - Math.log10(Math.abs(min - max)));
const range = Math.abs(max - min);
if (isNaN(range) || range == 0) {
// Return the constant number as is if the range isn't set of it is too small.
if (Math.abs(n) >= 1000) {
return n.toLocaleString("en-US");
}
return n.toString();
}
// Make sure n has 3 significant digits on the given range.
// This precision should be enough for most UX cases,
// since the remaining digits are usually a white noise.
let digits = 3 + Math.floor(1 + Math.log10(Math.max(Math.abs(min), Math.abs(max))) - Math.log10(range));
if (isNaN(digits) || digits > 20) {
digits = 20;
}
@ -74,6 +84,4 @@ export const sizeAxis = (u: uPlot, values: string[], axisIdx: number, cycleNum:
return Math.ceil(axisSize);
};
export const getColorLine = (label: string): string => getColorFromString(label);
export const getDashLine = (group: number): number[] => group <= 1 ? [] : [group*4, group*1.2];

36
app/vmui/packages/vmui/src/utils/uplot/series.ts
View file

@ -2,26 +2,34 @@ import { MetricResult } from "../../api/types";
import { Series } from "uplot";
import { getNameForMetric } from "../metric";
import { BarSeriesItem, Disp, Fill, LegendItemType, Stroke } from "./types";
import { getColorLine } from "./helpers";
import { HideSeriesArgs } from "./types";
import { baseContrastColors, getColorFromString } from "../color";
interface SeriesItem extends Series {
freeFormFields: {[key: string]: string};
}
export const getSeriesItem = (d: MetricResult, hideSeries: string[], alias: string[]): SeriesItem => {
const label = getNameForMetric(d, alias[d.group - 1]);
return {
label,
freeFormFields: d.metric,
width: 1.4,
stroke: getColorLine(label),
show: !includesHideSeries(label, hideSeries),
scale: "1",
points: {
size: 4.2,
width: 1.4
}
export const getSeriesItemContext = () => {
const colorState: {[key: string]: string} = {};
return (d: MetricResult, hideSeries: string[], alias: string[]): SeriesItem => {
const label = getNameForMetric(d, alias[d.group - 1]);
const countSavedColors = Object.keys(colorState).length;
const hasBasicColors = countSavedColors < baseContrastColors.length;
if (hasBasicColors) colorState[label] = colorState[label] || baseContrastColors[countSavedColors];
return {
label,
freeFormFields: d.metric,
width: 1.4,
stroke: colorState[label] || getColorFromString(label),
show: !includesHideSeries(label, hideSeries),
scale: "1",
points: {
size: 4.2,
width: 1.4
}
};
};
};

129
dashboards/operator.json
View file

@ -1,4 +1,38 @@
{
"__inputs": [],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "9.2.2"
},
{
"type": "panel",
"id": "graph",
"name": "Graph (old)",
"version": ""
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "stat",
"name": "Stat",
"version": ""
},
{
"type": "panel",
"id": "text",
"name": "Text",
"version": ""
}
],
"annotations": {
"list": [
{
@ -25,13 +59,16 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 38,
"iteration": 1653261405647,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"gridPos": {
"h": 1,
"w": 24,
@ -40,6 +77,15 @@
},
"id": 8,
"panels": [],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"refId": "A"
}
],
"title": "Overview",
"type": "row"
},
@ -56,10 +102,24 @@
},
"id": 24,
"options": {
"code": {
"language": "plaintext",
"showLineNumbers": false,
"showMiniMap": false
},
"content": "<div style=\"text-align: center;\">$version</div>",
"mode": "markdown"
},
"pluginVersion": "8.3.2",
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"refId": "A"
}
],
"title": "Version",
"type": "text"
},
@ -113,7 +173,7 @@
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.3.2",
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
@ -179,7 +239,7 @@
},
"textMode": "auto"
},
"pluginVersion": "8.3.2",
"pluginVersion": "9.2.2",
"targets": [
{
"datasource": {
@ -236,7 +296,7 @@
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.3.2",
"pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@ -324,7 +384,7 @@
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.3.2",
"pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@ -377,6 +437,10 @@
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"gridPos": {
"h": 1,
"w": 24,
@ -385,6 +449,15 @@
},
"id": 6,
"panels": [],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"refId": "A"
}
],
"title": "Troubleshooting",
"type": "row"
},
@ -762,6 +835,10 @@
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"gridPos": {
"h": 1,
"w": 24,
@ -770,6 +847,15 @@
},
"id": 4,
"panels": [],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"refId": "A"
}
],
"title": "resources",
"type": "row"
},
@ -1169,7 +1255,7 @@
}
],
"refresh": "",
"schemaVersion": 33,
"schemaVersion": 37,
"style": "dark",
"tags": [
"operator",
@ -1179,9 +1265,9 @@
"list": [
{
"current": {
"selected": true,
"text": "VictoriaMetrics",
"value": "VictoriaMetrics"
"selected": false,
"text": "cloud-c15",
"value": "cloud-c15"
},
"hide": 0,
"includeAll": false,
@ -1196,11 +1282,7 @@
"type": "datasource"
},
{
"current": {
"selected": false,
"text": "vm-operator-victoria-metrics-operator",
"value": "vm-operator-victoria-metrics-operator"
},
"current": {},
"datasource": {
"type": "prometheus",
"uid": "$ds"
@ -1222,11 +1304,7 @@
"type": "query"
},
{
"current": {
"selected": false,
"text": "All",
"value": "$__all"
},
"current": {},
"datasource": {
"type": "prometheus",
"uid": "$ds"
@ -1248,12 +1326,7 @@
"type": "query"
},
{
"current": {
"isNone": true,
"selected": false,
"text": "None",
"value": ""
},
"current": {},
"datasource": {
"type": "prometheus",
"uid": "$ds"
@ -1286,4 +1359,4 @@
"uid": "1H179hunk",
"version": 1,
"weekStart": ""
}
}

14
docs/CHANGELOG.md
View file

@ -16,9 +16,21 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add ability to show custom dashboards at vmui by specifying a path to a directory with dashboard config files via `-vmui.customDashboardsPath` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3322) and [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards).
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): apply the `step` globally to all the displayed graphs. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3574).
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): improve the appearance of graph lines by using more visually distinct colors. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3571).
* BUGFIX: do not slow down concurrently executed queries during assisted merges, since assisted merges already prioritize data ingestion over queries. The probability of assisted merges has been increased starting from [v1.85.0](https://docs.victoriametrics.com/CHANGELOG.html#v1850) because of internal refactoring. This could result in slowed down queries when there is a plenty of free CPU resources. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3647) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3641) issues.
* BUGFIX: reduce the increased CPU usage at `vmselect` to v1.85.3 level when processing heavy queries. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3641).
* BUGFIX: [retention filters](https://docs.victoriametrics.com/#retention-filters): fix `FATAL: cannot locate metric name for metricID=...: EOF` panic, which could occur when retention filters are enabled.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly cancel in-flight service discovery requests for [consul_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs) and [nomad_sd_configs](https://docs.victoriametrics.com/sd_configs.html#nomad_sd_configs) when the service list changes. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3468).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): [dockerswarm_sd_configs](https://docs.victoriametrics.com/sd_configs.html#dockerswarm_sd_configs): apply `filters` only to objects of the specified `role`. Previously filters were applied to all the objects, which could cause errors when different types of objects were used with filters that were not compatible with them. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3579).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): suppress all the scrape errors when `-promscrape.suppressScrapeErrors` is enabled. Previously some scrape errors were logged even if `-promscrape.suppressScrapeErrors` flag was set.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): consistently put the scrape url with scrape target labels to all error logs for failed scrapes. Previously some failed scrapes were logged without this information.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not send stale markers to remote storage for series exceeding the configured [series limit](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3660).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly apply [series limit](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter) when [staleness tracking](https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers) is disabled.
* BUGFIX: [Pushgateway import](https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format): properly return `200 OK` HTTP response code. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly parse `M` and `Mi` suffixes as `1e6` multipliers in `1M` and `1Mi` numeric constants. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3664). The issue has been introduced in [v1.86.0](https://docs.victoriametrics.com/CHANGELOG.html#v1860).
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly display range query results at `Table` view. For example, `up[5m]` query now shows all the raw samples for the last 5 minutes for the `up` metric at the `Table` view. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3516).
## [v1.86.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.86.1)

2
docs/guides/guide-delete-or-replace-metrics.md
View file

@ -75,7 +75,7 @@ When you're sure [time series selector](https://prometheus.io/docs/prometheus/la
<div class="with-copy" markdown="1">
```console
curl -s 'http://vmselect:8481/select/0/prometheus/api/v1/series?match[]=process_cpu_cores_available'
curl -s 'http://vmselect:8481/delete/0/prometheus/api/v1/admin/tsdb/delete_series?match[]=process_cpu_cores_available'
```
</div>

2
docs/guides/migrate-from-influx.md
View file

@ -123,7 +123,7 @@ In addition to InfluxDB line protocol, VictoriaMetrics supports many other ways
## Query data
VictoriaMetrics does not have a com\mand-line interface (CLI). Instead, it provides
VictoriaMetrics does not have a command-line interface (CLI). Instead, it provides
an [HTTP API](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#prometheus-querying-api-usage)
for serving read queries. This API is used in various integrations such as
[Grafana](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#grafana-setup). The same API is also used

12
go.mod
View file

@ -11,8 +11,8 @@ require (
// Do not use the original github.com/valyala/fasthttp because of issues
// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
github.com/VictoriaMetrics/fasthttp v1.1.0
github.com/VictoriaMetrics/metrics v1.23.0
github.com/VictoriaMetrics/metricsql v0.51.1
github.com/VictoriaMetrics/metrics v1.23.1
github.com/VictoriaMetrics/metricsql v0.51.2
github.com/aws/aws-sdk-go-v2 v1.17.3
github.com/aws/aws-sdk-go-v2/config v1.18.8
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.47
@ -42,7 +42,7 @@ require (
golang.org/x/net v0.5.0
golang.org/x/oauth2 v0.4.0
golang.org/x/sys v0.4.0
google.golang.org/api v0.106.0
google.golang.org/api v0.107.0
gopkg.in/yaml.v2 v2.4.0
)
@ -54,7 +54,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/aws/aws-sdk-go v1.44.177 // indirect
github.com/aws/aws-sdk-go v1.44.180 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.8 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect
@ -107,13 +107,13 @@ require (
go.opentelemetry.io/otel/trace v1.11.2 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/goleak v1.2.0 // indirect
golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a // indirect
golang.org/x/exp v0.0.0-20230113213754-f9f960f08ad4 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/text v0.6.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5 // indirect
google.golang.org/grpc v1.52.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect

24
go.sum
View file

@ -67,10 +67,10 @@ github.com/VictoriaMetrics/fastcache v1.12.0/go.mod h1:tjiYeEfYXCqacuvYw/7UoDIeJ
github.com/VictoriaMetrics/fasthttp v1.1.0 h1:3crd4YWHsMwu60GUXRH6OstowiFvqrwS4a/ueoLdLL0=
github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR2uydjiWvoLp5ZTqQ=
github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
github.com/VictoriaMetrics/metrics v1.23.0 h1:WzfqyzCaxUZip+OBbg1+lV33WChDSu4ssYII3nxtpeA=
github.com/VictoriaMetrics/metrics v1.23.0/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc=
github.com/VictoriaMetrics/metricsql v0.51.1 h1:gmh3ZGCDrqUTdhUrr87eJOXMOputDYs1PtLwTfySTsI=
github.com/VictoriaMetrics/metricsql v0.51.1/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
github.com/VictoriaMetrics/metrics v1.23.1 h1:/j8DzeJBxSpL2qSIdqnRFLvQQhbJyJbbEi22yMm7oL0=
github.com/VictoriaMetrics/metrics v1.23.1/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc=
github.com/VictoriaMetrics/metricsql v0.51.2 h1:GCbxti0I46x3Ld/WhcUyawvLr6J0x90IaMftkjosHJI=
github.com/VictoriaMetrics/metricsql v0.51.2/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
@ -87,8 +87,8 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu
github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo=
github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.44.177 h1:ckMJhU5Gj+4Rta+bJIUiUd7jvHom84aim3zkGPblq0s=
github.com/aws/aws-sdk-go v1.44.177/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.44.180 h1:VLZuAHI9fa/3WME5JjpVjcPCNfpGHVMiHx8sLHWhMgI=
github.com/aws/aws-sdk-go v1.44.180/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY=
github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
@ -487,8 +487,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU=
golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20230113213754-f9f960f08ad4 h1:CNkDRtCj8otM5CFz5jYvbr8ioXX8flVsLfDWEj0M5kk=
golang.org/x/exp v0.0.0-20230113213754-f9f960f08ad4/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -703,8 +703,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
google.golang.org/api v0.106.0 h1:ffmW0faWCwKkpbbtvlY/K/8fUl+JKvNS5CVzRoyfCv8=
google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
google.golang.org/api v0.107.0 h1:I2SlFjD8ZWabaIFOfeEDg3pf0BHJDh6iYQ1ic3Yu/UU=
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -742,8 +742,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5 h1:wJT65XLOzhpSPCdAmmKfz94SlmnQzDzjm3Cj9k3fsXY=
google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

9
lib/fs/fs_solaris.go
View file

@ -49,15 +49,8 @@ func createFlockFile(flockFile string) (*os.File, error) {
}
func mustGetFreeSpace(path string) uint64 {
d, err := os.Open(path)
if err != nil {
logger.Panicf("FATAL: cannot open dir for determining free disk space: %s", err)
}
defer MustClose(d)
fd := d.Fd()
var stat unix.Statvfs_t
if err := unix.Fstatvfs(int(fd), &stat); err != nil {
if err := unix.Statvfs(path, &stat); err != nil {
logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err)
}
return freeSpace(stat)

9
lib/fs/fs_unix.go
View file

@ -45,15 +45,8 @@ func createFlockFile(flockFile string) (*os.File, error) {
}
func mustGetFreeSpace(path string) uint64 {
d, err := os.Open(path)
if err != nil {
logger.Panicf("FATAL: cannot open dir for determining free disk space: %s", err)
}
defer MustClose(d)
fd := d.Fd()
var stat unix.Statfs_t
if err := unix.Fstatfs(int(fd), &stat); err != nil {
if err := unix.Statfs(path, &stat); err != nil {
logger.Panicf("FATAL: cannot determine free disk space on %q: %s", path, err)
}
return freeSpace(stat)

7
lib/mergeset/table.go
View file

@ -18,7 +18,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
)
@ -782,11 +781,8 @@ func (tb *Table) assistedMergeForInmemoryParts() {
return
}
// Prioritize assisted merges over searches.
storagepacelimiter.Search.Inc()
atomic.AddUint64(&tb.inmemoryAssistedMerges, 1)
err := tb.mergeInmemoryParts()
storagepacelimiter.Search.Dec()
if err == nil {
continue
}
@ -806,11 +802,8 @@ func (tb *Table) assistedMergeForFileParts() {
return
}
// Prioritize assisted merges over searches.
storagepacelimiter.Search.Inc()
atomic.AddUint64(&tb.fileAssistedMerges, 1)
err := tb.mergeExistingParts(false)
storagepacelimiter.Search.Dec()
if err == nil {
continue
}

63
lib/pacelimiter/pacelimiter.go
View file

@ -1,63 +0,0 @@
package pacelimiter
import (
"sync"
"sync/atomic"
)
// PaceLimiter throttles WaitIfNeeded callers while the number of Inc calls is bigger than the number of Dec calls.
//
// It is expected that Inc is called before performing high-priority work,
// while Dec is called when the work is done.
// WaitIfNeeded must be called inside the work which must be throttled (i.e. lower-priority work).
// It may be called in the loop before performing a part of low-priority work.
type PaceLimiter struct {
mu sync.Mutex
cond *sync.Cond
delaysTotal uint64
n int32
}
// New returns pace limiter that throttles WaitIfNeeded callers while the number of Inc calls is bigger than the number of Dec calls.
func New() *PaceLimiter {
var pl PaceLimiter
pl.cond = sync.NewCond(&pl.mu)
return &pl
}
// Inc increments pl.
func (pl *PaceLimiter) Inc() {
atomic.AddInt32(&pl.n, 1)
}
// Dec decrements pl.
func (pl *PaceLimiter) Dec() {
if atomic.AddInt32(&pl.n, -1) == 0 {
// Wake up all the goroutines blocked in WaitIfNeeded,
// since the number of Dec calls equals the number of Inc calls.
pl.cond.Broadcast()
}
}
// WaitIfNeeded blocks while the number of Inc calls is bigger than the number of Dec calls.
func (pl *PaceLimiter) WaitIfNeeded() {
if atomic.LoadInt32(&pl.n) <= 0 {
// Fast path - there is no need in lock.
return
}
// Slow path - wait until Dec is called.
pl.mu.Lock()
for atomic.LoadInt32(&pl.n) > 0 {
pl.delaysTotal++
pl.cond.Wait()
}
pl.mu.Unlock()
}
// DelaysTotal returns the number of delays inside WaitIfNeeded.
func (pl *PaceLimiter) DelaysTotal() uint64 {
pl.mu.Lock()
n := pl.delaysTotal
pl.mu.Unlock()
return n
}

144
lib/pacelimiter/pacelimiter_test.go
View file

@ -1,144 +0,0 @@
package pacelimiter
import (
"fmt"
"runtime"
"sync"
"testing"
"time"
)
func TestPacelimiter(t *testing.T) {
t.Run("nonblocking", func(t *testing.T) {
pl := New()
ch := make(chan struct{}, 10)
for i := 0; i < cap(ch); i++ {
go func() {
for j := 0; j < 10; j++ {
pl.WaitIfNeeded()
runtime.Gosched()
}
ch <- struct{}{}
}()
}
// Check that all the goroutines are finished.
timeoutCh := time.After(5 * time.Second)
for i := 0; i < cap(ch); i++ {
select {
case <-ch:
case <-timeoutCh:
t.Fatalf("timeout")
}
}
if n := pl.DelaysTotal(); n > 0 {
t.Fatalf("unexpected non-zero number of delays: %d", n)
}
})
t.Run("blocking", func(t *testing.T) {
pl := New()
pl.Inc()
ch := make(chan struct{}, 10)
var wg sync.WaitGroup
for i := 0; i < cap(ch); i++ {
wg.Add(1)
go func() {
wg.Done()
for j := 0; j < 10; j++ {
pl.WaitIfNeeded()
}
ch <- struct{}{}
}()
}
// Check that all the goroutines created above are started and blocked in WaitIfNeeded
wg.Wait()
select {
case <-ch:
t.Fatalf("the pl must be blocked")
default:
}
// Unblock goroutines and check that they are unblocked.
pl.Dec()
timeoutCh := time.After(5 * time.Second)
for i := 0; i < cap(ch); i++ {
select {
case <-ch:
case <-timeoutCh:
t.Fatalf("timeout")
}
}
if n := pl.DelaysTotal(); n == 0 {
t.Fatalf("expecting non-zero number of delays")
}
// Verify that the pl is unblocked now.
pl.WaitIfNeeded()
// Verify that negative count doesn't block pl.
pl.Dec()
pl.WaitIfNeeded()
if n := pl.DelaysTotal(); n == 0 {
t.Fatalf("expecting non-zero number of delays after subsequent pl.Dec()")
}
})
t.Run("negative_count", func(t *testing.T) {
n := 10
pl := New()
for i := 0; i < n; i++ {
pl.Dec()
}
doneCh := make(chan error)
go func() {
defer close(doneCh)
for i := 0; i < n; i++ {
pl.Inc()
pl.WaitIfNeeded()
if n := pl.DelaysTotal(); n != 0 {
doneCh <- fmt.Errorf("expecting zero number of delays")
return
}
}
doneCh <- nil
}()
select {
case err := <-doneCh:
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
case <-time.After(5 * time.Second):
t.Fatalf("timeout")
}
})
t.Run("concurrent_inc_dec", func(t *testing.T) {
pl := New()
ch := make(chan struct{}, 10)
for i := 0; i < cap(ch); i++ {
go func() {
for j := 0; j < 10; j++ {
pl.Inc()
runtime.Gosched()
pl.Dec()
}
ch <- struct{}{}
}()
}
// Verify that all the goroutines are finished
timeoutCh := time.After(5 * time.Second)
for i := 0; i < cap(ch); i++ {
select {
case <-ch:
case <-timeoutCh:
t.Fatalf("timeout")
}
}
// Verify that the pl is unblocked.
pl.WaitIfNeeded()
if n := pl.DelaysTotal(); n > 0 {
t.Fatalf("expecting zer number of delays; got %d", n)
}
})
}

13
lib/promscrape/config.go
View file

@ -41,9 +41,10 @@ import (
)
var (
noStaleMarkers = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series")
strictParse = flag.Bool("promscrape.config.strictParse", true, "Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields")
dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+
noStaleMarkers = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series")
seriesLimitPerTarget = flag.Int("promscrape.seriesLimitPerTarget", 0, "Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info")
strictParse = flag.Bool("promscrape.config.strictParse", true, "Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields")
dryRun = flag.Bool("promscrape.config.dryRun", false, "Checks -promscrape.config file for errors and unsupported fields and then exits. "+
"Returns non-zero exit code on parsing errors and emits these errors to stderr. "+
"See also -promscrape.config.strictParse command-line flag. "+
"Pass -loggerLevel=ERROR if you don't need to see info messages in the output.")
@ -971,6 +972,10 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
if sc.NoStaleMarkers != nil {
noStaleTracking = *sc.NoStaleMarkers
}
seriesLimit := *seriesLimitPerTarget
if sc.SeriesLimit > 0 {
seriesLimit = sc.SeriesLimit
}
swc := &scrapeWorkConfig{
scrapeInterval: scrapeInterval,
scrapeIntervalString: scrapeInterval.String(),
@ -995,7 +1000,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
streamParse: sc.StreamParse,
scrapeAlignInterval: sc.ScrapeAlignInterval.Duration(),
scrapeOffset: sc.ScrapeOffset.Duration(),
seriesLimit: sc.SeriesLimit,
seriesLimit: seriesLimit,
noStaleMarkers: noStaleTracking,
}
return swc, nil

5
lib/promscrape/discovery/consul/api.go
View file

@ -1,6 +1,7 @@
package consul
import (
"context"
"flag"
"fmt"
"net/http"
@ -157,7 +158,7 @@ func maxWaitTime() time.Duration {
// getBlockingAPIResponse perfoms blocking request to Consul via client and returns response.
//
// See https://www.consul.io/api-docs/features/blocking .
func getBlockingAPIResponse(client *discoveryutils.Client, path string, index int64) ([]byte, int64, error) {
func getBlockingAPIResponse(ctx context.Context, client *discoveryutils.Client, path string, index int64) ([]byte, int64, error) {
path += "&index=" + strconv.FormatInt(index, 10)
path += "&wait=" + fmt.Sprintf("%ds", int(maxWaitTime().Seconds()))
getMeta := func(resp *http.Response) {
@ -182,7 +183,7 @@ func getBlockingAPIResponse(client *discoveryutils.Client, path string, index in
}
index = newIndex
}
data, err := client.GetBlockingAPIResponse(path, getMeta)
data, err := client.GetBlockingAPIResponseCtx(ctx, path, getMeta)
if err != nil {
return nil, index, fmt.Errorf("cannot perform blocking Consul API request at %q: %w", path, err)
}

33
lib/promscrape/discovery/consul/watch.go
View file

@ -34,15 +34,17 @@ type consulWatcher struct {
servicesLock sync.Mutex
services map[string]*serviceWatcher
stopCh chan struct{}
stoppedCh chan struct{}
}
type serviceWatcher struct {
serviceName string
serviceNodes []ServiceNode
stopCh chan struct{}
stoppedCh chan struct{}
stoppedCh chan struct{}
requestCtx context.Context
requestCancel context.CancelFunc
}
// newConsulWatcher creates new watcher and starts background service discovery for Consul.
@ -71,7 +73,6 @@ func newConsulWatcher(client *discoveryutils.Client, sdc *SDConfig, datacenter,
watchServices: sdc.Services,
watchTags: sdc.Tags,
services: make(map[string]*serviceWatcher),
stopCh: make(chan struct{}),
stoppedCh: make(chan struct{}),
}
initCh := make(chan struct{})
@ -85,7 +86,6 @@ func newConsulWatcher(client *discoveryutils.Client, sdc *SDConfig, datacenter,
}
func (cw *consulWatcher) mustStop() {
close(cw.stopCh)
cw.client.Stop()
<-cw.stoppedCh
}
@ -100,10 +100,12 @@ func (cw *consulWatcher) updateServices(serviceNames []string) {
// The watcher for serviceName already exists.
continue
}
ctx, cancel := context.WithCancel(cw.client.Context())
sw := &serviceWatcher{
serviceName: serviceName,
stopCh: make(chan struct{}),
stoppedCh: make(chan struct{}),
serviceName: serviceName,
stoppedCh: make(chan struct{}),
requestCtx: ctx,
requestCancel: cancel,
}
cw.services[serviceName] = sw
serviceWatchersCreated.Inc()
@ -126,7 +128,7 @@ func (cw *consulWatcher) updateServices(serviceNames []string) {
if _, ok := newServiceNamesMap[serviceName]; ok {
continue
}
close(sw.stopCh)
sw.requestCancel()
delete(cw.services, serviceName)
swsStopped = append(swsStopped, sw)
}
@ -173,24 +175,26 @@ func (cw *consulWatcher) watchForServicesUpdates(initCh chan struct{}) {
checkInterval := getCheckInterval()
ticker := time.NewTicker(checkInterval / 2)
defer ticker.Stop()
stopCh := cw.client.Context().Done()
for {
select {
case <-ticker.C:
f()
case <-cw.stopCh:
case <-stopCh:
logger.Infof("stopping Consul service watchers for %q", apiServer)
startTime := time.Now()
var swsStopped []*serviceWatcher
cw.servicesLock.Lock()
for _, sw := range cw.services {
close(sw.stopCh)
sw.requestCancel()
swsStopped = append(swsStopped, sw)
}
cw.servicesLock.Unlock()
for _, sw := range swsStopped {
<-sw.stoppedCh
serviceWatchersStopped.Inc()
}
logger.Infof("stopped Consul service watcher for %q in %.3f seconds", apiServer, time.Since(startTime).Seconds())
return
@ -209,7 +213,7 @@ var (
// It returns an empty serviceNames list if response contains the same index.
func (cw *consulWatcher) getBlockingServiceNames(index int64) ([]string, int64, error) {
path := "/v1/catalog/services" + cw.serviceNamesQueryArgs
data, newIndex, err := getBlockingAPIResponse(cw.client, path, index)
data, newIndex, err := getBlockingAPIResponse(cw.client.Context(), cw.client, path, index)
if err != nil {
return nil, index, err
}
@ -242,7 +246,7 @@ func (sw *serviceWatcher) watchForServiceNodesUpdates(cw *consulWatcher, initWG
index := int64(0)
path := "/v1/health/service/" + sw.serviceName + cw.serviceNodesQueryArgs
f := func() {
data, newIndex, err := getBlockingAPIResponse(cw.client, path, index)
data, newIndex, err := getBlockingAPIResponse(sw.requestCtx, cw.client, path, index)
if err != nil {
if !errors.Is(err, context.Canceled) {
logger.Errorf("cannot obtain Consul serviceNodes for serviceName=%q from %q: %s", sw.serviceName, apiServer, err)
@ -273,11 +277,12 @@ func (sw *serviceWatcher) watchForServiceNodesUpdates(cw *consulWatcher, initWG
checkInterval := getCheckInterval()
ticker := time.NewTicker(checkInterval / 2)
defer ticker.Stop()
stopCh := sw.requestCtx.Done()
for {
select {
case <-ticker.C:
f()
case <-sw.stopCh:
case <-stopCh:
return
}
}

5
lib/promscrape/discovery/nomad/api.go
View file

@ -1,6 +1,7 @@
package nomad
import (
"context"
"flag"
"fmt"
"net/http"
@ -116,7 +117,7 @@ func maxWaitTime() time.Duration {
// getBlockingAPIResponse perfoms blocking request to Nomad via client and returns response.
// See https://developer.hashicorp.com/nomad/api-docs#blocking-queries .
func getBlockingAPIResponse(client *discoveryutils.Client, path string, index int64) ([]byte, int64, error) {
func getBlockingAPIResponse(ctx context.Context, client *discoveryutils.Client, path string, index int64) ([]byte, int64, error) {
path += "&index=" + strconv.FormatInt(index, 10)
path += "&wait=" + fmt.Sprintf("%ds", int(maxWaitTime().Seconds()))
getMeta := func(resp *http.Response) {
@ -142,7 +143,7 @@ func getBlockingAPIResponse(client *discoveryutils.Client, path string, index in
}
index = newIndex
}
data, err := client.GetBlockingAPIResponse(path, getMeta)
data, err := client.GetBlockingAPIResponseCtx(ctx, path, getMeta)
if err != nil {
return nil, index, fmt.Errorf("cannot perform blocking Nomad API request at %q: %w", path, err)
}

33
lib/promscrape/discovery/nomad/watch.go
View file

@ -30,15 +30,17 @@ type nomadWatcher struct {
servicesLock sync.Mutex
services map[string]*serviceWatcher
stopCh chan struct{}
stoppedCh chan struct{}
}
type serviceWatcher struct {
serviceName string
services []Service
stopCh chan struct{}
stoppedCh chan struct{}
stoppedCh chan struct{}
requestCtx context.Context
requestCancel context.CancelFunc
}
// newNomadWatcher creates new watcher and starts background service discovery for Nomad.
@ -62,7 +64,6 @@ func newNomadWatcher(client *discoveryutils.Client, sdc *SDConfig, namespace, re
client: client,
serviceNamesQueryArgs: queryArgs,
services: make(map[string]*serviceWatcher),
stopCh: make(chan struct{}),
stoppedCh: make(chan struct{}),
}
initCh := make(chan struct{})
@ -76,7 +77,6 @@ func newNomadWatcher(client *discoveryutils.Client, sdc *SDConfig, namespace, re
}
func (cw *nomadWatcher) mustStop() {
close(cw.stopCh)
cw.client.Stop()
<-cw.stoppedCh
}
@ -91,10 +91,12 @@ func (cw *nomadWatcher) updateServices(serviceNames []string) {
// The watcher for serviceName already exists.
continue
}
ctx, cancel := context.WithCancel(cw.client.Context())
sw := &serviceWatcher{
serviceName: serviceName,
stopCh: make(chan struct{}),
stoppedCh: make(chan struct{}),
serviceName: serviceName,
stoppedCh: make(chan struct{}),
requestCtx: ctx,
requestCancel: cancel,
}
cw.services[serviceName] = sw
serviceWatchersCreated.Inc()
@ -117,7 +119,7 @@ func (cw *nomadWatcher) updateServices(serviceNames []string) {
if _, ok := newServiceNamesMap[serviceName]; ok {
continue
}
close(sw.stopCh)
sw.requestCancel()
delete(cw.services, serviceName)
swsStopped = append(swsStopped, sw)
}
@ -164,24 +166,26 @@ func (cw *nomadWatcher) watchForServicesUpdates(initCh chan struct{}) {
checkInterval := getCheckInterval()
ticker := time.NewTicker(checkInterval / 2)
defer ticker.Stop()
stopCh := cw.client.Context().Done()
for {
select {
case <-ticker.C:
f()
case <-cw.stopCh:
case <-stopCh:
logger.Infof("stopping Nomad service watchers for %q", apiServer)
startTime := time.Now()
var swsStopped []*serviceWatcher
cw.servicesLock.Lock()
for _, sw := range cw.services {
close(sw.stopCh)
sw.requestCancel()
swsStopped = append(swsStopped, sw)
}
cw.servicesLock.Unlock()
for _, sw := range swsStopped {
<-sw.stoppedCh
serviceWatchersStopped.Inc()
}
logger.Infof("stopped Nomad service watcher for %q in %.3f seconds", apiServer, time.Since(startTime).Seconds())
return
@ -200,7 +204,7 @@ var (
// It returns an empty serviceNames list if response contains the same index.
func (cw *nomadWatcher) getBlockingServiceNames(index int64) ([]string, int64, error) {
path := "/v1/services" + cw.serviceNamesQueryArgs
data, newIndex, err := getBlockingAPIResponse(cw.client, path, index)
data, newIndex, err := getBlockingAPIResponse(cw.client.Context(), cw.client, path, index)
if err != nil {
return nil, index, err
}
@ -244,7 +248,7 @@ func (sw *serviceWatcher) watchForServiceAddressUpdates(nw *nomadWatcher, initWG
// TODO: Maybe use a different query arg.
path := "/v1/service/" + sw.serviceName + nw.serviceNamesQueryArgs
f := func() {
data, newIndex, err := getBlockingAPIResponse(nw.client, path, index)
data, newIndex, err := getBlockingAPIResponse(sw.requestCtx, nw.client, path, index)
if err != nil {
if !errors.Is(err, context.Canceled) {
logger.Errorf("cannot obtain Nomad services for serviceName=%q from %q: %s", sw.serviceName, apiServer, err)
@ -275,11 +279,12 @@ func (sw *serviceWatcher) watchForServiceAddressUpdates(nw *nomadWatcher, initWG
checkInterval := getCheckInterval()
ticker := time.NewTicker(checkInterval / 2)
defer ticker.Stop()
stopCh := sw.requestCtx.Done()
for {
select {
case <-ticker.C:
f()
case <-sw.stopCh:
case <-stopCh:
return
}
}

18
lib/promscrape/discoveryutils/client.go
View file

@ -159,6 +159,11 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
return c, nil
}
// Context returns context for the client requests.
func (c *Client) Context() context.Context {
return c.clientCtx
}
// GetAPIResponseWithReqParams returns response for given absolute path with optional callback for request.
func (c *Client) GetAPIResponseWithReqParams(path string, modifyRequest func(request *http.Request)) ([]byte, error) {
return c.getAPIResponse(path, modifyRequest)
@ -185,16 +190,21 @@ func (c *Client) getAPIResponse(path string, modifyRequest func(request *http.Re
defer func() {
<-concurrencyLimitCh
}()
return c.getAPIResponseWithParamsAndClient(c.client, path, modifyRequest, nil)
return c.getAPIResponseWithParamsAndClientCtx(c.clientCtx, c.client, path, modifyRequest, nil)
}
// GetBlockingAPIResponse returns response for given absolute path with blocking client and optional callback for api response,
func (c *Client) GetBlockingAPIResponse(path string, inspectResponse func(resp *http.Response)) ([]byte, error) {
return c.getAPIResponseWithParamsAndClient(c.blockingClient, path, nil, inspectResponse)
return c.getAPIResponseWithParamsAndClientCtx(c.clientCtx, c.blockingClient, path, nil, inspectResponse)
}
// GetBlockingAPIResponseCtx returns response for given absolute path with blocking client and optional callback for api response,
func (c *Client) GetBlockingAPIResponseCtx(ctx context.Context, path string, inspectResponse func(resp *http.Response)) ([]byte, error) {
return c.getAPIResponseWithParamsAndClientCtx(ctx, c.blockingClient, path, nil, inspectResponse)
}
// getAPIResponseWithParamsAndClient returns response for the given absolute path with optional callback for request and for response.
func (c *Client) getAPIResponseWithParamsAndClient(client *HTTPClient, path string, modifyRequest func(req *http.Request), inspectResponse func(resp *http.Response)) ([]byte, error) {
func (c *Client) getAPIResponseWithParamsAndClientCtx(ctx context.Context, client *HTTPClient, path string, modifyRequest func(req *http.Request), inspectResponse func(resp *http.Response)) ([]byte, error) {
requestURL := c.apiServer + path
u, err := url.Parse(requestURL)
if err != nil {
@ -202,7 +212,7 @@ func (c *Client) getAPIResponseWithParamsAndClient(client *HTTPClient, path stri
}
deadline := time.Now().Add(client.ReadTimeout)
ctx, cancel := context.WithDeadline(c.clientCtx, deadline)
ctx, cancel := context.WithDeadline(ctx, deadline)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil {

46
lib/promscrape/scrapework.go
View file

@ -37,7 +37,6 @@ var (
"See also -promscrape.suppressScrapeErrorsDelay")
suppressScrapeErrorsDelay = flag.Duration("promscrape.suppressScrapeErrorsDelay", 0, "The delay for suppressing repeated scrape errors logging per each scrape targets. "+
"This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors")
seriesLimitPerTarget = flag.Int("promscrape.seriesLimitPerTarget", 0, "Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter for more info")
minResponseSizeForStreamParse = flagutil.NewBytes("promscrape.minResponseSizeForStreamParse", 1e6, "The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode")
)
@ -451,7 +450,7 @@ func (sw *scrapeWork) processScrapedData(scrapeTimestamp, realTimestamp int64, b
wc := writeRequestCtxPool.Get(sw.prevLabelsLen)
lastScrape := sw.loadLastScrape()
bodyString := bytesutil.ToUnsafeString(body.B)
areIdenticalSeries := sw.Config.NoStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
areIdenticalSeries := sw.areIdenticalSeries(lastScrape, bodyString)
if err != nil {
up = 0
scrapesFailed.Inc()
@ -485,9 +484,6 @@ func (sw *scrapeWork) processScrapedData(scrapeTimestamp, realTimestamp int64, b
samplesDropped := 0
if sw.seriesLimitExceeded || !areIdenticalSeries {
samplesDropped = sw.applySeriesLimit(wc)
if samplesDropped > 0 {
sw.seriesLimitExceeded = true
}
}
am := &autoMetrics{
up: up,
@ -577,7 +573,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
err = sbr.Init(sr)
if err == nil {
bodyString = bytesutil.ToUnsafeString(sbr.body)
areIdenticalSeries = sw.Config.NoStaleMarkers || parser.AreIdenticalSeriesFast(lastScrape, bodyString)
areIdenticalSeries = sw.areIdenticalSeries(lastScrape, bodyString)
err = parser.ParseStream(&sbr, scrapeTimestamp, false, func(rows []parser.Row) error {
mu.Lock()
defer mu.Unlock()
@ -594,9 +590,6 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
}
if sw.seriesLimitExceeded || !areIdenticalSeries {
samplesDropped += sw.applySeriesLimit(wc)
if samplesDropped > 0 && !sw.seriesLimitExceeded {
sw.seriesLimitExceeded = true
}
}
// Push the collected rows to sw before returning from the callback, since they cannot be held
// after returning from the callback - this will result in data race.
@ -655,6 +648,15 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
return err
}
func (sw *scrapeWork) areIdenticalSeries(prevData, currData string) bool {
if sw.Config.NoStaleMarkers && sw.Config.SeriesLimit <= 0 {
// Do not spend CPU time on tracking the changes in series if stale markers are disabled.
// The check for series_limit is needed for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3660
return true
}
return parser.AreIdenticalSeriesFast(prevData, currData)
}
// leveledWriteRequestCtxPool allows reducing memory usage when writeRequesCtx
// structs contain mixed number of labels.
//
@ -738,17 +740,13 @@ func (sw *scrapeWork) getSeriesAdded(lastScrape, currScrape string) int {
}
func (sw *scrapeWork) applySeriesLimit(wc *writeRequestCtx) int {
seriesLimit := *seriesLimitPerTarget
if sw.Config.SeriesLimit > 0 {
seriesLimit = sw.Config.SeriesLimit
}
if sw.seriesLimiter == nil && seriesLimit > 0 {
sw.seriesLimiter = bloomfilter.NewLimiter(seriesLimit, 24*time.Hour)
}
sl := sw.seriesLimiter
if sl == nil {
if sw.Config.SeriesLimit <= 0 {
return 0
}
if sw.seriesLimiter == nil {
sw.seriesLimiter = bloomfilter.NewLimiter(sw.Config.SeriesLimit, 24*time.Hour)
}
sl := sw.seriesLimiter
dstSeries := wc.writeRequest.Timeseries[:0]
samplesDropped := 0
for _, ts := range wc.writeRequest.Timeseries {
@ -761,6 +759,9 @@ func (sw *scrapeWork) applySeriesLimit(wc *writeRequestCtx) int {
}
prompbmarshal.ResetTimeSeries(wc.writeRequest.Timeseries[len(dstSeries):])
wc.writeRequest.Timeseries = dstSeries
if samplesDropped > 0 && !sw.seriesLimitExceeded {
sw.seriesLimitExceeded = true
}
return samplesDropped
}
@ -774,7 +775,7 @@ func (sw *scrapeWork) sendStaleSeries(lastScrape, currScrape string, timestamp i
}
wc := &writeRequestCtx{}
if bodyString != "" {
wc.rows.Unmarshal(bodyString)
wc.rows.UnmarshalWithErrLogger(bodyString, sw.logError)
srcRows := wc.rows.Rows
for i := range srcRows {
sw.addRowToTimeseries(wc, &srcRows[i], timestamp, true)
@ -784,6 +785,13 @@ func (sw *scrapeWork) sendStaleSeries(lastScrape, currScrape string, timestamp i
am := &autoMetrics{}
sw.addAutoMetrics(am, wc, timestamp)
}
// Apply series limit to stale markers in order to prevent sending stale markers for newly created series.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3660
if sw.seriesLimitExceeded {
sw.applySeriesLimit(wc)
}
series := wc.writeRequest.Timeseries
if len(series) == 0 {
return

13
lib/storage/partition.go
View file

@ -20,7 +20,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/mergeset"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
)
@ -610,14 +609,8 @@ func (pt *partition) assistedMergeForInmemoryParts() {
return
}
// There are too many unmerged inmemory parts.
// This usually means that the app cannot keep up with the data ingestion rate.
// Assist with mering inmemory parts.
// Prioritize assisted merges over searches.
storagepacelimiter.Search.Inc()
atomic.AddUint64(&pt.inmemoryAssistedMerges, 1)
err := pt.mergeInmemoryParts()
storagepacelimiter.Search.Dec()
if err == nil {
continue
}
@ -637,14 +630,8 @@ func (pt *partition) assistedMergeForSmallParts() {
return
}
// There are too many unmerged small parts.
// This usually means that the app cannot keep up with the data ingestion rate.
// Assist with mering small parts.
// Prioritize assisted merges over searches.
storagepacelimiter.Search.Inc()
atomic.AddUint64(&pt.smallAssistedMerges, 1)
err := pt.mergeExistingParts(false)
storagepacelimiter.Search.Dec()
if err == nil {
continue
}

2
lib/storage/search.go
View file

@ -10,7 +10,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter"
)
// BlockRef references a Block.
@ -448,7 +447,6 @@ func checkSearchDeadlineAndPace(deadline uint64) error {
if fasttime.UnixTimestamp() > deadline {
return ErrDeadlineExceeded
}
storagepacelimiter.Search.WaitIfNeeded()
return nil
}

5
lib/storage/storage.go
View file

@ -26,7 +26,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storagepacelimiter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
"github.com/VictoriaMetrics/fastcache"
@ -446,8 +445,6 @@ type Metrics struct {
TooSmallTimestampRows uint64
TooBigTimestampRows uint64
SearchDelays uint64
SlowRowInserts uint64
SlowPerDayIndexInserts uint64
SlowMetricNameLoads uint64
@ -517,8 +514,6 @@ func (s *Storage) UpdateMetrics(m *Metrics) {
m.TooSmallTimestampRows += atomic.LoadUint64(&s.tooSmallTimestampRows)
m.TooBigTimestampRows += atomic.LoadUint64(&s.tooBigTimestampRows)
m.SearchDelays = storagepacelimiter.Search.DelaysTotal()
m.SlowRowInserts += atomic.LoadUint64(&s.slowRowInserts)
m.SlowPerDayIndexInserts += atomic.LoadUint64(&s.slowPerDayIndexInserts)
m.SlowMetricNameLoads += atomic.LoadUint64(&s.slowMetricNameLoads)

10
lib/storagepacelimiter/storagepacelimiter.go
View file

@ -1,10 +0,0 @@
package storagepacelimiter
import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pacelimiter"
)
// Search limits the pace of search calls when there is at least a single in-flight assisted merge.
//
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/291
var Search = pacelimiter.New()

12
vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go generated vendored
View file

@ -9,6 +9,7 @@ import (
"os"
"strconv"
"strings"
"sync/atomic"
"time"
)
@ -48,6 +49,7 @@ func writeProcessMetrics(w io.Writer) {
log.Printf("ERROR: metrics: cannot open %s: %s", statFilepath, err)
return
}
// Search for the end of command.
n := bytes.LastIndex(data, []byte(") "))
if n < 0 {
@ -85,12 +87,20 @@ func writeProcessMetrics(w io.Writer) {
writeIOMetrics(w)
}
var procSelfIOErrLogged uint32
func writeIOMetrics(w io.Writer) {
ioFilepath := "/proc/self/io"
data, err := ioutil.ReadFile(ioFilepath)
if err != nil {
log.Printf("ERROR: metrics: cannot open %q: %s", ioFilepath, err)
// Do not spam the logs with errors - this error cannot be fixed without process restart.
// See https://github.com/VictoriaMetrics/metrics/issues/42
if atomic.CompareAndSwapUint32(&procSelfIOErrLogged, 0, 1) {
log.Printf("ERROR: metrics: cannot read process_io_* metrics from %q, so these metrics won't be updated until the error is fixed; "+
"see https://github.com/VictoriaMetrics/metrics/issues/42 ; The error: %s", ioFilepath, err)
}
}
getInt := func(s string) int64 {
n := strings.IndexByte(s, ' ')
if n < 0 {

28
vendor/github.com/VictoriaMetrics/metricsql/lexer.go generated vendored
View file

@ -289,6 +289,9 @@ func scanPositiveNumber(s string) (string, error) {
}
func scanNumMultiplier(s string) int {
if len(s) > 3 {
s = s[:3]
}
s = strings.ToLower(s)
switch true {
case strings.HasPrefix(s, "kib"):
@ -616,7 +619,6 @@ func scanSingleDuration(s string, canBeNegative bool) int {
if len(s) == 0 {
return -1
}
s = strings.ToLower(s)
i := 0
if s[0] == '-' && canBeNegative {
i++
@ -637,14 +639,26 @@ func scanSingleDuration(s string, canBeNegative bool) int {
return -1
}
}
switch s[i] {
switch unicode.ToLower(rune(s[i])) {
case 'm':
if i+1 < len(s) && s[i+1] == 's' {
// duration in ms
return i + 2
if i+1 < len(s) {
switch unicode.ToLower(rune(s[i+1])) {
case 's':
// duration in ms
return i + 2
case 'i', 'b':
// This is not a duration, but Mi or MB suffix.
// See parsePositiveNumber() and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3664
return -1
}
}
// duration in minutes
return i + 1
// Allow small m for durtion in minutes.
// Big M means 1e6.
// See parsePositiveNumber() and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3664
if s[i] == 'm' {
return i + 1
}
return -1
case 's', 'h', 'd', 'w', 'y', 'i':
return i + 1
default:

92
vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go generated vendored
View file

@ -4143,6 +4143,43 @@ var awsPartition = partition{
},
},
},
"cleanrooms": service{
Endpoints: serviceEndpoints{
endpointKey{
Region: "ap-northeast-1",
}: endpoint{},
endpointKey{
Region: "ap-northeast-2",
}: endpoint{},
endpointKey{
Region: "ap-southeast-1",
}: endpoint{},
endpointKey{
Region: "ap-southeast-2",
}: endpoint{},
endpointKey{
Region: "eu-central-1",
}: endpoint{},
endpointKey{
Region: "eu-north-1",
}: endpoint{},
endpointKey{
Region: "eu-west-1",
}: endpoint{},
endpointKey{
Region: "eu-west-2",
}: endpoint{},
endpointKey{
Region: "us-east-1",
}: endpoint{},
endpointKey{
Region: "us-east-2",
}: endpoint{},
endpointKey{
Region: "us-west-2",
}: endpoint{},
},
},
"cloud9": service{
Endpoints: serviceEndpoints{
endpointKey{
@ -22763,6 +22800,9 @@ var awsPartition = partition{
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "me-central-1",
}: endpoint{},
endpointKey{
Region: "me-south-1",
}: endpoint{},
@ -27093,6 +27133,15 @@ var awsPartition = partition{
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-me-central-1",
}: endpoint{
Hostname: "waf-regional-fips.me-central-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "me-central-1",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-me-south-1",
}: endpoint{
@ -27147,6 +27196,23 @@ var awsPartition = partition{
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "me-central-1",
}: endpoint{
Hostname: "waf-regional.me-central-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "me-central-1",
},
},
endpointKey{
Region: "me-central-1",
Variant: fipsVariant,
}: endpoint{
Hostname: "waf-regional-fips.me-central-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "me-central-1",
},
},
endpointKey{
Region: "me-south-1",
}: endpoint{
@ -27669,6 +27735,15 @@ var awsPartition = partition{
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-me-central-1",
}: endpoint{
Hostname: "wafv2-fips.me-central-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "me-central-1",
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "fips-me-south-1",
}: endpoint{
@ -27723,6 +27798,23 @@ var awsPartition = partition{
},
Deprecated: boxedTrue,
},
endpointKey{
Region: "me-central-1",
}: endpoint{
Hostname: "wafv2.me-central-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "me-central-1",
},
},
endpointKey{
Region: "me-central-1",
Variant: fipsVariant,
}: endpoint{
Hostname: "wafv2-fips.me-central-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "me-central-1",
},
},
endpointKey{
Region: "me-south-1",
}: endpoint{

2
vendor/github.com/aws/aws-sdk-go/aws/version.go generated vendored
View file

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.44.177"
const SDKVersion = "1.44.180"

47
vendor/golang.org/x/exp/slices/sort.go generated vendored
View file

@ -62,15 +62,22 @@ func IsSortedFunc[E any](x []E, less func(a, b E) bool) bool {
// sort order; it also returns a bool saying whether the target is really found
// in the slice. The slice must be sorted in increasing order.
func BinarySearch[E constraints.Ordered](x []E, target E) (int, bool) {
// search returns the leftmost position where f returns true, or len(x) if f
// returns false for all x. This is the insertion position for target in x,
// and could point to an element that's either == target or not.
pos := search(len(x), func(i int) bool { return x[i] >= target })
if pos >= len(x) || x[pos] != target {
return pos, false
} else {
return pos, true
// Inlining is faster than calling BinarySearchFunc with a lambda.
n := len(x)
// Define x[-1] < target and x[n] >= target.
// Invariant: x[i-1] < target, x[j] >= target.
i, j := 0, n
for i < j {
h := int(uint(i+j) >> 1) // avoid overflow when computing h
// i ≤ h < j
if x[h] < target {
i = h + 1 // preserves x[i-1] < target
} else {
j = h // preserves x[j] >= target
}
}
// i == j, x[i-1] < target, and x[j] (= x[i]) >= target => answer is i.
return i, i < n && x[i] == target
}
// BinarySearchFunc works like BinarySearch, but uses a custom comparison
@ -79,29 +86,21 @@ func BinarySearch[E constraints.Ordered](x []E, target E) (int, bool) {
// parameters: 0 if a == b, a negative number if a < b and a positive number if
// a > b.
func BinarySearchFunc[E any](x []E, target E, cmp func(E, E) int) (int, bool) {
pos := search(len(x), func(i int) bool { return cmp(x[i], target) >= 0 })
if pos >= len(x) || cmp(x[pos], target) != 0 {
return pos, false
} else {
return pos, true
}
}
func search(n int, f func(int) bool) int {
// Define f(-1) == false and f(n) == true.
// Invariant: f(i-1) == false, f(j) == true.
n := len(x)
// Define cmp(x[-1], target) < 0 and cmp(x[n], target) >= 0 .
// Invariant: cmp(x[i - 1], target) < 0, cmp(x[j], target) >= 0.
i, j := 0, n
for i < j {
h := int(uint(i+j) >> 1) // avoid overflow when computing h
// i ≤ h < j
if !f(h) {
i = h + 1 // preserves f(i-1) == false
if cmp(x[h], target) < 0 {
i = h + 1 // preserves cmp(x[i - 1], target) < 0
} else {
j = h // preserves f(j) == true
j = h // preserves cmp(x[j], target) >= 0
}
}
// i == j, f(i-1) == false, and f(j) (= f(i)) == true => answer is i.
return i
// i == j, cmp(x[i-1], target) < 0, and cmp(x[j], target) (= cmp(x[i], target)) >= 0 => answer is i.
return i, i < n && cmp(x[i], target) == 0
}
type sortedHint int // hint for pdqsort when choosing the pivot

2
vendor/google.golang.org/api/iamcredentials/v1/iamcredentials-gen.go generated vendored
View file

@ -1,4 +1,4 @@
// Copyright 2022 Google LLC.
// Copyright 2023 Google LLC.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

16
vendor/google.golang.org/api/internal/gensupport/resumable.go generated vendored
View file

@ -193,22 +193,28 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err
// Each chunk gets its own initialized-at-zero backoff and invocation ID.
bo := rx.Retry.backoff()
quitAfter := time.After(retryDeadline)
quitAfterTimer := time.NewTimer(retryDeadline)
rx.attempts = 1
rx.invocationID = uuid.New().String()
// Retry loop for a single chunk.
for {
pauseTimer := time.NewTimer(pause)
select {
case <-ctx.Done():
quitAfterTimer.Stop()
pauseTimer.Stop()
if err == nil {
err = ctx.Err()
}
return prepareReturn(resp, err)
case <-time.After(pause):
case <-quitAfter:
case <-pauseTimer.C:
quitAfterTimer.Stop()
case <-quitAfterTimer.C:
pauseTimer.Stop()
return prepareReturn(resp, err)
}
pauseTimer.Stop()
// Check for context cancellation or timeout once more. If more than one
// case in the select statement above was satisfied at the same time, Go
@ -217,13 +223,15 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err
// canceled before or the timeout was reached.
select {
case <-ctx.Done():
quitAfterTimer.Stop()
if err == nil {
err = ctx.Err()
}
return prepareReturn(resp, err)
case <-quitAfter:
case <-quitAfterTimer.C:
return prepareReturn(resp, err)
default:
quitAfterTimer.Stop()
}
resp, err = rx.transferChunk(ctx)

4
vendor/google.golang.org/api/internal/gensupport/send.go generated vendored
View file

@ -115,15 +115,17 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r
var errorFunc = retry.errorFunc()
for {
t := time.NewTimer(pause)
select {
case <-ctx.Done():
t.Stop()
// If we got an error and the context has been canceled, return an error acknowledging
// both the context cancelation and the service error.
if err != nil {
return resp, wrappedCallErr{ctx.Err(), err}
}
return resp, ctx.Err()
case <-time.After(pause):
case <-t.C:
}
if ctx.Err() != nil {

2
vendor/google.golang.org/api/internal/version.go generated vendored
View file

@ -5,4 +5,4 @@
package internal
// Version is the current tagged release of the library.
const Version = "0.106.0"
const Version = "0.107.0"

2
vendor/google.golang.org/api/storage/v1/storage-gen.go generated vendored
View file

@ -1,4 +1,4 @@
// Copyright 2022 Google LLC.
// Copyright 2023 Google LLC.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

12
vendor/modules.txt vendored
View file

@ -67,10 +67,10 @@ github.com/VictoriaMetrics/fastcache
github.com/VictoriaMetrics/fasthttp
github.com/VictoriaMetrics/fasthttp/fasthttputil
github.com/VictoriaMetrics/fasthttp/stackless
# github.com/VictoriaMetrics/metrics v1.23.0
# github.com/VictoriaMetrics/metrics v1.23.1
## explicit; go 1.15
github.com/VictoriaMetrics/metrics
# github.com/VictoriaMetrics/metricsql v0.51.1
# github.com/VictoriaMetrics/metricsql v0.51.2
## explicit; go 1.13
github.com/VictoriaMetrics/metricsql
github.com/VictoriaMetrics/metricsql/binaryop
@ -80,7 +80,7 @@ github.com/VividCortex/ewma
# github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
## explicit; go 1.15
github.com/alecthomas/units
# github.com/aws/aws-sdk-go v1.44.177
# github.com/aws/aws-sdk-go v1.44.180
## explicit; go 1.11
github.com/aws/aws-sdk-go/aws
github.com/aws/aws-sdk-go/aws/awserr
@ -528,7 +528,7 @@ go.uber.org/atomic
## explicit; go 1.18
go.uber.org/goleak
go.uber.org/goleak/internal/stack
# golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a
# golang.org/x/exp v0.0.0-20230113213754-f9f960f08ad4
## explicit; go 1.18
golang.org/x/exp/constraints
golang.org/x/exp/slices
@ -575,7 +575,7 @@ golang.org/x/time/rate
## explicit; go 1.17
golang.org/x/xerrors
golang.org/x/xerrors/internal
# google.golang.org/api v0.106.0
# google.golang.org/api v0.107.0
## explicit; go 1.19
google.golang.org/api/googleapi
google.golang.org/api/googleapi/transport
@ -608,7 +608,7 @@ google.golang.org/appengine/internal/socket
google.golang.org/appengine/internal/urlfetch
google.golang.org/appengine/socket
google.golang.org/appengine/urlfetch
# google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f
# google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5
## explicit; go 1.19
google.golang.org/genproto/googleapis/api
google.golang.org/genproto/googleapis/api/annotations