lib/storage: show top labels with the highest number of series in cardinality explorer

This commit is contained in:
Aliaksandr Valialkin 2022-06-14 16:32:38 +03:00
parent af3dc91a51
commit b6c1ca12b7
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
17 changed files with 240 additions and 184 deletions

View file

@ -268,6 +268,7 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric
VictoriaMetrics provides an ability to explore time series cardinality at `cardinality` tab in [vmui](#vmui) in the following ways:
- To identify metric names with the highest number of series.
- To idnetify labels with the highest number of series.
- To identify label=name pairs with the highest number of series.
- To identify labels with the highest number of unique values.
@ -275,8 +276,6 @@ By default cardinality explorer analyzes time series for the current date. It pr
By default all the time series for the selected date are analyzed. It is possible to narrow down the analysis to series
matching the specified [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors).
Cardinality explorer takes into account [deleted time series](#how-to-delete-time-series), because they stay in the inverted index for up to [-retentionPeriod](#retention). This means that the deleted time series take RAM, CPU, disk IO and disk space for the inverted index in the same way as other time series.
Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).

View file

@ -12,6 +12,7 @@ TSDBStatusResponse generates response for /api/v1/status/tsdb .
"totalSeries": {%dul= status.TotalSeries %},
"totalLabelValuePairs": {%dul= status.TotalLabelValuePairs %},
"seriesCountByMetricName":{%= tsdbStatusEntries(status.SeriesCountByMetricName) %},
"seriesCountByLabelName":{%= tsdbStatusEntries(status.SeriesCountByLabelName) %},
"seriesCountByLabelValuePair":{%= tsdbStatusEntries(status.SeriesCountByLabelValuePair) %},
"labelValueCountByLabelName":{%= tsdbStatusEntries(status.LabelValueCountByLabelName) %}
}

View file

@ -40,102 +40,106 @@ func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, status *storage.TSDBSta
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
streamtsdbStatusEntries(qw422016, status.SeriesCountByMetricName)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
qw422016.N().S(`,"seriesCountByLabelName":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelName)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
qw422016.N().S(`,"seriesCountByLabelValuePair":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelValuePair)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
qw422016.N().S(`,"labelValueCountByLabelName":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
streamtsdbStatusEntries(qw422016, status.LabelValueCountByLabelName)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
qw422016.N().S(`}`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
//line app/vmselect/prometheus/tsdb_status_response.qtpl:19
qt.Done()
//line app/vmselect/prometheus/tsdb_status_response.qtpl:19
//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
streamdumpQueryTrace(qw422016, qt)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:19
//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
qw422016.N().S(`}`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
func WriteTSDBStatusResponse(qq422016 qtio422016.Writer, status *storage.TSDBStatus, qt *querytracer.Tracer) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
StreamTSDBStatusResponse(qw422016, status, qt)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
func TSDBStatusResponse(status *storage.TSDBStatus, qt *querytracer.Tracer) string {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
WriteTSDBStatusResponse(qb422016, status, qt)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
return qs422016
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
func streamtsdbStatusEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
qw422016.N().S(`[`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:25
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
for i, e := range a {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:25
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
qw422016.N().S(`{"name":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
//line app/vmselect/prometheus/tsdb_status_response.qtpl:28
qw422016.N().Q(e.Name)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
//line app/vmselect/prometheus/tsdb_status_response.qtpl:28
qw422016.N().S(`,"value":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:28
//line app/vmselect/prometheus/tsdb_status_response.qtpl:29
qw422016.N().D(int(e.Count))
//line app/vmselect/prometheus/tsdb_status_response.qtpl:28
//line app/vmselect/prometheus/tsdb_status_response.qtpl:29
qw422016.N().S(`}`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
if i+1 < len(a) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
qw422016.N().S(`,`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
//line app/vmselect/prometheus/tsdb_status_response.qtpl:32
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
//line app/vmselect/prometheus/tsdb_status_response.qtpl:32
qw422016.N().S(`]`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
func writetsdbStatusEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
streamtsdbStatusEntries(qw422016, a)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
func tsdbStatusEntries(a []storage.TopHeapEntry) string {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
writetsdbStatusEntries(qb422016, a)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
return qs422016
//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
}

View file

@ -1,12 +1,12 @@
{
"files": {
"main.css": "./static/css/main.7e6d0c89.css",
"main.js": "./static/js/main.a6bca65f.js",
"main.js": "./static/js/main.f7185a13.js",
"static/js/27.939f971b.chunk.js": "./static/js/27.939f971b.chunk.js",
"index.html": "./index.html"
},
"entrypoints": [
"static/css/main.7e6d0c89.css",
"static/js/main.a6bca65f.js"
"static/js/main.f7185a13.js"
]
}

View file

@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.a6bca65f.js"></script><link href="./static/css/main.7e6d0c89.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.f7185a13.js"></script><link href="./static/css/main.7e6d0c89.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

View file

@ -54,12 +54,12 @@ const CardinalityConfigurator: FC<CardinalityConfiguratorProps> = ({
<QueryEditor
query={query} index={0} autocomplete={autocomplete} queryOptions={queryOptions}
error={error} setHistoryIndex={onSetHistory} runQuery={onRunQuery} setQuery={onSetQuery}
label={"Arbitrary time series selector"}
label={"Time series selector"}
/>
<Box display="flex" alignItems="center">
<Box ml={2}>
<TextField
label="Number of top entries"
label="Number of entries per table"
type="number"
size="small"
variant="outlined"
@ -82,7 +82,7 @@ const CardinalityConfigurator: FC<CardinalityConfiguratorProps> = ({
</Box>
</Box>
<Box>
Analyzed <b>{totalSeries}</b> series and <b>{totalLabelValuePairs}</b> label=value pairs
Analyzed <b>{totalSeries}</b> series with <b>{totalLabelValuePairs}</b> label=value pairs
at <b>{date}</b> {match && <span>for series selector <b>{match}</b></span>}. Show top {topN} entries per table.
</Box>
</Box>;

View file

@ -3,12 +3,10 @@ import {SyntheticEvent} from "react";
import {Alert} from "@mui/material";
import {useFetchQuery} from "../../hooks/useCardinalityFetch";
import {
LABEL_VALUE_PAIR_CONTENT_TITLE,
LABEL_VALUE_PAIRS_TABLE_HEADERS,
LABEL_WITH_UNIQUE_VALUES_TABLE_HEADERS,
LABELS_CONTENT_TITLE, METRICS_TABLE_HEADERS,
SERIES_CONTENT_TITLE,
SPINNER_TITLE,
METRIC_NAMES_HEADERS,
LABEL_NAMES_HEADERS,
LABEL_VALUE_PAIRS_HEADERS,
LABELS_WITH_UNIQUE_VALUES_HEADERS,
spinnerContainerStyles
} from "./consts";
import {defaultProperties, queryUpdater} from "./helpers";
@ -76,7 +74,7 @@ const CardinalityPanel: FC = () => {
height={"800px"}
containerStyles={spinnerContainerStyles("100%")}
title={<Alert color="error" severity="error" sx={{whiteSpace: "pre-wrap", mt: 2}}>
{SPINNER_TITLE}
Please wait while cardinality stats is calculated. This may take some time if the db contains big number of time series
</Alert>}
/>}
<CardinalityConfigurator error={configError} query={query} onRunQuery={onRunQuery} onSetQuery={onSetQuery}
@ -84,6 +82,7 @@ const CardinalityPanel: FC = () => {
totalSeries={tsdbStatus.totalSeries} totalLabelValuePairs={tsdbStatus.totalLabelValuePairs}/>
{error && <Alert color="error" severity="error" sx={{whiteSpace: "pre-wrap", mt: 2}}>{error}</Alert>}
<MetricsContent
sectionTitle={"Metric names with the highest number of series"}
activeTab={stateTabs.seriesCountByMetricName}
rows={tsdbStatus.seriesCountByMetricName as unknown as Data[]}
onChange={handleTabChange}
@ -92,10 +91,22 @@ const CardinalityPanel: FC = () => {
chartContainer={defaultProps.containerRefs.seriesCountByMetricName}
totalSeries={tsdbStatus.totalSeries}
tabId={"seriesCountByMetricName"}
sectionTitle={SERIES_CONTENT_TITLE}
tableHeaderCells={METRICS_TABLE_HEADERS}
tableHeaderCells={METRIC_NAMES_HEADERS}
/>
<MetricsContent
sectionTitle={"Labels with the highest number of series"}
activeTab={stateTabs.seriesCountByLabelName}
rows={tsdbStatus.seriesCountByLabelName as unknown as Data[]}
onChange={handleTabChange}
onActionClick={handleFilterClick("seriesCountByLabelName")}
tabs={defaultProps.tabs.seriesCountByLabelName}
chartContainer={defaultProps.containerRefs.seriesCountByLabelName}
totalSeries={tsdbStatus.totalSeries}
tabId={"seriesCountByLabelName"}
tableHeaderCells={LABEL_NAMES_HEADERS}
/>
<MetricsContent
sectionTitle={"Label=value pairs with the highest number of series"}
activeTab={stateTabs.seriesCountByLabelValuePair}
rows={tsdbStatus.seriesCountByLabelValuePair as unknown as Data[]}
onChange={handleTabChange}
@ -104,10 +115,10 @@ const CardinalityPanel: FC = () => {
chartContainer={defaultProps.containerRefs.seriesCountByLabelValuePair}
totalSeries={tsdbStatus.totalSeries}
tabId={"seriesCountByLabelValuePair"}
sectionTitle={LABEL_VALUE_PAIR_CONTENT_TITLE}
tableHeaderCells={LABEL_VALUE_PAIRS_TABLE_HEADERS}
tableHeaderCells={LABEL_VALUE_PAIRS_HEADERS}
/>
<MetricsContent
sectionTitle={"Labels with the highest number of unique values"}
activeTab={stateTabs.labelValueCountByLabelName}
rows={tsdbStatus.labelValueCountByLabelName as unknown as Data[]}
onChange={handleTabChange}
@ -116,8 +127,7 @@ const CardinalityPanel: FC = () => {
chartContainer={defaultProps.containerRefs.labelValueCountByLabelName}
totalSeries={-1}
tabId={"labelValueCountByLabelName"}
sectionTitle={LABELS_CONTENT_TITLE}
tableHeaderCells={LABEL_WITH_UNIQUE_VALUES_TABLE_HEADERS}
tableHeaderCells={LABELS_WITH_UNIQUE_VALUES_HEADERS}
/>
</>
);

View file

@ -1,10 +1,64 @@
import {HeadCell} from "../Table/types";
export const METRICS_TABLE_HEADERS = [
export const METRIC_NAMES_HEADERS = [
{
disablePadding: false,
id: "name",
label: "Metrics name",
label: "Metric name",
numeric: false,
},
{
disablePadding: false,
id: "value",
label: "Number of series",
numeric: false,
},
{
disablePadding: false,
id: "percentage",
label: "Percent of series",
numeric: false,
},
{
disablePadding: false,
id: "action",
label: "Action",
numeric: false,
}
] as HeadCell[];
export const LABEL_NAMES_HEADERS = [
{
disablePadding: false,
id: "name",
label: "Label name",
numeric: false,
},
{
disablePadding: false,
id: "value",
label: "Number of series",
numeric: false,
},
{
disablePadding: false,
id: "percentage",
label: "Percent of series",
numeric: false,
},
{
disablePadding: false,
id: "action",
label: "Action",
numeric: false,
}
] as HeadCell[];
export const LABEL_VALUE_PAIRS_HEADERS = [
{
disablePadding: false,
id: "name",
label: "Label=value pair",
numeric: false,
},
{
@ -27,34 +81,7 @@ export const METRICS_TABLE_HEADERS = [
}
]as HeadCell[];
export const LABEL_VALUE_PAIRS_TABLE_HEADERS = [
{
disablePadding: false,
id: "name",
label: "Lable=value pair",
numeric: false,
},
{
disablePadding: false,
id: "value",
label: "Number of series",
numeric: false,
},
{
disablePadding: false,
id: "percentage",
label: "Percent of total label value pairs",
numeric: false,
},
{
disablePadding: false,
id: "action",
label: "Action",
numeric: false,
}
]as HeadCell[];
export const LABEL_WITH_UNIQUE_VALUES_TABLE_HEADERS = [
export const LABELS_WITH_UNIQUE_VALUES_HEADERS = [
{
disablePadding: false,
id: "name",
@ -86,9 +113,3 @@ export const spinnerContainerStyles = (height: string) => {
zIndex: 1000,
};
};
export const SPINNER_TITLE = "Please wait while cardinality stats is calculated. " +
"This may take some time if the db contains big number of time series";
export const SERIES_CONTENT_TITLE = "Metric names with the highest number of series";
export const LABEL_VALUE_PAIR_CONTENT_TITLE = "Label=value pairs with the highest number of series";
export const LABELS_CONTENT_TITLE = "Labels with the highest number of unique values";

View file

@ -2,16 +2,17 @@ import {Containers, DefaultState, QueryUpdater, Tabs, TSDBStatus} from "./types"
import {useRef} from "preact/compat";
export const queryUpdater: QueryUpdater = {
labelValueCountByLabelName: (query: string): string => `{${query}!=""}`,
seriesCountByMetricName: (query: string): string => {
return getSeriesSelector("__name__", query);
},
seriesCountByLabelName: (query: string): string => `{${query}!=""}`,
seriesCountByLabelValuePair: (query: string): string => {
const a = query.split("=");
const label = a[0];
const value = a.slice(1).join("=");
return getSeriesSelector(label, value);
},
seriesCountByMetricName: (query: string): string => {
return getSeriesSelector("__name__", query);
},
labelValueCountByLabelName: (query: string): string => `{${query}!=""}`,
};
const getSeriesSelector = (label: string, value: string): string => {

View file

@ -1,11 +1,12 @@
import {MutableRef} from "preact/hooks";
export interface TSDBStatus {
labelValueCountByLabelName: TopHeapEntry[];
seriesCountByLabelValuePair: TopHeapEntry[];
seriesCountByMetricName: TopHeapEntry[];
totalSeries: number;
totalLabelValuePairs: number;
seriesCountByMetricName: TopHeapEntry[];
seriesCountByLabelName: TopHeapEntry[];
seriesCountByLabelValuePair: TopHeapEntry[];
labelValueCountByLabelName: TopHeapEntry[];
}
export interface TopHeapEntry {
@ -18,19 +19,22 @@ export type QueryUpdater = {
}
export interface Tabs {
labelValueCountByLabelName: string[];
seriesCountByLabelValuePair: string[];
seriesCountByMetricName: string[];
seriesCountByLabelName: string[];
seriesCountByLabelValuePair: string[];
labelValueCountByLabelName: string[];
}
export interface Containers<T> {
labelValueCountByLabelName: MutableRef<T>;
seriesCountByLabelValuePair: MutableRef<T>;
seriesCountByMetricName: MutableRef<T>;
seriesCountByLabelName: MutableRef<T>;
seriesCountByLabelValuePair: MutableRef<T>;
labelValueCountByLabelName: MutableRef<T>;
}
export interface DefaultState {
labelValueCountByLabelName: number;
seriesCountByLabelValuePair: number;
seriesCountByMetricName: number;
seriesCountByLabelName: number;
seriesCountByLabelValuePair: number;
labelValueCountByLabelName: number;
}

View file

@ -12,6 +12,7 @@ const defaultTSDBStatus = {
totalSeries: 0,
totalLabelValuePairs: 0,
seriesCountByMetricName: [],
seriesCountByLabelName: [],
seriesCountByLabelValuePair: [],
labelValueCountByLabelName: [],
};

View file

@ -268,6 +268,7 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric
VictoriaMetrics provides an ability to explore time series cardinality at `cardinality` tab in [vmui](#vmui) in the following ways:
- To identify metric names with the highest number of series.
- To idnetify labels with the highest number of series.
- To identify label=name pairs with the highest number of series.
- To identify labels with the highest number of unique values.
@ -275,8 +276,6 @@ By default cardinality explorer analyzes time series for the current date. It pr
By default all the time series for the selected date are analyzed. It is possible to narrow down the analysis to series
matching the specified [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors).
Cardinality explorer takes into account [deleted time series](#how-to-delete-time-series), because they stay in the inverted index for up to [-retentionPeriod](#retention). This means that the deleted time series take RAM, CPU, disk IO and disk space for the inverted index in the same way as other time series.
Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).

View file

@ -272,6 +272,7 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric
VictoriaMetrics provides an ability to explore time series cardinality at `cardinality` tab in [vmui](#vmui) in the following ways:
- To identify metric names with the highest number of series.
- To idnetify labels with the highest number of series.
- To identify label=name pairs with the highest number of series.
- To identify labels with the highest number of unique values.
@ -279,8 +280,6 @@ By default cardinality explorer analyzes time series for the current date. It pr
By default all the time series for the selected date are analyzed. It is possible to narrow down the analysis to series
matching the specified [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors).
Cardinality explorer takes into account [deleted time series](#how-to-delete-time-series), because they stay in the inverted index for up to [-retentionPeriod](#retention). This means that the deleted time series take RAM, CPU, disk IO and disk space for the inverted index in the same way as other time series.
Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).

View file

@ -834,10 +834,7 @@ func (is *indexSearch) searchLabelNamesWithFiltersOnDate(qt *querytracer.Tracer,
if err := mp.Init(item, nsPrefixExpected); err != nil {
return err
}
if mp.IsDeletedTag(dmis) {
continue
}
if mp.GetMatchingSeriesCount(filter) == 0 {
if mp.GetMatchingSeriesCount(filter, dmis) == 0 {
continue
}
labelName := mp.Tag.Key
@ -1000,10 +997,7 @@ func (is *indexSearch) searchLabelValuesWithFiltersOnDate(qt *querytracer.Tracer
if err := mp.Init(item, nsPrefixExpected); err != nil {
return err
}
if mp.IsDeletedTag(dmis) {
continue
}
if mp.GetMatchingSeriesCount(filter) == 0 {
if mp.GetMatchingSeriesCount(filter, dmis) == 0 {
continue
}
labelValue := mp.Tag.Value
@ -1150,7 +1144,7 @@ func (is *indexSearch) searchTagValueSuffixesForPrefix(tvss map[string]struct{},
if err := mp.Init(item, nsPrefix); err != nil {
return err
}
if mp.IsDeletedTag(dmis) {
if mp.GetMatchingSeriesCount(nil, dmis) == 0 {
continue
}
tagValue := mp.Tag.Value
@ -1284,12 +1278,14 @@ func (is *indexSearch) getTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, t
ts := &is.ts
kb := &is.kb
mp := &is.mp
thLabelValueCountByLabelName := newTopHeap(topN)
thSeriesCountByLabelValuePair := newTopHeap(topN)
dmis := is.db.s.getDeletedMetricIDs()
thSeriesCountByMetricName := newTopHeap(topN)
var tmp, labelName, labelNameValue []byte
thSeriesCountByLabelName := newTopHeap(topN)
thSeriesCountByLabelValuePair := newTopHeap(topN)
thLabelValueCountByLabelName := newTopHeap(topN)
var tmp, prevLabelName, prevLabelValuePair []byte
var labelValueCountByLabelName, seriesCountByLabelValuePair uint64
var totalSeries, totalLabelValuePairs uint64
var totalSeries, labelSeries, totalLabelValuePairs uint64
nameEqualBytes := []byte("__name__=")
loopsPaceLimiter := 0
@ -1314,69 +1310,80 @@ func (is *indexSearch) getTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, t
if err := mp.Init(item, nsPrefixExpected); err != nil {
return nil, err
}
matchingSeriesCount := mp.GetMatchingSeriesCount(filter)
matchingSeriesCount := mp.GetMatchingSeriesCount(filter, dmis)
if matchingSeriesCount == 0 {
// Skip rows without matching metricIDs.
continue
}
tmp = append(tmp[:0], mp.Tag.Key...)
tagKey := tmp
if isArtificialTagKey(tagKey) {
labelName := tmp
if isArtificialTagKey(labelName) {
// Skip artificially created tag keys.
kb.B = append(kb.B[:0], prefix...)
if len(tagKey) > 0 && tagKey[0] == compositeTagKeyPrefix {
if len(labelName) > 0 && labelName[0] == compositeTagKeyPrefix {
kb.B = append(kb.B, compositeTagKeyPrefix)
} else {
kb.B = marshalTagValue(kb.B, tagKey)
kb.B = marshalTagValue(kb.B, labelName)
}
kb.B[len(kb.B)-1]++
ts.Seek(kb.B)
continue
}
if len(tagKey) == 0 {
tagKey = append(tagKey, "__name__"...)
tmp = tagKey
if len(labelName) == 0 {
labelName = append(labelName, "__name__"...)
tmp = labelName
}
if string(labelName) == "__name__" {
totalSeries += uint64(matchingSeriesCount)
}
tmp = append(tmp, '=')
tmp = append(tmp, mp.Tag.Value...)
tagKeyValue := tmp
if string(tagKey) == "__name__" {
totalSeries += uint64(matchingSeriesCount)
labelValuePair := tmp
if len(prevLabelName) == 0 {
prevLabelName = append(prevLabelName[:0], labelName...)
}
if !bytes.Equal(tagKey, labelName) {
thLabelValueCountByLabelName.pushIfNonEmpty(labelName, labelValueCountByLabelName)
if string(labelName) != string(prevLabelName) {
thLabelValueCountByLabelName.push(prevLabelName, labelValueCountByLabelName)
thSeriesCountByLabelName.push(prevLabelName, labelSeries)
labelSeries = 0
labelValueCountByLabelName = 0
labelName = append(labelName[:0], tagKey...)
prevLabelName = append(prevLabelName[:0], labelName...)
}
if !bytes.Equal(tagKeyValue, labelNameValue) {
thSeriesCountByLabelValuePair.pushIfNonEmpty(labelNameValue, seriesCountByLabelValuePair)
if bytes.HasPrefix(labelNameValue, nameEqualBytes) {
thSeriesCountByMetricName.pushIfNonEmpty(labelNameValue[len(nameEqualBytes):], seriesCountByLabelValuePair)
if len(prevLabelValuePair) == 0 {
prevLabelValuePair = append(prevLabelValuePair[:0], labelValuePair...)
labelValueCountByLabelName++
}
if string(labelValuePair) != string(prevLabelValuePair) {
thSeriesCountByLabelValuePair.push(prevLabelValuePair, seriesCountByLabelValuePair)
if bytes.HasPrefix(prevLabelValuePair, nameEqualBytes) {
thSeriesCountByMetricName.push(prevLabelValuePair[len(nameEqualBytes):], seriesCountByLabelValuePair)
}
seriesCountByLabelValuePair = 0
labelValueCountByLabelName++
labelNameValue = append(labelNameValue[:0], tagKeyValue...)
prevLabelValuePair = append(prevLabelValuePair[:0], labelValuePair...)
}
// Take into account deleted timeseries too.
// It is OK if series can be counted multiple times in rare cases -
// the returned number is an estimation.
labelSeries += uint64(matchingSeriesCount)
seriesCountByLabelValuePair += uint64(matchingSeriesCount)
totalLabelValuePairs += uint64(matchingSeriesCount)
}
if err := ts.Error(); err != nil {
return nil, fmt.Errorf("error when counting time series by metric names: %w", err)
}
thLabelValueCountByLabelName.pushIfNonEmpty(labelName, labelValueCountByLabelName)
thSeriesCountByLabelValuePair.pushIfNonEmpty(labelNameValue, seriesCountByLabelValuePair)
if bytes.HasPrefix(labelNameValue, nameEqualBytes) {
thSeriesCountByMetricName.pushIfNonEmpty(labelNameValue[len(nameEqualBytes):], seriesCountByLabelValuePair)
thLabelValueCountByLabelName.push(prevLabelName, labelValueCountByLabelName)
thSeriesCountByLabelName.push(prevLabelName, labelSeries)
thSeriesCountByLabelValuePair.push(prevLabelValuePair, seriesCountByLabelValuePair)
if bytes.HasPrefix(prevLabelValuePair, nameEqualBytes) {
thSeriesCountByMetricName.push(prevLabelValuePair[len(nameEqualBytes):], seriesCountByLabelValuePair)
}
status := &TSDBStatus{
SeriesCountByMetricName: thSeriesCountByMetricName.getSortedResult(),
LabelValueCountByLabelName: thLabelValueCountByLabelName.getSortedResult(),
SeriesCountByLabelValuePair: thSeriesCountByLabelValuePair.getSortedResult(),
TotalSeries: totalSeries,
TotalLabelValuePairs: totalLabelValuePairs,
SeriesCountByMetricName: thSeriesCountByMetricName.getSortedResult(),
SeriesCountByLabelName: thSeriesCountByLabelName.getSortedResult(),
SeriesCountByLabelValuePair: thSeriesCountByLabelValuePair.getSortedResult(),
LabelValueCountByLabelName: thLabelValueCountByLabelName.getSortedResult(),
}
return status, nil
}
@ -1388,8 +1395,9 @@ type TSDBStatus struct {
TotalSeries uint64
TotalLabelValuePairs uint64
SeriesCountByMetricName []TopHeapEntry
LabelValueCountByLabelName []TopHeapEntry
SeriesCountByLabelName []TopHeapEntry
SeriesCountByLabelValuePair []TopHeapEntry
LabelValueCountByLabelName []TopHeapEntry
}
func (status *TSDBStatus) hasEntries() bool {
@ -1415,7 +1423,7 @@ type TopHeapEntry struct {
Count uint64
}
func (th *topHeap) pushIfNonEmpty(name []byte, count uint64) {
func (th *topHeap) push(name []byte, count uint64) {
if count == 0 {
return
}
@ -3108,39 +3116,27 @@ func (mp *tagToMetricIDsRowParser) ParseMetricIDs() {
mp.metricIDsParsed = true
}
// GetMatchingSeriesCount returns the number of series in mp, which match metricIDs from the given filter.
// GetMatchingSeriesCount returns the number of series in mp, which match metricIDs from the given filter
// and do not match metricIDs from negativeFilter.
//
// if filter is empty, then all series in mp are taken into account.
func (mp *tagToMetricIDsRowParser) GetMatchingSeriesCount(filter *uint64set.Set) int {
if filter == nil {
func (mp *tagToMetricIDsRowParser) GetMatchingSeriesCount(filter, negativeFilter *uint64set.Set) int {
if filter == nil && negativeFilter.Len() == 0 {
return mp.MetricIDsLen()
}
mp.ParseMetricIDs()
n := 0
for _, metricID := range mp.MetricIDs {
if filter.Has(metricID) {
if filter != nil && !filter.Has(metricID) {
continue
}
if !negativeFilter.Has(metricID) {
n++
}
}
return n
}
// IsDeletedTag verifies whether the tag from mp is deleted according to dmis.
//
// dmis must contain deleted MetricIDs.
func (mp *tagToMetricIDsRowParser) IsDeletedTag(dmis *uint64set.Set) bool {
if dmis.Len() == 0 {
return false
}
mp.ParseMetricIDs()
for _, metricID := range mp.MetricIDs {
if !dmis.Has(metricID) {
return false
}
}
return true
}
func mergeTagToMetricIDsRows(data []byte, items []mergeset.Item) ([]byte, []mergeset.Item) {
data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixTagToMetricIDs)
data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixDateTagToMetricIDs)

View file

@ -1824,6 +1824,27 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
}
expectedSeriesCountByLabelName := []TopHeapEntry{
{
Name: "__name__",
Count: 1000,
},
{
Name: "constant",
Count: 1000,
},
{
Name: "day",
Count: 1000,
},
{
Name: "uniqueid",
Count: 1000,
},
}
if !reflect.DeepEqual(status.SeriesCountByLabelName, expectedSeriesCountByLabelName) {
t.Fatalf("unexpected SeriesCountByLabelName;\ngot\n%v\nwant\n%v", status.SeriesCountByLabelName, expectedSeriesCountByLabelName)
}
expectedLabelValueCountByLabelName := []TopHeapEntry{
{
Name: "uniqueid",